moves scan classes into one namespace, makes file scanner work, adds user secrets support to config

.gitignore

@@ -303,8 +303,6 @@ node_modules/
 *.dsp
 
 # Visual Studio 6 technical files
-*.ncb
-*.aps
 
 # Visual Studio LightSwitch build output
 **/*.HTMLClient/GeneratedArtifacts
@@ -400,4 +398,7 @@ FodyWeavers.xsd
 
 # sqlite databases
 *.db
+*.db-shm
+*.db-wal
 *.sqlite
+/.idea/.idea.PiwigoDirectorySync/.idea/dataSources.xml

PiwigoDirectorySync/AppSettings.cs

@@ -4,14 +4,18 @@ namespace PiwigoDirectorySync;
 
 public static class AppSettings
 {
+    public static readonly IReadOnlySet<string> SupportedExtensions = new HashSet<string> { "jpg", "jpeg", "png" };
+
     public static IConfigurationRoot Config { get; } = new ConfigurationBuilder().SetBasePath(AppContext.BaseDirectory)
         .AddJsonFile("appsettings.json", true)
         .AddJsonFile(Path.Combine(Environment.CurrentDirectory, "appsettings.json"), true)
         .AddJsonFile("/etc/PiwigoDirectorySync/appsettings.json", true)
+        .AddUserSecrets<Program>(true)
         .AddEnvironmentVariables()
         .Build();
 
     public static Settings Settings { get; } = Config.GetSection("settings").Get<Settings>() ?? throw new InvalidOperationException("Could not parse settings");
 
-    public static string ConnectionString => Config.GetConnectionString(Settings.DbProvider) ?? throw new InvalidOperationException($"Could not find connection string for provider {Settings.DbProvider}");
+    public static string ConnectionString =>
+        Config.GetConnectionString(Settings.DbProvider) ?? throw new InvalidOperationException($"Could not find connection string for provider {Settings.DbProvider}");
 }

PiwigoDirectorySync/Commands/Scan/FileIndexer.cs

@@ -0,0 +1,36 @@
+using System.Threading.Channels;
+using NLog;
+
+namespace PiwigoDirectorySync.Commands.Scan;
+
+public class FileIndexer
+{
+    private static readonly ILogger Logger = LogManager.GetCurrentClassLogger();
+
+    private readonly Channel<string> _fileQueue;
+    private readonly int _piwigoServerId;
+
+    public FileIndexer(Channel<string> fileQueue, int piwigoServerId)
+    {
+        _fileQueue = fileQueue ?? throw new ArgumentNullException(nameof(fileQueue));
+        _piwigoServerId = piwigoServerId;
+    }
+
+    public int TotalFilesScanned { get; private set; }
+
+    public async Task StartProcessingAsync(CancellationToken ct)
+    {
+        await foreach (var filePath in _fileQueue.Reader.ReadAllAsync(ct))
+        {
+            try
+            {
+                Logger.Info($"Indexing file {filePath}");
+                TotalFilesScanned++;
+            }
+            catch (Exception ex)
+            {
+                Logger.Error(ex, $"could not delete file {filePath}");
+            }
+        }
+    }
+}

PiwigoDirectorySync/Commands/Scan/FileSystemScanner.cs

@@ -0,0 +1,76 @@
+using System.Threading.Channels;
+using NLog;
+using PiwigoDirectorySync.Persistence;
+
+namespace PiwigoDirectorySync.Commands.Scan;
+
+public class FileSystemScanner
+{
+    private static readonly ILogger Logger = LogManager.GetCurrentClassLogger();
+
+    private readonly Channel<string> _fileQueue;
+    private readonly int _piwigoServerId;
+
+    public FileSystemScanner(Channel<string> fileQueue, int piwigoServerId)
+    {
+        _fileQueue = fileQueue ?? throw new ArgumentNullException(nameof(fileQueue));
+        _piwigoServerId = piwigoServerId;
+    }
+
+    public async Task ScanAsync(CancellationToken ct)
+    {
+        await using var db = new PersistenceContext();
+
+        var piwigoServer = await db.PiwigoServers.GetByIdAsync(_piwigoServerId, ct);
+        Logger.Info($"Scanning files for piwigo server {piwigoServer.Name} in directory {piwigoServer.RootDirectory}");
+
+        await ScanRootDirectory(new DirectoryInfo(piwigoServer.RootDirectory), ct);
+    }
+
+    private async ValueTask ScanRootDirectory(DirectoryInfo directory, CancellationToken ct)
+    {
+        Logger.Info($"Scanning root directory {directory.FullName} for sidecars to delete");
+        var parallelOptions = new ParallelOptions
+        {
+            CancellationToken = ct
+        };
+        await Parallel.ForEachAsync(GetDirectories(directory), parallelOptions, FindAndEnqueueFilesToAdd);
+    }
+
+    private async ValueTask FindAndEnqueueFilesToAdd(DirectoryInfo directory, CancellationToken ct)
+    {
+        try
+        {
+            Logger.Info($"Scanning directory {directory.FullName} for images");
+
+            var imageFiles = AppSettings.SupportedExtensions.SelectMany(ext => directory.GetFiles($"*.{ext}", SearchOption.TopDirectoryOnly))
+                .Select(f => f.FullName)
+                .ToHashSet(StringComparer.OrdinalIgnoreCase);
+
+            if (!imageFiles.Any())
+            {
+                Logger.Debug($"No iamges in {directory.FullName} found, skipping");
+                return;
+            }
+
+            foreach (var imageFile in imageFiles.Select(f => new FileInfo(f)))
+            {
+                Logger.Debug($"Found image {imageFile.FullName}, enqueue index");
+                await _fileQueue.Writer.WriteAsync(imageFile.FullName, ct);
+            }
+        }
+        catch (Exception ex)
+        {
+            Logger.Error(ex, $"could not scan directory {directory.FullName}");
+        }
+    }
+
+    private static IEnumerable<DirectoryInfo> GetDirectories(DirectoryInfo directoryInfo)
+    {
+        yield return directoryInfo;
+        foreach (var directory in directoryInfo.EnumerateDirectories().SelectMany(GetDirectories))
+        {
+            yield return directory;
+        }
+    }
+}

PiwigoDirectorySync/Commands/Scan/ScanCommand.cs

@@ -1,10 +1,9 @@
 using System.Diagnostics;
 using System.Threading.Channels;
 using NLog;
-using PiwigoDirectorySync.Service;
 using Spectre.Console.Cli;
 
-namespace PiwigoDirectorySync.Commands;
+namespace PiwigoDirectorySync.Commands.Scan;
 
 public class ScanCommand : AsyncCommand<ScanSettings>
 {
@@ -19,10 +18,10 @@ public class ScanCommand : AsyncCommand<ScanSettings>
 
         var fileQueue = Channel.CreateUnbounded<string>();
 
-        var indexer = new FileIndexer(fileQueue);
+        var indexer = new FileIndexer(fileQueue, settings.PiwigoServerId);
         var indexerTask = indexer.StartProcessingAsync(cancellationTokenSource.Token);
 
-        var scanner = new FileScanner(fileQueue);
+        var scanner = new FileSystemScanner(fileQueue, settings.PiwigoServerId);
         await scanner.ScanAsync(cancellationTokenSource.Token);
 
         fileQueue.Writer.Complete();

PiwigoDirectorySync/Commands/Scan/ScanSettings.cs

@@ -0,0 +1,16 @@
+using System.ComponentModel;
+using System.Diagnostics.CodeAnalysis;
+using Spectre.Console.Cli;
+
+namespace PiwigoDirectorySync.Commands.Scan;
+
+[SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")]
+public class ScanSettings : CommandSettings
+{
+    [CommandArgument(0, "<PiwigoServerId>")]
+    public int PiwigoServerId { get; set; }
+
+    [CommandOption("-d|--mark-for-delete")]
+    [DefaultValue(false)]
+    public bool MarkForDelete { get; set; }
+}

PiwigoDirectorySync/Commands/ScanSettings.cs

@@ -1,7 +0,0 @@
-using Spectre.Console.Cli;
-
-namespace PiwigoDirectorySync.Commands;
-
-public class ScanSettings : CommandSettings
-{
-}

PiwigoDirectorySync/Persistence/ExtensionMethods.cs

@@ -0,0 +1,11 @@
+using Microsoft.EntityFrameworkCore;
+
+namespace PiwigoDirectorySync.Persistence;
+
+public static class ExtensionMethods
+{
+    public static Task<PiwigoServerEntity> GetByIdAsync(this DbSet<PiwigoServerEntity> dbSet, int serverId, CancellationToken ct)
+    {
+        return dbSet.Where(a => a.Id == serverId).FirstAsync(ct);
+    }
+}

PiwigoDirectorySync/Persistence/PersistenceContext.cs

@@ -4,9 +4,9 @@ namespace PiwigoDirectorySync.Persistence;
 
 public class PersistenceContext : DbContext
 {
-    public DbSet<PiwigoServerEntity> PiwigoServers { get; set; }
+    public DbSet<PiwigoServerEntity> PiwigoServers { get; set; } = null!;
-    public DbSet<AlbumEntity> PiwigoAlbums { get; set; }
+    public DbSet<AlbumEntity> PiwigoAlbums { get; set; } = null!;
-    public DbSet<ImageEntity> PiwigoImages { get; set; }
+    public DbSet<ImageEntity> PiwigoImages { get; set; } = null!;
 
     protected override void OnConfiguring(DbContextOptionsBuilder options)
     {

PiwigoDirectorySync/PiwigoDirectorySync.csproj

@@ -12,6 +12,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="7.0.0" />
         <PackageReference Include="Piwigo.Client" Version="0.1.0.17"/>
         <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.0"/>
         <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0"/>
@@ -40,7 +41,10 @@
         <None Update="appsettings.json">
             <CopyToOutputDirectory>Always</CopyToOutputDirectory>
         </None>
+
     </ItemGroup>
 
 
+
+
 </Project>

PiwigoDirectorySync/Program.cs

@@ -1,7 +1,7 @@
 using NLog;
 using NLog.Extensions.Logging;
 using PiwigoDirectorySync;
-using PiwigoDirectorySync.Commands;
+using PiwigoDirectorySync.Commands.Scan;
 using Spectre.Console.Cli;
 
 LogManager.Configuration = new NLogLoggingConfiguration(AppSettings.Config.GetSection("NLog"));
@@ -10,6 +10,14 @@ var logger = logFactory.GetCurrentClassLogger();
 
 logger.Info("Starting command app");
 var app = new CommandApp();
-app.Configure(c => { c.AddCommand<ScanCommand>("scan"); });
+app.Configure(config =>
+{
+#if DEBUG
+    config.PropagateExceptions();
+    config.ValidateExamples();
+#endif
+
+    config.AddCommand<ScanCommand>("scan");
+});
 
 return app.Run(args);

PiwigoDirectorySync/Properties/launchSettings.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "PiwigoDirectorySync": {
+      "commandName": "Project",
+      "commandLineArgs": "scan 1",
+      "environmentVariables": {
+      }
+    }
+  }
+}

PiwigoDirectorySync/Service/FileIndexer.cs

@@ -1,17 +0,0 @@
-using System.Threading.Channels;
-
-namespace PiwigoDirectorySync.Service;
-
-public class FileIndexer
-{
-    private readonly Channel<string> _fileQueue;
-
-    public FileIndexer(Channel<string> fileQueue)
-    {
-        _fileQueue = fileQueue ?? throw new ArgumentNullException(nameof(fileQueue));
-    }
-
-    public int TotalFilesScanned { get; }
-
-    public Task StartProcessingAsync(CancellationToken token) => throw new NotImplementedException();
-}

PiwigoDirectorySync/Service/FileScanner.cs

@@ -1,15 +0,0 @@
-using System.Threading.Channels;
-
-namespace PiwigoDirectorySync.Service;
-
-public class FileScanner
-{
-    private readonly Channel<string> _fileQueue;
-
-    public FileScanner(Channel<string> fileQueue)
-    {
-        _fileQueue = fileQueue ?? throw new ArgumentNullException(nameof(fileQueue));
-    }
-
-    public async Task ScanAsync(CancellationToken token) => throw new NotImplementedException();
-}

README.md

@@ -17,8 +17,8 @@ When you configure your environment for the first time and you do not like the d
 
 ``
 cd PiwigoDirectorySync
-dotnet user-secrets set "DbProvider" "MariaDb"
+dotnet user-secrets set "Settings:DbProvider" "Sqlite"
-dotnet user-secrets set "ConnectionStrings:MariaDb" "Server=localhost;User Id=photowfdev;Password=password123;Database=photowfdev"
+dotnet user-secrets set "ConnectionStrings:Sqlite" "Data Source=.\piwigoSync.db"
 ``
 
 You'll find your secrets under `~/.microsoft/usersecrets/c68c0447-8c7d-4e88-bcc6-96a9853828c7/secrets.json`