From 0a25ce2cc099480bff940ad6c8a60a6e87178518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20H=C3=A4felfinger?= Date: Wed, 30 Aug 2023 22:47:40 +0200 Subject: [PATCH] moves scan classes into one namespace, makes file scanner work, adds user secrets support to config --- .gitignore | 7 +- PiwigoDirectorySync/AppSettings.cs | 6 +- .../Commands/Scan/FileIndexer.cs | 36 +++++++++ .../Commands/Scan/FileSystemScanner.cs | 76 +++++++++++++++++++ .../Commands/{ => Scan}/ScanCommand.cs | 7 +- .../Commands/Scan/ScanSettings.cs | 16 ++++ PiwigoDirectorySync/Commands/ScanSettings.cs | 7 -- .../Persistence/ExtensionMethods.cs | 11 +++ .../Persistence/PersistenceContext.cs | 6 +- .../PiwigoDirectorySync.csproj | 4 + PiwigoDirectorySync/Program.cs | 12 ++- .../Properties/launchSettings.json | 11 +++ PiwigoDirectorySync/Service/FileIndexer.cs | 17 ----- PiwigoDirectorySync/Service/FileScanner.cs | 15 ---- README.md | 4 +- 15 files changed, 181 insertions(+), 54 deletions(-) create mode 100644 PiwigoDirectorySync/Commands/Scan/FileIndexer.cs create mode 100644 PiwigoDirectorySync/Commands/Scan/FileSystemScanner.cs rename PiwigoDirectorySync/Commands/{ => Scan}/ScanCommand.cs (80%) create mode 100644 PiwigoDirectorySync/Commands/Scan/ScanSettings.cs delete mode 100644 PiwigoDirectorySync/Commands/ScanSettings.cs create mode 100644 PiwigoDirectorySync/Persistence/ExtensionMethods.cs create mode 100644 PiwigoDirectorySync/Properties/launchSettings.json delete mode 100644 PiwigoDirectorySync/Service/FileIndexer.cs delete mode 100644 PiwigoDirectorySync/Service/FileScanner.cs diff --git a/.gitignore b/.gitignore index 33e5e1e..8ac769e 100644 --- a/.gitignore +++ b/.gitignore @@ -303,8 +303,6 @@ node_modules/ *.dsp # Visual Studio 6 technical files -*.ncb -*.aps # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts @@ -400,4 +398,7 @@ FodyWeavers.xsd # sqlite databases *.db -*.sqlite \ No newline at end of file +*.db-shm +*.db-wal +*.sqlite +/.idea/.idea.PiwigoDirectorySync/.idea/dataSources.xml diff --git a/PiwigoDirectorySync/AppSettings.cs b/PiwigoDirectorySync/AppSettings.cs index 130b9b3..d524c7b 100644 --- a/PiwigoDirectorySync/AppSettings.cs +++ b/PiwigoDirectorySync/AppSettings.cs @@ -4,14 +4,18 @@ namespace PiwigoDirectorySync; public static class AppSettings { + public static readonly IReadOnlySet SupportedExtensions = new HashSet { "jpg", "jpeg", "png" }; + public static IConfigurationRoot Config { get; } = new ConfigurationBuilder().SetBasePath(AppContext.BaseDirectory) .AddJsonFile("appsettings.json", true) .AddJsonFile(Path.Combine(Environment.CurrentDirectory, "appsettings.json"), true) .AddJsonFile("/etc/PiwigoDirectorySync/appsettings.json", true) + .AddUserSecrets(true) .AddEnvironmentVariables() .Build(); public static Settings Settings { get; } = Config.GetSection("settings").Get() ?? throw new InvalidOperationException("Could not parse settings"); - public static string ConnectionString => Config.GetConnectionString(Settings.DbProvider) ?? throw new InvalidOperationException($"Could not find connection string for provider {Settings.DbProvider}"); + public static string ConnectionString => + Config.GetConnectionString(Settings.DbProvider) ?? throw new InvalidOperationException($"Could not find connection string for provider {Settings.DbProvider}"); } \ No newline at end of file diff --git a/PiwigoDirectorySync/Commands/Scan/FileIndexer.cs b/PiwigoDirectorySync/Commands/Scan/FileIndexer.cs new file mode 100644 index 0000000..8289c84 --- /dev/null +++ b/PiwigoDirectorySync/Commands/Scan/FileIndexer.cs @@ -0,0 +1,36 @@ +using System.Threading.Channels; +using NLog; + +namespace PiwigoDirectorySync.Commands.Scan; + +public class FileIndexer +{ + private static readonly ILogger Logger = LogManager.GetCurrentClassLogger(); + + private readonly Channel _fileQueue; + private readonly int _piwigoServerId; + + public FileIndexer(Channel fileQueue, int piwigoServerId) + { + _fileQueue = fileQueue ?? throw new ArgumentNullException(nameof(fileQueue)); + _piwigoServerId = piwigoServerId; + } + + public int TotalFilesScanned { get; private set; } + + public async Task StartProcessingAsync(CancellationToken ct) + { + await foreach (var filePath in _fileQueue.Reader.ReadAllAsync(ct)) + { + try + { + Logger.Info($"Indexing file {filePath}"); + TotalFilesScanned++; + } + catch (Exception ex) + { + Logger.Error(ex, $"could not delete file {filePath}"); + } + } + } +} \ No newline at end of file diff --git a/PiwigoDirectorySync/Commands/Scan/FileSystemScanner.cs b/PiwigoDirectorySync/Commands/Scan/FileSystemScanner.cs new file mode 100644 index 0000000..9e24c6f --- /dev/null +++ b/PiwigoDirectorySync/Commands/Scan/FileSystemScanner.cs @@ -0,0 +1,76 @@ +using System.Threading.Channels; +using NLog; +using PiwigoDirectorySync.Persistence; + +namespace PiwigoDirectorySync.Commands.Scan; + +public class FileSystemScanner +{ + private static readonly ILogger Logger = LogManager.GetCurrentClassLogger(); + + private readonly Channel _fileQueue; + private readonly int _piwigoServerId; + + public FileSystemScanner(Channel fileQueue, int piwigoServerId) + { + _fileQueue = fileQueue ?? throw new ArgumentNullException(nameof(fileQueue)); + _piwigoServerId = piwigoServerId; + } + + public async Task ScanAsync(CancellationToken ct) + { + await using var db = new PersistenceContext(); + + var piwigoServer = await db.PiwigoServers.GetByIdAsync(_piwigoServerId, ct); + Logger.Info($"Scanning files for piwigo server {piwigoServer.Name} in directory {piwigoServer.RootDirectory}"); + + await ScanRootDirectory(new DirectoryInfo(piwigoServer.RootDirectory), ct); + } + + private async ValueTask ScanRootDirectory(DirectoryInfo directory, CancellationToken ct) + { + Logger.Info($"Scanning root directory {directory.FullName} for sidecars to delete"); + var parallelOptions = new ParallelOptions + { + CancellationToken = ct + }; + await Parallel.ForEachAsync(GetDirectories(directory), parallelOptions, FindAndEnqueueFilesToAdd); + } + + private async ValueTask FindAndEnqueueFilesToAdd(DirectoryInfo directory, CancellationToken ct) + { + try + { + Logger.Info($"Scanning directory {directory.FullName} for images"); + + var imageFiles = AppSettings.SupportedExtensions.SelectMany(ext => directory.GetFiles($"*.{ext}", SearchOption.TopDirectoryOnly)) + .Select(f => f.FullName) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (!imageFiles.Any()) + { + Logger.Debug($"No iamges in {directory.FullName} found, skipping"); + return; + } + + foreach (var imageFile in imageFiles.Select(f => new FileInfo(f))) + { + Logger.Debug($"Found image {imageFile.FullName}, enqueue index"); + await _fileQueue.Writer.WriteAsync(imageFile.FullName, ct); + } + } + catch (Exception ex) + { + Logger.Error(ex, $"could not scan directory {directory.FullName}"); + } + } + + private static IEnumerable GetDirectories(DirectoryInfo directoryInfo) + { + yield return directoryInfo; + foreach (var directory in directoryInfo.EnumerateDirectories().SelectMany(GetDirectories)) + { + yield return directory; + } + } +} \ No newline at end of file diff --git a/PiwigoDirectorySync/Commands/ScanCommand.cs b/PiwigoDirectorySync/Commands/Scan/ScanCommand.cs similarity index 80% rename from PiwigoDirectorySync/Commands/ScanCommand.cs rename to PiwigoDirectorySync/Commands/Scan/ScanCommand.cs index d0865e8..e57d59d 100644 --- a/PiwigoDirectorySync/Commands/ScanCommand.cs +++ b/PiwigoDirectorySync/Commands/Scan/ScanCommand.cs @@ -1,10 +1,9 @@ using System.Diagnostics; using System.Threading.Channels; using NLog; -using PiwigoDirectorySync.Service; using Spectre.Console.Cli; -namespace PiwigoDirectorySync.Commands; +namespace PiwigoDirectorySync.Commands.Scan; public class ScanCommand : AsyncCommand { @@ -19,10 +18,10 @@ public class ScanCommand : AsyncCommand var fileQueue = Channel.CreateUnbounded(); - var indexer = new FileIndexer(fileQueue); + var indexer = new FileIndexer(fileQueue, settings.PiwigoServerId); var indexerTask = indexer.StartProcessingAsync(cancellationTokenSource.Token); - var scanner = new FileScanner(fileQueue); + var scanner = new FileSystemScanner(fileQueue, settings.PiwigoServerId); await scanner.ScanAsync(cancellationTokenSource.Token); fileQueue.Writer.Complete(); diff --git a/PiwigoDirectorySync/Commands/Scan/ScanSettings.cs b/PiwigoDirectorySync/Commands/Scan/ScanSettings.cs new file mode 100644 index 0000000..78e4b06 --- /dev/null +++ b/PiwigoDirectorySync/Commands/Scan/ScanSettings.cs @@ -0,0 +1,16 @@ +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using Spectre.Console.Cli; + +namespace PiwigoDirectorySync.Commands.Scan; + +[SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")] +public class ScanSettings : CommandSettings +{ + [CommandArgument(0, "")] + public int PiwigoServerId { get; set; } + + [CommandOption("-d|--mark-for-delete")] + [DefaultValue(false)] + public bool MarkForDelete { get; set; } +} \ No newline at end of file diff --git a/PiwigoDirectorySync/Commands/ScanSettings.cs b/PiwigoDirectorySync/Commands/ScanSettings.cs deleted file mode 100644 index bf7c7e0..0000000 --- a/PiwigoDirectorySync/Commands/ScanSettings.cs +++ /dev/null @@ -1,7 +0,0 @@ -using Spectre.Console.Cli; - -namespace PiwigoDirectorySync.Commands; - -public class ScanSettings : CommandSettings -{ -} \ No newline at end of file diff --git a/PiwigoDirectorySync/Persistence/ExtensionMethods.cs b/PiwigoDirectorySync/Persistence/ExtensionMethods.cs new file mode 100644 index 0000000..a6c4340 --- /dev/null +++ b/PiwigoDirectorySync/Persistence/ExtensionMethods.cs @@ -0,0 +1,11 @@ +using Microsoft.EntityFrameworkCore; + +namespace PiwigoDirectorySync.Persistence; + +public static class ExtensionMethods +{ + public static Task GetByIdAsync(this DbSet dbSet, int serverId, CancellationToken ct) + { + return dbSet.Where(a => a.Id == serverId).FirstAsync(ct); + } +} \ No newline at end of file diff --git a/PiwigoDirectorySync/Persistence/PersistenceContext.cs b/PiwigoDirectorySync/Persistence/PersistenceContext.cs index 58672c8..e560a2d 100644 --- a/PiwigoDirectorySync/Persistence/PersistenceContext.cs +++ b/PiwigoDirectorySync/Persistence/PersistenceContext.cs @@ -4,9 +4,9 @@ namespace PiwigoDirectorySync.Persistence; public class PersistenceContext : DbContext { - public DbSet PiwigoServers { get; set; } - public DbSet PiwigoAlbums { get; set; } - public DbSet PiwigoImages { get; set; } + public DbSet PiwigoServers { get; set; } = null!; + public DbSet PiwigoAlbums { get; set; } = null!; + public DbSet PiwigoImages { get; set; } = null!; protected override void OnConfiguring(DbContextOptionsBuilder options) { diff --git a/PiwigoDirectorySync/PiwigoDirectorySync.csproj b/PiwigoDirectorySync/PiwigoDirectorySync.csproj index 0968c8f..5b179e9 100644 --- a/PiwigoDirectorySync/PiwigoDirectorySync.csproj +++ b/PiwigoDirectorySync/PiwigoDirectorySync.csproj @@ -12,6 +12,7 @@ + @@ -40,7 +41,10 @@ Always + + + diff --git a/PiwigoDirectorySync/Program.cs b/PiwigoDirectorySync/Program.cs index f62bf05..9603beb 100644 --- a/PiwigoDirectorySync/Program.cs +++ b/PiwigoDirectorySync/Program.cs @@ -1,7 +1,7 @@ using NLog; using NLog.Extensions.Logging; using PiwigoDirectorySync; -using PiwigoDirectorySync.Commands; +using PiwigoDirectorySync.Commands.Scan; using Spectre.Console.Cli; LogManager.Configuration = new NLogLoggingConfiguration(AppSettings.Config.GetSection("NLog")); @@ -10,6 +10,14 @@ var logger = logFactory.GetCurrentClassLogger(); logger.Info("Starting command app"); var app = new CommandApp(); -app.Configure(c => { c.AddCommand("scan"); }); +app.Configure(config => +{ +#if DEBUG + config.PropagateExceptions(); + config.ValidateExamples(); +#endif + + config.AddCommand("scan"); +}); return app.Run(args); \ No newline at end of file diff --git a/PiwigoDirectorySync/Properties/launchSettings.json b/PiwigoDirectorySync/Properties/launchSettings.json new file mode 100644 index 0000000..3856d80 --- /dev/null +++ b/PiwigoDirectorySync/Properties/launchSettings.json @@ -0,0 +1,11 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "PiwigoDirectorySync": { + "commandName": "Project", + "commandLineArgs": "scan 1", + "environmentVariables": { + } + } + } +} diff --git a/PiwigoDirectorySync/Service/FileIndexer.cs b/PiwigoDirectorySync/Service/FileIndexer.cs deleted file mode 100644 index 37af38d..0000000 --- a/PiwigoDirectorySync/Service/FileIndexer.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.Threading.Channels; - -namespace PiwigoDirectorySync.Service; - -public class FileIndexer -{ - private readonly Channel _fileQueue; - - public FileIndexer(Channel fileQueue) - { - _fileQueue = fileQueue ?? throw new ArgumentNullException(nameof(fileQueue)); - } - - public int TotalFilesScanned { get; } - - public Task StartProcessingAsync(CancellationToken token) => throw new NotImplementedException(); -} \ No newline at end of file diff --git a/PiwigoDirectorySync/Service/FileScanner.cs b/PiwigoDirectorySync/Service/FileScanner.cs deleted file mode 100644 index 80bc9dd..0000000 --- a/PiwigoDirectorySync/Service/FileScanner.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System.Threading.Channels; - -namespace PiwigoDirectorySync.Service; - -public class FileScanner -{ - private readonly Channel _fileQueue; - - public FileScanner(Channel fileQueue) - { - _fileQueue = fileQueue ?? throw new ArgumentNullException(nameof(fileQueue)); - } - - public async Task ScanAsync(CancellationToken token) => throw new NotImplementedException(); -} \ No newline at end of file diff --git a/README.md b/README.md index b620a34..1a3db3a 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ When you configure your environment for the first time and you do not like the d `` cd PiwigoDirectorySync -dotnet user-secrets set "DbProvider" "MariaDb" -dotnet user-secrets set "ConnectionStrings:MariaDb" "Server=localhost;User Id=photowfdev;Password=password123;Database=photowfdev" +dotnet user-secrets set "Settings:DbProvider" "Sqlite" +dotnet user-secrets set "ConnectionStrings:Sqlite" "Data Source=.\piwigoSync.db" `` You'll find your secrets under `~/.microsoft/usersecrets/c68c0447-8c7d-4e88-bcc6-96a9853828c7/secrets.json`