changes to dependency injection and microsoft logging, removes nlog, adds first version of album sync form server

This commit is contained in:
Philipp Häfelfinger 2023-08-31 22:47:49 +02:00
parent 8f7496a3df
commit c06e8763a4
18 changed files with 327 additions and 119 deletions

View File

@ -1,6 +1,6 @@
using System.Diagnostics;
using System.Threading.Channels;
using NLog;
using Microsoft.Extensions.Logging;
using PiwigoDirectorySync.Services;
using Spectre.Console.Cli;
@ -8,31 +8,40 @@ namespace PiwigoDirectorySync.Commands;
public class ScanCommand : AsyncCommand<ScanSettings>
{
private static readonly ILogger Logger = LogManager.GetCurrentClassLogger();
private readonly IFileIndexer _fileIndexer;
private readonly IFileSystemScanner _fileSystemScanner;
private readonly ILogger<ScanCommand> _logger;
public ScanCommand(ILogger<ScanCommand> logger, IFileIndexer fileIndexer, IFileSystemScanner fileSystemScanner)
{
_logger = logger;
_fileIndexer = fileIndexer;
_fileSystemScanner = fileSystemScanner;
}
public override async Task<int> ExecuteAsync(CommandContext context, ScanSettings settings)
{
//TODO: check files for deletion -> files in db but no longer exist
Logger.Info("Starting scanner and remover");
_logger.LogInformation("Starting scanner and remover");
var stopWatch = Stopwatch.StartNew();
var cancellationTokenSource = new CancellationTokenSource();
var fileQueue = Channel.CreateUnbounded<string>();
var indexer = new FileIndexer(fileQueue, settings.PiwigoServerId);
var indexerTask = indexer.StartProcessingAsync(cancellationTokenSource.Token);
var indexerTask = _fileIndexer.StartProcessingAsync(fileQueue, settings.PiwigoServerId, cancellationTokenSource.Token);
var scanner = new FileSystemScanner(fileQueue, settings.PiwigoServerId);
await scanner.ScanAsync(cancellationTokenSource.Token);
await _fileSystemScanner.ScanAsync(fileQueue, settings.PiwigoServerId, cancellationTokenSource.Token);
fileQueue.Writer.Complete();
await Task.WhenAll(fileQueue.Reader.Completion, indexerTask);
stopWatch.Stop();
Logger.Info($"Processed {indexer.TotalFilesScanned} image files in {stopWatch.Elapsed.TotalSeconds} seconds");
_logger.LogInformation("Processed {IndexerTotalFilesScanned} image files in {ElapsedTotalSeconds} seconds", _fileIndexer.TotalFilesScanned, stopWatch.Elapsed.TotalSeconds);
//TODO: write failed files to log
return 0;
}

View File

@ -0,0 +1,33 @@
using System.Diagnostics;
using Microsoft.Extensions.Logging;
using PiwigoDirectorySync.Services;
using Spectre.Console.Cli;
namespace PiwigoDirectorySync.Commands;
public class SyncAlbumsCommand : AsyncCommand<SyncAlbumsSettings>
{
private readonly IAlbumSynchronizer _albumSynchronizer;
private readonly ILogger<SyncAlbumsCommand> _logger;
public SyncAlbumsCommand(ILogger<SyncAlbumsCommand> logger, IAlbumSynchronizer albumSynchronizer)
{
_logger = logger;
_albumSynchronizer = albumSynchronizer;
}
public override async Task<int> ExecuteAsync(CommandContext context, SyncAlbumsSettings settings)
{
_logger.LogInformation("Starting album synchronization");
var stopWatch = Stopwatch.StartNew();
var cancellationTokenSource = new CancellationTokenSource();
await _albumSynchronizer.SynchronizeAlbums(settings.PiwigoServerId, cancellationTokenSource.Token);
stopWatch.Stop();
_logger.LogInformation("Synchronized all albums with piwigo server {SettingsPiwigoServerId} in {ElapsedTotalSeconds} seconds", settings.PiwigoServerId,
stopWatch.Elapsed.TotalSeconds);
return 0;
}
}

View File

@ -0,0 +1,11 @@
using System.Diagnostics.CodeAnalysis;
using Spectre.Console.Cli;
namespace PiwigoDirectorySync.Commands;
[SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")]
public class SyncAlbumsSettings : CommandSettings
{
[CommandArgument(0, "<PiwigoServerId>")]
public int PiwigoServerId { get; set; }
}

View File

@ -0,0 +1,36 @@
using Microsoft.Extensions.DependencyInjection;
using Spectre.Console.Cli;
namespace PiwigoDirectorySync.Infrastructure;
public sealed class DependencyInjectionTypeRegistrar : ITypeRegistrar
{
private readonly IServiceCollection _builder;
public DependencyInjectionTypeRegistrar(IServiceCollection builder)
{
_builder = builder;
}
public ITypeResolver Build() => new DependencyTypeResolver(_builder.BuildServiceProvider());
public void Register(Type service, Type implementation)
{
_builder.AddSingleton(service, implementation);
}
public void RegisterInstance(Type service, object implementation)
{
_builder.AddSingleton(service, implementation);
}
public void RegisterLazy(Type service, Func<object> func)
{
if (func is null)
{
throw new ArgumentNullException(nameof(func));
}
_builder.AddSingleton(service, _ => func());
}
}

View File

@ -0,0 +1,23 @@
using Spectre.Console.Cli;
namespace PiwigoDirectorySync.Infrastructure;
public sealed class DependencyTypeResolver : ITypeResolver, IDisposable
{
private readonly IServiceProvider _provider;
public DependencyTypeResolver(IServiceProvider provider)
{
_provider = provider ?? throw new ArgumentNullException(nameof(provider));
}
public void Dispose()
{
if (_provider is IDisposable disposable)
{
disposable.Dispose();
}
}
public object? Resolve(Type? type) => type == null ? null : _provider.GetService(type);
}

View File

@ -0,0 +1,15 @@
using Microsoft.Extensions.Logging;
using Piwigo.Client;
using PiwigoDirectorySync.Persistence;
namespace PiwigoDirectorySync.Infrastructure;
public static class PiwigoClientFactory
{
public static async Task<IPiwigoClient> GetPiwigoClientAsync(ServerEntity piwigoServer, ILoggerFactory loggerFactory, CancellationToken ct)
{
var piwigoClient = PiwigoClient.CreateClient(piwigoServer.Url, piwigoServer.Username, piwigoServer.Password, loggerFactory);
await piwigoClient.Session.LoginAsync(ct);
return piwigoClient;
}
}

View File

@ -22,5 +22,4 @@ public class AlbumEntity
public required int ServerId { get; set; }
public ServerEntity Server { get; set; } = null!;
}

View File

@ -13,4 +13,9 @@ public static class ExtensionMethods
{
return await dbSet.Where(a => a.ServerId == serverId && a.Path == relativePath).FirstOrDefaultAsync(ct);
}
public static async Task<AlbumEntity?> FindByServerIdAsync(this DbSet<AlbumEntity> dbSet, int piwigoServerId, int serverAlbumId, CancellationToken ct)
{
return await dbSet.Where(a => a.ServerId == piwigoServerId && a.ServerAlbumId == serverAlbumId).FirstOrDefaultAsync(ct);
}
}

View File

@ -12,23 +12,24 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="7.0.0" />
<PackageReference Include="Piwigo.Client" Version="0.1.0.17"/>
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.0"/>
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0"/>
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0"/>
<PackageReference Include="NLog.Extensions.Logging" Version="5.2.0"/>
<PackageReference Include="Spectre.Console.Analyzer" Version="0.47.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Spectre.Console.Cli" Version="0.47.0"/>
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.10">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="7.0.10"/>
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="7.0.10"/>
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.0"/>
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0"/>
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0"/>
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="7.0.0" />
<PackageReference Include="Piwigo.Client" Version="0.1.0.17"/>
<PackageReference Include="Spectre.Console.Analyzer" Version="0.47.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Spectre.Console.Cli" Version="0.47.0"/>
</ItemGroup>
<ItemGroup>
@ -48,6 +49,7 @@
</ItemGroup>

View File

@ -1,15 +1,25 @@
using NLog;
using NLog.Extensions.Logging;
using PiwigoDirectorySync;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using PiwigoDirectorySync.Commands;
using PiwigoDirectorySync.Infrastructure;
using PiwigoDirectorySync.Services;
using Spectre.Console.Cli;
LogManager.Configuration = new NLogLoggingConfiguration(AppSettings.Config.GetSection("NLog"));
var logFactory = LogManager.Setup().LogFactory;
var logger = logFactory.GetCurrentClassLogger();
var registrations = new ServiceCollection();
registrations.AddLogging(l => l.AddSimpleConsole(c =>
{
c.SingleLine = true;
c.IncludeScopes = true;
})
.AddDebug());
logger.Info("Starting command app");
var app = new CommandApp();
registrations.AddTransient<IFileIndexer, FileIndexer>();
registrations.AddTransient<IFileSystemScanner, FileSystemScanner>();
registrations.AddTransient<IAlbumSynchronizer, AlbumSynchronizer>();
var registrar = new DependencyInjectionTypeRegistrar(registrations);
var app = new CommandApp(registrar);
app.Configure(config =>
{
#if DEBUG
@ -18,6 +28,7 @@ app.Configure(config =>
#endif
config.AddCommand<ScanCommand>("scan");
config.AddBranch("sync", c => { c.AddCommand<SyncAlbumsCommand>("albums"); });
});
return app.Run(args);

View File

@ -1,17 +1,23 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"profiles": {
"PiwigoDirectorySync": {
"SyncLocalJpgs": {
"commandName": "Project",
"commandLineArgs": "scan 1",
"environmentVariables": {
}
},
"PiwigoDirectorySyncPng": {
"SyncLocalPngs": {
"commandName": "Project",
"commandLineArgs": "scan 2",
"environmentVariables": {
}
},
"SyncAlbums": {
"commandName": "Project",
"commandLineArgs": "sync albums 1",
"environmentVariables": {
}
}
}
}

View File

@ -0,0 +1,92 @@
using Microsoft.Extensions.Logging;
using Piwigo.Client;
using Piwigo.Client.Albums;
using PiwigoDirectorySync.Infrastructure;
using PiwigoDirectorySync.Persistence;
namespace PiwigoDirectorySync.Services;
public class AlbumSynchronizer : IAlbumSynchronizer
{
private readonly ILogger<AlbumSynchronizer> _logger;
private readonly ILoggerFactory _loggerFactory;
public AlbumSynchronizer(ILogger<AlbumSynchronizer> logger, ILoggerFactory loggerFactory)
{
_logger = logger;
_loggerFactory = loggerFactory;
}
public async Task SynchronizeAlbums(int piwigoServerId, CancellationToken ct)
{
await using var dbContext = new PersistenceContext();
var piwigoServer = await dbContext.PiwigoServers.FindAsync(new object?[] { piwigoServerId }, ct);
if (piwigoServer is null)
{
_logger.LogError("Could not sync albums with piwigo server {PiwigoServerId}", piwigoServerId);
return;
}
var piwigoClient = await PiwigoClientFactory.GetPiwigoClientAsync(piwigoServer, _loggerFactory, ct);
await UpdatePiwigoAlbumsFromServerAsync(dbContext, piwigoClient, piwigoServer, ct);
}
private static async Task UpdatePiwigoAlbumsFromServerAsync(PersistenceContext dbContext, IPiwigoClient piwigoClient, ServerEntity piwigoServer, CancellationToken ct)
{
var serverAlbums = await piwigoClient.Album.GetListAsync(null, true, false, ThumbnailSize.Thumb, ct);
var serverAlbumDictionary = serverAlbums.ToDictionary(a => a.Id, a => a);
foreach (var serverAlbum in serverAlbums)
{
var albumEntity = await GetOrAddPiwigoAlbumEntityAsync(dbContext, piwigoServer, serverAlbum, serverAlbumDictionary, ct);
if (serverAlbum.IdUpperCat.HasValue)
{
albumEntity.ParentId = (await dbContext.PiwigoAlbums.FindByServerIdAsync(piwigoServer.Id, serverAlbum.IdUpperCat.Value, ct))?.Id;
}
await dbContext.SaveChangesAsync(ct);
}
}
private static async Task<AlbumEntity> GetOrAddPiwigoAlbumEntityAsync(PersistenceContext dbContext, ServerEntity piwigoServer, Album serverAlbum,
IDictionary<int, Album> serverAlbumDictionary, CancellationToken ct)
{
var albumEntity = await dbContext.PiwigoAlbums.FindByServerIdAsync(piwigoServer.Id, serverAlbum.Id, ct);
if (albumEntity != null)
{
return albumEntity;
}
albumEntity = new AlbumEntity
{
ServerId = piwigoServer.Id,
Server = piwigoServer,
Name = serverAlbum.Name,
ServerAlbumId = serverAlbum.Id,
Path = GeneratePath(serverAlbum, serverAlbumDictionary)
};
dbContext.PiwigoAlbums.Add(albumEntity);
return albumEntity;
}
private static string GeneratePath(Album serverAlbum, IDictionary<int, Album> serverAlbumDictionary)
{
if (!serverAlbum.IdUpperCat.HasValue)
{
return serverAlbum.Name;
}
var parentId = serverAlbum.IdUpperCat.Value;
var path = serverAlbum.Name;
while (parentId != 0)
{
var currentParent = serverAlbumDictionary[parentId];
path = Path.Combine(currentParent.Name, path);
parentId = currentParent.IdUpperCat ?? 0;
}
return path;
}
}

View File

@ -1,49 +1,45 @@
using System.Security.Cryptography;
using System.Threading.Channels;
using Microsoft.EntityFrameworkCore;
using NLog;
using Microsoft.Extensions.Logging;
using PiwigoDirectorySync.Persistence;
namespace PiwigoDirectorySync.Services;
public class FileIndexer
public class FileIndexer : IFileIndexer
{
private static readonly ILogger Logger = LogManager.GetCurrentClassLogger();
private readonly IList<string> _failedFiles = new List<string>();
private readonly ILogger<FileIndexer> _logger;
private readonly Channel<string> _fileQueue;
private readonly int _piwigoServerId;
public FileIndexer(Channel<string> fileQueue, int piwigoServerId)
public FileIndexer(ILogger<FileIndexer> logger)
{
_fileQueue = fileQueue ?? throw new ArgumentNullException(nameof(fileQueue));
_piwigoServerId = piwigoServerId;
_logger = logger;
}
public int TotalFilesScanned { get; private set; }
public IReadOnlyCollection<string> FailedFiles => _failedFiles.AsReadOnly();
public async Task StartProcessingAsync(CancellationToken ct)
public async Task StartProcessingAsync(Channel<string> fileQueue, int piwigoServerId, CancellationToken ct)
{
await using var db = new PersistenceContext();
var piwigoServer = await db.PiwigoServers.GetByIdAsync(_piwigoServerId, ct);
var piwigoServer = await db.PiwigoServers.GetByIdAsync(piwigoServerId, ct);
await foreach (var fullFilePath in _fileQueue.Reader.ReadAllAsync(ct))
await foreach (var fullFilePath in fileQueue.Reader.ReadAllAsync(ct))
{
try
{
if (ct.IsCancellationRequested)
{
Logger.Warn("Indexing cancelled");
_logger.LogWarning("Indexing cancelled");
break;
}
Logger.Info($"Indexing file {fullFilePath}");
_logger.LogInformation("Indexing file {FullFilePath}", fullFilePath);
var fileInfo = new FileInfo(fullFilePath);
if (!fileInfo.Exists)
{
Logger.Warn($"File {fullFilePath} not found");
_logger.LogWarning("File {FullFilePath} not found", fullFilePath);
_failedFiles.Add(fullFilePath);
continue;
}
@ -70,7 +66,7 @@ public class FileIndexer
catch (Exception ex)
{
_failedFiles.Add(fullFilePath);
Logger.Error(ex, $"could not delete file {fullFilePath}");
_logger.LogError(ex, "could not delete file {FullFilePath}", fullFilePath);
}
}
}

View File

@ -1,47 +1,44 @@
using System.Threading.Channels;
using NLog;
using Microsoft.Extensions.Logging;
using PiwigoDirectorySync.Persistence;
namespace PiwigoDirectorySync.Services;
public class FileSystemScanner
public class FileSystemScanner : IFileSystemScanner
{
private static readonly ILogger Logger = LogManager.GetCurrentClassLogger();
private readonly ILogger<FileSystemScanner> _logger;
private readonly Channel<string> _fileQueue;
private readonly int _piwigoServerId;
public FileSystemScanner(Channel<string> fileQueue, int piwigoServerId)
public FileSystemScanner(ILogger<FileSystemScanner> logger)
{
_fileQueue = fileQueue ?? throw new ArgumentNullException(nameof(fileQueue));
_piwigoServerId = piwigoServerId;
_logger = logger;
}
public async Task ScanAsync(CancellationToken ct)
public async Task ScanAsync(Channel<string> fileQueue, int piwigoServerId, CancellationToken ct)
{
await using var db = new PersistenceContext();
var piwigoServer = await db.PiwigoServers.GetByIdAsync(_piwigoServerId, ct);
Logger.Info($"Scanning files for piwigo server {piwigoServer.Name} in directory {piwigoServer.RootDirectory}");
var piwigoServer = await db.PiwigoServers.GetByIdAsync(piwigoServerId, ct);
_logger.LogInformation("Scanning files for piwigo server {PiwigoServerName} in directory {PiwigoServerRootDirectory}", piwigoServer.Name, piwigoServer.RootDirectory);
await ScanRootDirectory(new DirectoryInfo(piwigoServer.RootDirectory), ct);
await ScanRootDirectory(fileQueue, new DirectoryInfo(piwigoServer.RootDirectory), ct);
}
private async ValueTask ScanRootDirectory(DirectoryInfo directory, CancellationToken ct)
private async ValueTask ScanRootDirectory(Channel<string> fileQueue, DirectoryInfo directory, CancellationToken ct)
{
Logger.Info($"Scanning root directory {directory.FullName} for sidecars to delete");
_logger.LogInformation("Scanning root directory {DirectoryFullName} for sidecars to delete", directory.FullName);
var parallelOptions = new ParallelOptions
{
CancellationToken = ct
};
await Parallel.ForEachAsync(GetDirectories(directory), parallelOptions, FindAndEnqueueFilesToAdd);
await Parallel.ForEachAsync(GetDirectories(directory), parallelOptions, (d, c) => FindAndEnqueueFilesToAdd(fileQueue, d, c));
}
private async ValueTask FindAndEnqueueFilesToAdd(DirectoryInfo directory, CancellationToken ct)
private async ValueTask FindAndEnqueueFilesToAdd(Channel<string> fileQueue, DirectoryInfo directory, CancellationToken ct)
{
try
{
Logger.Info($"Scanning directory {directory.FullName} for images");
_logger.LogInformation("Scanning directory {DirectoryFullName} for images", directory.FullName);
var imageFiles = AppSettings.SupportedExtensions.SelectMany(ext => directory.GetFiles($"*.{ext}", SearchOption.TopDirectoryOnly))
.Select(f => f.FullName)
@ -49,19 +46,19 @@ public class FileSystemScanner
if (!imageFiles.Any())
{
Logger.Debug($"No iamges in {directory.FullName} found, skipping");
_logger.LogDebug("No images in {DirectoryFullName} found, skipping", directory.FullName);
return;
}
foreach (var imageFile in imageFiles.Select(f => new FileInfo(f)))
{
Logger.Debug($"Found image {imageFile.FullName}, enqueue index");
await _fileQueue.Writer.WriteAsync(imageFile.FullName, ct);
_logger.LogDebug("Found image {ImageFileFullName}, enqueue index", imageFile.FullName);
await fileQueue.Writer.WriteAsync(imageFile.FullName, ct);
}
}
catch (Exception ex)
{
Logger.Error(ex, $"could not scan directory {directory.FullName}");
_logger.LogError(ex, "could not scan directory {DirectoryFullName}", directory.FullName);
}
}

View File

@ -0,0 +1,6 @@
namespace PiwigoDirectorySync.Services;
public interface IAlbumSynchronizer
{
Task SynchronizeAlbums(int piwigoServerId, CancellationToken ct);
}

View File

@ -0,0 +1,10 @@
using System.Threading.Channels;
namespace PiwigoDirectorySync.Services;
public interface IFileIndexer
{
int TotalFilesScanned { get; }
IReadOnlyCollection<string> FailedFiles { get; }
Task StartProcessingAsync(Channel<string> fileQueue, int piwigoServerId, CancellationToken ct);
}

View File

@ -0,0 +1,8 @@
using System.Threading.Channels;
namespace PiwigoDirectorySync.Services;
public interface IFileSystemScanner
{
Task ScanAsync(Channel<string> fileQueue, int piwigoServerId, CancellationToken ct);
}

View File

@ -13,56 +13,5 @@
"Settings": {
"DbProvider": "Sqlite",
"ImageRootDirectory": ".\\"
},
"NLog": {
"autoReload": true,
"throwConfigExceptions": true,
"default-wrapper": {
"type": "AsyncWrapper",
"overflowAction": "Block"
},
"targets": {
"cli-console": {
"type": "ColoredConsole",
"layout": "${longdate} | ${uppercase:${level}} | ${logger} | ${message} ${exception:format=tostring}",
"rowHighlightingRules": [
{
"condition": "level == LogLevel.Trace",
"foregroundColor": "DarkGray"
},
{
"condition": "level == LogLevel.Debug",
"foregroundColor": "White"
},
{
"condition": "level == LogLevel.Info",
"foregroundColor": "DarkGreen"
},
{
"condition": "level == LogLevel.Warn",
"foregroundColor": "Yellow"
},
{
"condition": "level == LogLevel.Error",
"foregroundColor": "DarkMagenta"
},
{
"condition": "level == LogLevel.Fatal",
"foregroundColor": "DarkRed"
}
]
}
},
"rules": [
{
"logger": "*",
"minLevel": "Info",
"writeTo": "cli-console"
},
{
"logger": "Microsoft.*",
"maxLevel": "Info"
}
]
}
}