From b910987a24393c20d7f618ff0d9d06d8f90cb4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20H=C3=A4felfinger?= Date: Fri, 15 Sep 2023 23:20:37 +0200 Subject: [PATCH] moves download to sync download and make it more stable --- ...magesCommand.cs => SyncDownloadCommand.cs} | 10 ++-- PiwigoDirectorySync/Program.cs | 2 +- .../Properties/launchSettings.json | 2 +- .../Services/ImageSynchronizer.cs | 54 ++++++++++++++----- PiwigoDirectorySync/appsettings.json | 2 +- 5 files changed, 49 insertions(+), 21 deletions(-) rename PiwigoDirectorySync/Commands/{DownloadImagesCommand.cs => SyncDownloadCommand.cs} (67%) diff --git a/PiwigoDirectorySync/Commands/DownloadImagesCommand.cs b/PiwigoDirectorySync/Commands/SyncDownloadCommand.cs similarity index 67% rename from PiwigoDirectorySync/Commands/DownloadImagesCommand.cs rename to PiwigoDirectorySync/Commands/SyncDownloadCommand.cs index 4d00615..bdf23a7 100644 --- a/PiwigoDirectorySync/Commands/DownloadImagesCommand.cs +++ b/PiwigoDirectorySync/Commands/SyncDownloadCommand.cs @@ -6,17 +6,17 @@ using Spectre.Console.Cli; namespace PiwigoDirectorySync.Commands; -internal class DownloadImagesCommand : CancellableAsyncCommand +internal class SyncDownloadCommand : CancellableAsyncCommand { private readonly IImageSynchronizer _imageSynchronizer; - private readonly ILogger _logger = Log.ForContext(); + private readonly ILogger _logger = Log.ForContext(); - public DownloadImagesCommand(IImageSynchronizer imageSynchronizer) + public SyncDownloadCommand(IImageSynchronizer imageSynchronizer) { _imageSynchronizer = imageSynchronizer; } - protected override async Task ExecuteAsync(CommandContext context, DownloadImagesSettings settings, CancellationToken cancellationToken) + protected override async Task ExecuteAsync(CommandContext context, SyncDownloadSettings settings, CancellationToken cancellationToken) { _logger.Information("Starting image download for piwigo server {SettingsPiwigoServerId}", settings.PiwigoServerId); var stopWatch = Stopwatch.StartNew(); @@ -30,7 +30,7 @@ internal class DownloadImagesCommand : CancellableAsyncCommand #endif config.AddCommand("scan"); - config.AddCommand("download"); config.AddBranch("sync", c => { c.AddCommand("full"); c.AddCommand("albums"); c.AddCommand("images"); + c.AddCommand("download"); c.AddCommand("pending"); }); config.AddBranch("piwigo", c => diff --git a/PiwigoDirectorySync/Properties/launchSettings.json b/PiwigoDirectorySync/Properties/launchSettings.json index 98995d6..0ce2fb5 100644 --- a/PiwigoDirectorySync/Properties/launchSettings.json +++ b/PiwigoDirectorySync/Properties/launchSettings.json @@ -39,7 +39,7 @@ }, "Download": { "commandName": "Project", - "commandLineArgs": "download", + "commandLineArgs": "sync download 5", "environmentVariables": { } }, diff --git a/PiwigoDirectorySync/Services/ImageSynchronizer.cs b/PiwigoDirectorySync/Services/ImageSynchronizer.cs index eedb72e..440d802 100644 --- a/PiwigoDirectorySync/Services/ImageSynchronizer.cs +++ b/PiwigoDirectorySync/Services/ImageSynchronizer.cs @@ -49,6 +49,12 @@ internal class ImageSynchronizer : IImageSynchronizer foreach (var albumId in albumIdsToDownload) { var albumInfos = await piwigoClient.Album.GetListAsync(albumId, false, false, ThumbnailSize.Thumb, ct); + if (!albumInfos.Any()) + { + Log.Warning("Album with server {AlbumId} not found on piwigo server", albumId); + continue; + } + var albumInfo = albumInfos.First(); _logger.Information("Starting downloads for album {AlbumInfoName}", albumInfo.Name); @@ -101,7 +107,19 @@ internal class ImageSynchronizer : IImageSynchronizer foreach (var image in images.Images) { - await DownloadImageAsync(piwigoServer, albumInfo, image, ct); + try + { + await DownloadImageAsync(piwigoServer, albumInfo, image, ct); + } + catch (OperationCanceledException) + { + Log.Debug("Cancel of download requested"); + return; + } + catch (Exception ex) + { + Log.Warning(ex, "Download of server image {ImageId} / {ImageName} / {ImageElementUrl}", image.Id, image.Name, image.ElementUrl); + } } currentPage++; @@ -123,10 +141,11 @@ internal class ImageSynchronizer : IImageSynchronizer var fileInfo = new FileInfo(Path.Combine(piwigoServer.RootDirectory, localImage.FilePath)); if (fileInfo.Exists) { - _logger.Warning("Tried to download image {ImageFile} but it already exists", image.File); + _logger.Debug("Tried to download image {ImageFile} but it already exists", fileInfo.FullName); return; } + _logger.Information("Downloading image {ImageFile}", fileInfo.FullName); await image.ElementUrl.DownloadFileAsync(fileInfo.Directory!.FullName, fileInfo.Name, cancellationToken: ct); localImage.Md5Sum = await FilesystemHelpers.CalculateMd5SumAsync(fileInfo.FullName, ct); await _persistenceContext.SaveChangesAsync(ct); @@ -134,21 +153,30 @@ internal class ImageSynchronizer : IImageSynchronizer private async Task GetOrAddImageFromServerAsync(AlbumEntity album, Image image, CancellationToken ct) { + var imageFullPath = Path.Combine(album.Path, image.File!); var imageEntity = await _persistenceContext.PiwigoImages.Where(i => i.AlbumId == album.Id && i.ServerImageId == image.Id).FirstOrDefaultAsync(ct); - if (imageEntity is null) + if (imageEntity is not null) { - imageEntity = new ImageEntity - { - AlbumId = album.Id, - Album = album, - ServerImageId = image.Id, - FilePath = Path.Combine(album.Path, image.File!), - UploadRequired = false, - DeleteRequired = false - }; - _persistenceContext.PiwigoImages.Add(imageEntity); + return imageEntity; } + imageEntity = await _persistenceContext.PiwigoImages.Where(i => i.AlbumId == album.Id && i.FilePath == imageFullPath).FirstOrDefaultAsync(ct); + if (imageEntity is not null) + { + imageEntity.ServerImageId = image.Id; + return imageEntity; + } + + imageEntity = new ImageEntity + { + AlbumId = album.Id, + Album = album, + ServerImageId = image.Id, + FilePath = imageFullPath, + UploadRequired = false, + DeleteRequired = false + }; + _persistenceContext.PiwigoImages.Add(imageEntity); return imageEntity; } diff --git a/PiwigoDirectorySync/appsettings.json b/PiwigoDirectorySync/appsettings.json index 5bb3b1b..c3b86bc 100644 --- a/PiwigoDirectorySync/appsettings.json +++ b/PiwigoDirectorySync/appsettings.json @@ -8,7 +8,7 @@ } }, "ConnectionStrings": { - "Sqlite": "Data Source=piwigoSync.db", + "Sqlite": "Data Source=../../../piwigoSync.db", "InMemory": "InMemorySyncDb" }, "Settings": {