From b92aabfbcb535a0103b302c2a5bcdba1064da035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20H=C3=A4felfinger?= Date: Sat, 2 Sep 2023 15:54:53 +0200 Subject: [PATCH] makes image upload work --- .../Persistence/ImageEntity.cs | 3 + .../Services/ImageSynchronizer.cs | 102 +++++++++++++++++- PiwigoDirectorySync/appsettings.json | 3 +- 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/PiwigoDirectorySync/Persistence/ImageEntity.cs b/PiwigoDirectorySync/Persistence/ImageEntity.cs index 3141473..196b1e6 100644 --- a/PiwigoDirectorySync/Persistence/ImageEntity.cs +++ b/PiwigoDirectorySync/Persistence/ImageEntity.cs @@ -15,6 +15,9 @@ public class ImageEntity public required string FilePath { get; set; } + public string Name => Path.GetFileNameWithoutExtension(FilePath); + public string FileName => Path.GetFileName(FilePath); + public DateTime LastChange { get; set; } public string? Md5Sum { get; set; } diff --git a/PiwigoDirectorySync/Services/ImageSynchronizer.cs b/PiwigoDirectorySync/Services/ImageSynchronizer.cs index 785c3e7..b149dc1 100644 --- a/PiwigoDirectorySync/Services/ImageSynchronizer.cs +++ b/PiwigoDirectorySync/Services/ImageSynchronizer.cs @@ -1,11 +1,103 @@ -namespace PiwigoDirectorySync.Services; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Piwigo.Client; +using Piwigo.Client.Images; +using PiwigoDirectorySync.Infrastructure; +using PiwigoDirectorySync.Persistence; + +namespace PiwigoDirectorySync.Services; public class ImageSynchronizer : IImageSynchronizer { - public Task SynchronizeImages(int piwigoServerId, CancellationToken ct) + private readonly ILogger _logger; + private readonly PersistenceContext _persistenceContext; + private readonly IPiwigoClientFactory _piwigoClientFactory; + + public ImageSynchronizer(ILogger logger, IPiwigoClientFactory piwigoClientFactory, PersistenceContext persistenceContext) { - - - throw new NotImplementedException(); + _logger = logger; + _piwigoClientFactory = piwigoClientFactory; + _persistenceContext = persistenceContext; + } + + public async Task SynchronizeImages(int piwigoServerId, CancellationToken ct) + { + var piwigoServer = await _persistenceContext.PiwigoServers.FindAsync(new object?[] { piwigoServerId }, ct); + if (piwigoServer is null) + { + _logger.LogError("Could not sync images with piwigo server {PiwigoServerId}", piwigoServerId); + return; + } + + _logger.LogInformation("Synchronizing images of piwigo server {PiwigoServerName} using base path {PiwigoServerRootDirectory}", piwigoServer.Name, + piwigoServer.RootDirectory); + + var piwigoClient = await _piwigoClientFactory.GetPiwigoClientAsync(piwigoServer, ct); + + await GetImageIdsFromServerAsync(piwigoClient, piwigoServer, ct); + await UploadNewImagesToServerAsync(piwigoClient, piwigoServer, ct); + } + + private async Task UploadNewImagesToServerAsync(IPiwigoClient piwigoClient, ServerEntity piwigoServer, CancellationToken ct) + { + var imagesToUpload = await _persistenceContext.PiwigoImages.Include(i => i.Album) + .Where(i => i.ServerImageId == null && i.Album.ServerId == piwigoServer.Id) + .ToListAsync(ct); + + foreach (var imageEntity in imagesToUpload) + { + var fileInfo = new FileInfo(Path.Combine(piwigoServer.RootDirectory, imageEntity.FilePath)); + var imageUpload = GetImageUpload(imageEntity, fileInfo.CreationTime); + var imageUploaded = await piwigoClient.UploadImageAsync(fileInfo, imageUpload, ct); + + imageEntity.ServerImageId = imageUploaded.ImageId; + imageEntity.UploadRequired = false; + + _logger.LogInformation("Uploaded image {ImageEntityName} ({ImageEntityId}) to piwigo server with id {ImageEntityServerImageId}", imageEntity.Name, imageEntity.Id, + imageEntity.ServerImageId); + + await _persistenceContext.SaveChangesAsync(ct); + } + } + + private static ImageUpload GetImageUpload(ImageEntity imageEntity, DateTime createdAt) + { + var albums = new List<(int AlbumId, int? Rank)> { (AlbumId: imageEntity.Album.ServerAlbumId!.Value, Rank: null) }; + return new ImageUpload(imageEntity.Md5Sum!) + { + Name = imageEntity.Name, + FileName = imageEntity.FileName, + OriginalSum = imageEntity.Md5Sum!, + Albums = albums, + CreatedAt = createdAt + }; + } + + private async Task GetImageIdsFromServerAsync(IPiwigoClient piwigoClient, ServerEntity piwigoServer, CancellationToken ct) + { + var imagesToSearch = await _persistenceContext.PiwigoImages.Include(i => i.Album) + .Where(i => i.ServerImageId == null && i.Album.ServerId == piwigoServer.Id) + .ToListAsync(ct); + + var md5SumsToCheck = imagesToSearch.Where(i => i.Md5Sum != null).DistinctBy(i => i.Md5Sum).ToDictionary(i => i.Md5Sum!, i => i, StringComparer.OrdinalIgnoreCase); + + var processedImages = 0; + while (processedImages < md5SumsToCheck.Keys.Count) + { + var chunkKeys = md5SumsToCheck.Keys.Skip(processedImages).Take(100).ToArray(); + var existingImages = await piwigoClient.Image.ExistsByMd5SumsAsync(chunkKeys, ct); + + foreach (var existingImage in existingImages.Where(i => i.Value.HasValue)) + { + var imageEntity = md5SumsToCheck[existingImage.Key]; + imageEntity.ServerImageId = existingImage.Value; + _logger.LogInformation("Found image {ImageEntityName} ({ImageEntityId}) on piwigo server with id {ImageEntityServerImageId}", imageEntity.Name, imageEntity.Id, + imageEntity.ServerImageId); + } + + await _persistenceContext.SaveChangesAsync(ct); + + processedImages += chunkKeys.Length; + } } } \ No newline at end of file diff --git a/PiwigoDirectorySync/appsettings.json b/PiwigoDirectorySync/appsettings.json index 94d1ba5..484a5bc 100644 --- a/PiwigoDirectorySync/appsettings.json +++ b/PiwigoDirectorySync/appsettings.json @@ -3,7 +3,8 @@ "LogLevel": { "Default": "Debug", "System": "Warning", - "Microsoft": "Warning" + "Microsoft": "Warning", + "Piwigo.Client": "Warning" } }, "ConnectionStrings": {