From f3f864ec49f8ec328ea7ff92de4e4425ce5f8fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20H=C3=A4felfinger?= Date: Tue, 19 Mar 2019 23:08:28 +0100 Subject: [PATCH] Removed old empty method added synchronizePiwigoMetadata implementations to update local metadata from piwigo --- internal/app/app.go | 8 +- internal/app/images.go | 150 ++++++++++++++++++------------------ internal/app/images_test.go | 4 + 3 files changed, 85 insertions(+), 77 deletions(-) diff --git a/internal/app/app.go b/internal/app/app.go index 8c1ad83..3effcb2 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -53,10 +53,10 @@ func Run() { logErrorAndExit(err, 7) } - err = synchronizeImages(context.piwigo, context.dataStore, categories) - if err != nil { - logErrorAndExit(err, 8) - } + //err = synchronizeImages(context.piwigo, context.dataStore, categories) + //if err != nil { + // logErrorAndExit(err, 8) + //} _ = piwigo.Logout(context.piwigo) } diff --git a/internal/app/images.go b/internal/app/images.go index 7f5f392..a90bb8d 100644 --- a/internal/app/images.go +++ b/internal/app/images.go @@ -1,26 +1,22 @@ package app import ( - "errors" "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure" "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo" "github.com/sirupsen/logrus" "path/filepath" ) -type fileChecksumCalculator func(filePath string) (string, error) - // to make use of the new local data store, we have to rethink and refactor the whole local detection process // extend the storage of the images to keep track of upload state // TBD: How to deal with updates -> delete / upload all based on md5 sums -func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error { - // STEP 1 - update and sync local datastore with filesystem - // - walk through all files of the fileSystem map - // - get file metadata from filesystem (date, filename, dir, modtime etc.) - // - recalculate md5 sum if file changed referring to the stored record (reduces load after first calculation a lot) - // - mark metadata as upload required if changed or new - logrus.Debugf("Starting synchronizeLocalImageMetadata") +type fileChecksumCalculator func(filePath string) (string, error) + +// Update the local image metadata by walking through all found files and check if the modification date has changed +// or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well. +func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error { + logrus.Debugf("Starting synchronizeLocalImageMetadata") logrus.Info("Synchronizing local image metadata database with local available images") for _, file := range fileSystemNodes { @@ -64,81 +60,89 @@ func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSy return nil } -func synchronizePiwigoMetadata(piwigo *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider) error { - // STEP 2 - get file states from piwigo (pwg.images.checkFiles) - // - make bulk query possible - // - get upload status of md5 sum from piwigo for all marked to upload - // - build method to update database with new piwigoId by md5 sum - // - set UploadRequired to false and the returned piwigoid - // - check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later - +// This method agregates the check for files with missing piwigoids and if changed files need to be uploaded again. +func synchronizePiwigoMetadata(piwigoCtx *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider) error { + // TODO: check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later logrus.Debugf("Starting synchronizePiwigoMetadata") - images, err := metadataStorage.ImageMetadataToUpload() - if err != nil { + err := updatePiwigoIdIfAlreadyUploaded(metadataStorage, piwigoCtx) + if err != nil { return err } - logrus.Infof("Synchronizing metadata with existing Piwigo data...") - for img := range images { - logrus.Debug(img) + err = checkPiwigoForChangedImages(metadataStorage, piwigoCtx) + if err != nil { + return err } - logrus.Debugf("Finished synchronizePiwigoMetadata") - return errors.New("N/A") + return nil +} + +// Check all images with upload required if they are really changed and need to be uploaded to the server. +func checkPiwigoForChangedImages(provider ImageMetadataProvider, piwigoCtx *piwigo.PiwigoContext) error { + logrus.Infof("checking for pending files that are already on piwigo and updating piwigoids...") + + images, err := provider.ImageMetadataToUpload() + if err != nil { + return err + } + + for _, img := range images { + if img.PiwigoId == 0 { + continue + } + state, err := piwigo.ImageCheckFile(piwigoCtx, img.PiwigoId, img.Md5Sum) + if err != nil { + logrus.Warnf("Error during file change check of file %s", img.RelativeImagePath) + continue + } + + if state == piwigo.ImageStateUptodate { + logrus.Debugf("File %s - %d has not changed", img.RelativeImagePath, img.PiwigoId) + img.UploadRequired = false + err = provider.SaveImageMetadata(*img) + if err != nil { + logrus.Warnf("Could not save image data of image %s", img.RelativeImagePath) + } + } + } + + return nil +} + +// This function calls piwigo and checks if the given md5sum is already present. +// Only files without a piwigo id are used to query the server. +func updatePiwigoIdIfAlreadyUploaded(provider ImageMetadataProvider, piwiCtx *piwigo.PiwigoContext) error { + logrus.Infof("checking for pending files that are already on piwigo and updating piwigoids...") + images, err := provider.ImageMetadataToUpload() + if err != nil { + return err + } + + logrus.Debugln("Preparing lookuplist for missing piwigo ids...") + files := make([]string, 0, len(images)) + for _, img := range images { + if img.PiwigoId == 0 { + files = append(files, img.Md5Sum) + } + } + missingResults, err := piwigo.ImagesExistOnPiwigo(piwiCtx, files) + if err != nil { + return err + } + for md5sum, piwigoId := range missingResults { + logrus.Debugf("Setting piwigo id of %s to %d", md5sum, piwigoId) + err = provider.SavePiwigoIdAndUpdateUploadFlag(md5sum, piwigoId) + if err != nil { + logrus.Warnf("Could not save piwigo id %d for file %s", piwigoId, md5sum) + } + } + return nil } // STEP 3: Upload missing images // - upload file in chunks // - assign image to category -func synchronizeImages(piwigo *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider, existingCategories map[string]*piwigo.PiwigoCategory) error { - //imageFiles, err := localFileStructure.GetImageList(fileSystem) - //if err != nil { - // return err - //} - // - //missingFiles, err := findMissingImages(context, imageFiles) - //if err != nil { - // return err - //} - // - //err = uploadImages(context, missingFiles, existingCategories) - //if err != nil { - // return err - //} - // - //logrus.Infof("Synchronized %d files.", len(missingFiles)) - - return nil -} - -//func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) { -// -// logrus.Debugln("Preparing lookuplist for missing files...") -// -// files := make([]string, 0, len(imageFiles)) -// md5map := make(map[string]*localFileStructure.ImageNode, len(imageFiles)) -// for _, file := range imageFiles { -// md5map[file.Md5Sum] = file -// files = append(files, file.Md5Sum) -// } -// -// missingSums, err := piwigo.ImageUploadRequired(context.piwigo, files) -// if err != nil { -// return nil, err -// } -// -// missingFiles := make([]*localFileStructure.ImageNode, 0, len(missingSums)) -// for _, sum := range missingSums { -// file := md5map[sum] -// logrus.Infof("Found missing file %s", file.Path) -// missingFiles = append(missingFiles, file) -// } -// -// logrus.Infof("Found %d missing files", len(missingFiles)) -// -// return missingFiles, nil -//} // //func uploadImages(context *appContext, missingFiles []*localFileStructure.ImageNode, existingCategories map[string]*piwigo.PiwigoCategory) error { // diff --git a/internal/app/images_test.go b/internal/app/images_test.go index d0bb89a..f5a7e19 100644 --- a/internal/app/images_test.go +++ b/internal/app/images_test.go @@ -206,6 +206,10 @@ func (d *testStore) ImageMetadataToUpload() ([]*ImageMetaData, error) { return nil, errors.New("N/A") } +func (d *testStore) SavePiwigoIdAndUpdateUploadFlag(md5Sum string, piwigoId int) error { + return errors.New("N/A") +} + // to make the sync testable, we pass in a simple mock that returns the filepath as checksum func testChecksumCalculator(file string) (string, error) { return file, nil