Removed old empty method

added synchronizePiwigoMetadata implementations to update local metadata from piwigo

internal/app/app.go

@@ -53,10 +53,10 @@ func Run() {
 		logErrorAndExit(err, 7)
 	}
 
-	err = synchronizeImages(context.piwigo, context.dataStore, categories)
+	//err = synchronizeImages(context.piwigo, context.dataStore, categories)
-	if err != nil {
+	//if err != nil {
-		logErrorAndExit(err, 8)
+	//	logErrorAndExit(err, 8)
-	}
+	//}
 
 	_ = piwigo.Logout(context.piwigo)
 }

internal/app/images.go

@@ -1,26 +1,22 @@
 package app
 
 import (
-	"errors"
 	"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure"
 	"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
 	"github.com/sirupsen/logrus"
 	"path/filepath"
 )
 
-type fileChecksumCalculator func(filePath string) (string, error)
-
 // to make use of the new local data store, we have to rethink and refactor the whole local detection process
 // extend the storage of the images to keep track of upload state
 // TBD: How to deal with updates -> delete / upload all based on md5 sums
-func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error {
-	// STEP 1 - update and sync local datastore with filesystem
-	// - walk through all files of the fileSystem map
-	// - get file metadata from filesystem (date, filename, dir, modtime etc.)
-	// - recalculate md5 sum if file changed referring to the stored record (reduces load after first calculation a lot)
-	// - mark metadata as upload required if changed or new
-	logrus.Debugf("Starting synchronizeLocalImageMetadata")
 
+type fileChecksumCalculator func(filePath string) (string, error)
+
+// Update the local image metadata by walking through all found files and check if the modification date has changed
+// or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well.
+func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error {
+	logrus.Debugf("Starting synchronizeLocalImageMetadata")
 	logrus.Info("Synchronizing local image metadata database with local available images")
 
 	for _, file := range fileSystemNodes {
@@ -64,81 +60,89 @@ func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSy
 	return nil
 }
 
-func synchronizePiwigoMetadata(piwigo *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider) error  {
+// This method agregates the check for files with missing piwigoids and if changed files need to be uploaded again.
-	// STEP 2 - get file states from piwigo (pwg.images.checkFiles)
+func synchronizePiwigoMetadata(piwigoCtx *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider) error {
-	// - make bulk query possible
+	// TODO: check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later
-	// - get upload status of md5 sum from piwigo for all marked to upload
-	// - build method to update database with new piwigoId by md5 sum
-	// - set UploadRequired to false and the returned piwigoid
-	// - check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later
-
 	logrus.Debugf("Starting synchronizePiwigoMetadata")
-	images, err := metadataStorage.ImageMetadataToUpload()
+	err := updatePiwigoIdIfAlreadyUploaded(metadataStorage, piwigoCtx)
-	if err != nil  {
+	if err != nil {
 		return err
 	}
 
-	logrus.Infof("Synchronizing metadata with existing Piwigo data...")
+	err = checkPiwigoForChangedImages(metadataStorage, piwigoCtx)
-	for img := range images {
+	if err != nil {
-		logrus.Debug(img)
+		return err
 	}
 
-	logrus.Debugf("Finished synchronizePiwigoMetadata")
+	return nil
-	return errors.New("N/A")
+}
+
+// Check all images with upload required if they are really changed and need to be uploaded to the server.
+func checkPiwigoForChangedImages(provider ImageMetadataProvider, piwigoCtx *piwigo.PiwigoContext) error {
+	logrus.Infof("checking for pending files that are already on piwigo and updating piwigoids...")
+
+	images, err := provider.ImageMetadataToUpload()
+	if err != nil {
+		return err
+	}
+
+	for _, img := range images {
+		if img.PiwigoId == 0 {
+			continue
+		}
+		state, err := piwigo.ImageCheckFile(piwigoCtx, img.PiwigoId, img.Md5Sum)
+		if err != nil {
+			logrus.Warnf("Error during file change check of file %s", img.RelativeImagePath)
+			continue
+		}
+
+		if state == piwigo.ImageStateUptodate {
+			logrus.Debugf("File %s - %d has not changed", img.RelativeImagePath, img.PiwigoId)
+			img.UploadRequired = false
+			err = provider.SaveImageMetadata(*img)
+			if err != nil {
+				logrus.Warnf("Could not save image data of image %s", img.RelativeImagePath)
+			}
+		}
+	}
+
+	return nil
+}
+
+// This function calls piwigo and checks if the given md5sum is already present.
+// Only files without a piwigo id are used to query the server.
+func updatePiwigoIdIfAlreadyUploaded(provider ImageMetadataProvider, piwiCtx *piwigo.PiwigoContext) error {
+	logrus.Infof("checking for pending files that are already on piwigo and updating piwigoids...")
+	images, err := provider.ImageMetadataToUpload()
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugln("Preparing lookuplist for missing piwigo ids...")
+	files := make([]string, 0, len(images))
+	for _, img := range images {
+		if img.PiwigoId == 0 {
+			files = append(files, img.Md5Sum)
+		}
+	}
+	missingResults, err := piwigo.ImagesExistOnPiwigo(piwiCtx, files)
+	if err != nil {
+		return err
+	}
+	for md5sum, piwigoId := range missingResults {
+		logrus.Debugf("Setting piwigo id of %s to %d", md5sum, piwigoId)
+		err = provider.SavePiwigoIdAndUpdateUploadFlag(md5sum, piwigoId)
+		if err != nil {
+			logrus.Warnf("Could not save piwigo id %d for file %s", piwigoId, md5sum)
+		}
+	}
+	return nil
 }
 
 // STEP 3: Upload missing images
 // - upload file in chunks
 // - assign image to category
 
-func synchronizeImages(piwigo *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider, existingCategories map[string]*piwigo.PiwigoCategory) error {
-	//imageFiles, err := localFileStructure.GetImageList(fileSystem)
-	//if err != nil {
-	//	return err
-	//}
-	//
-	//missingFiles, err := findMissingImages(context, imageFiles)
-	//if err != nil {
-	//	return err
-	//}
-	//
-	//err = uploadImages(context, missingFiles, existingCategories)
-	//if err != nil {
-	//	return err
-	//}
-	//
-	//logrus.Infof("Synchronized %d files.", len(missingFiles))
-
-	return nil
-}
-
-//func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) {
-//
-//	logrus.Debugln("Preparing lookuplist for missing files...")
-//
-//	files := make([]string, 0, len(imageFiles))
-//	md5map := make(map[string]*localFileStructure.ImageNode, len(imageFiles))
-//	for _, file := range imageFiles {
-//		md5map[file.Md5Sum] = file
-//		files = append(files, file.Md5Sum)
-//	}
-//
-//	missingSums, err := piwigo.ImageUploadRequired(context.piwigo, files)
-//	if err != nil {
-//		return nil, err
-//	}
-//
-//	missingFiles := make([]*localFileStructure.ImageNode, 0, len(missingSums))
-//	for _, sum := range missingSums {
-//		file := md5map[sum]
-//		logrus.Infof("Found missing file %s", file.Path)
-//		missingFiles = append(missingFiles, file)
-//	}
-//
-//	logrus.Infof("Found %d missing files", len(missingFiles))
-//
-//	return missingFiles, nil
-//}
 //
 //func uploadImages(context *appContext, missingFiles []*localFileStructure.ImageNode, existingCategories map[string]*piwigo.PiwigoCategory) error {
 //

internal/app/images_test.go

@@ -206,6 +206,10 @@ func (d *testStore) ImageMetadataToUpload() ([]*ImageMetaData, error) {
 	return nil, errors.New("N/A")
 }
 
+func (d *testStore) SavePiwigoIdAndUpdateUploadFlag(md5Sum string, piwigoId int) error {
+	return errors.New("N/A")
+}
+
 // to make the sync testable, we pass in a simple mock that returns the filepath as checksum
 func testChecksumCalculator(file string) (string, error) {
 	return file, nil