Implemented synchronizeLocalImageMetadata to build the local image database

internal/app/app.go

@@ -43,11 +43,16 @@ func Run() {
 		logErrorAndExit(err, 5)
 	}
 
-	err = synchronizeImages(context, filesystemNodes, categories)
+	err = synchronizeLocalImageMetadata(context.dataStore, filesystemNodes, localFileStructure.CalculateFileCheckSums)
 	if err != nil {
 		logErrorAndExit(err, 6)
 	}
 
+	err = synchronizeImages(context.piwigo, context.dataStore, categories)
+	if err != nil {
+		logErrorAndExit(err, 7)
+	}
+
 	_ = piwigo.Logout(context.piwigo)
 }
 

internal/app/images.go

@@ -4,95 +4,136 @@ import (
 	"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure"
 	"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
 	"github.com/sirupsen/logrus"
-	"sort"
+	"path/filepath"
 )
 
-func synchronizeImages(context *appContext, fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*piwigo.PiwigoCategory) error {
+type fileChecksumCalculator func(filePath string) (string, error)
-
-	// to make use of the new local data store, we have to rethink and refactor the whole local detection process
-
-	// extend the storage of the images to keep track of upload state
-
-	// TBD: How to deal with updates -> delete / upload all based on md5 sums
 
+// to make use of the new local data store, we have to rethink and refactor the whole local detection process
+// extend the storage of the images to keep track of upload state
+// TBD: How to deal with updates -> delete / upload all based on md5 sums
+func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error {
 	// STEP 1 - update and sync local datastore with filesystem
 	// - walk through all files of the fileSystem map
 	// - get file metadata from filesystem (date, filename, dir, modtime etc.)
 	// - recalculate md5 sum if file changed referring to the stored record (reduces load after first calculation a lot)
 	// - mark metadata as upload required if changed or new
 
-	// STEP 2 - get file states from piwigo (pwg.images.checkFiles)
+	logrus.Info("Synchronizing local image metadata database with local available images")
-	// - get upload status of md5 sum from piwigo for all marked to upload
-	// - check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times)
 
-	// STEP 3: Upload missing images
+	for _, file := range fileSystemNodes {
-	// - upload file in chunks
+		if file.IsDir {
-	// - assign image to category
+			// we are only interested in files not directories
+			continue
+		}
 
-	imageFiles, err := localFileStructure.GetImageList(fileSystem)
+		metadata, err := metadataStorage.GetImageMetadata(file.Key)
-	if err != nil {
+		if err == ErrorRecordNotFound {
+			logrus.Debugf("No metadata for %s found. Creating new entry.", file.Key)
+			metadata = ImageMetaData{}
+			metadata.Filename = file.Name
+			metadata.RelativeImagePath = file.Key
+			metadata.CategoryPath = filepath.Dir(file.Key)
+		} else if err != nil {
+			logrus.Errorf("Could not get metadata due to trouble. Cancelling - %s", err)
 			return err
 		}
 
-	missingFiles, err := findMissingImages(context, imageFiles)
+		if metadata.LastChange.Equal(file.ModTime) {
+			logrus.Infof("No changed detected on file %s -> keeping current state", file.Key)
+			continue
+		}
+
+		metadata.LastChange = file.ModTime
+		metadata.UploadRequired = true
+		metadata.Md5Sum, err = checksumCalculator(file.Path)
+		if err != nil {
+			logrus.Warnf("Could not calculate checksum for file %s. Skipping...", file.Path)
+			continue
+		}
+
+		err = metadataStorage.SaveImageMetadata(metadata)
 		if err != nil {
 			return err
 		}
-
-	err = uploadImages(context, missingFiles, existingCategories)
-	if err != nil {
-		return err
 	}
 
-	logrus.Infof("Synchronized %d files.", len(missingFiles))
-
 	return nil
 }
 
-func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) {
+// STEP 2 - get file states from piwigo (pwg.images.checkFiles)
+// - get upload status of md5 sum from piwigo for all marked to upload
+// - check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times)
 
-	logrus.Debugln("Preparing lookuplist for missing files...")
+// STEP 3: Upload missing images
+// - upload file in chunks
+// - assign image to category
 
-	files := make([]string, 0, len(imageFiles))
+func synchronizeImages(piwigo *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider, existingCategories map[string]*piwigo.PiwigoCategory) error {
-	md5map := make(map[string]*localFileStructure.ImageNode, len(imageFiles))
+	//imageFiles, err := localFileStructure.GetImageList(fileSystem)
-	for _, file := range imageFiles {
+	//if err != nil {
-		md5map[file.Md5Sum] = file
+	//	return err
-		files = append(files, file.Md5Sum)
+	//}
-	}
+	//
-
+	//missingFiles, err := findMissingImages(context, imageFiles)
-	missingSums, err := piwigo.ImageUploadRequired(context.piwigo, files)
+	//if err != nil {
-	if err != nil {
+	//	return err
-		return nil, err
+	//}
-	}
+	//
-
+	//err = uploadImages(context, missingFiles, existingCategories)
-	missingFiles := make([]*localFileStructure.ImageNode, 0, len(missingSums))
+	//if err != nil {
-	for _, sum := range missingSums {
+	//	return err
-		file := md5map[sum]
+	//}
-		logrus.Infof("Found missing file %s", file.Path)
+	//
-		missingFiles = append(missingFiles, file)
+	//logrus.Infof("Synchronized %d files.", len(missingFiles))
-	}
-
-	logrus.Infof("Found %d missing files", len(missingFiles))
-
-	return missingFiles, nil
-}
-
-func uploadImages(context *appContext, missingFiles []*localFileStructure.ImageNode, existingCategories map[string]*piwigo.PiwigoCategory) error {
-
-	// We sort the files by path to populate per category and not random by file
-	sort.Slice(missingFiles, func(i, j int) bool {
-		return missingFiles[i].Path < missingFiles[j].Path
-	})
-
-	for _, file := range missingFiles {
-		categoryId := existingCategories[file.CategoryName].Id
-
-		imageId, err := piwigo.UploadImage(context.piwigo, file.Path, file.Md5Sum, categoryId)
-		if err != nil {
-			return err
-		}
-		file.ImageId = imageId
-	}
 
 	return nil
 }
+
+//func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) {
+//
+//	logrus.Debugln("Preparing lookuplist for missing files...")
+//
+//	files := make([]string, 0, len(imageFiles))
+//	md5map := make(map[string]*localFileStructure.ImageNode, len(imageFiles))
+//	for _, file := range imageFiles {
+//		md5map[file.Md5Sum] = file
+//		files = append(files, file.Md5Sum)
+//	}
+//
+//	missingSums, err := piwigo.ImageUploadRequired(context.piwigo, files)
+//	if err != nil {
+//		return nil, err
+//	}
+//
+//	missingFiles := make([]*localFileStructure.ImageNode, 0, len(missingSums))
+//	for _, sum := range missingSums {
+//		file := md5map[sum]
+//		logrus.Infof("Found missing file %s", file.Path)
+//		missingFiles = append(missingFiles, file)
+//	}
+//
+//	logrus.Infof("Found %d missing files", len(missingFiles))
+//
+//	return missingFiles, nil
+//}
+//
+//func uploadImages(context *appContext, missingFiles []*localFileStructure.ImageNode, existingCategories map[string]*piwigo.PiwigoCategory) error {
+//
+//	// We sort the files by path to populate per category and not random by file
+//	sort.Slice(missingFiles, func(i, j int) bool {
+//		return missingFiles[i].Path < missingFiles[j].Path
+//	})
+//
+//	for _, file := range missingFiles {
+//		categoryId := existingCategories[file.CategoryName].Id
+//
+//		imageId, err := piwigo.UploadImage(context.piwigo, file.Path, file.Md5Sum, categoryId)
+//		if err != nil {
+//			return err
+//		}
+//		file.ImageId = imageId
+//	}
+//
+//	return nil
+//}

internal/app/images_test.go

@@ -0,0 +1,188 @@
+package app
+
+import (
+	"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure"
+	"testing"
+	"time"
+)
+
+func TestSynchronizeLocalImageMetadataShouldDoNothingIfEmpty(t *testing.T) {
+	db := NewtestStore()
+	fileSystemNodes := map[string]*localFileStructure.FilesystemNode{}
+
+	err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(db.savedMetadata) > 0 {
+		t.Error("There were metadata records saved but non expected!")
+	}
+}
+
+func TestSynchronizeLocalImageMetadataShouldAddNewMetadata(t *testing.T) {
+	db := NewtestStore()
+
+	testFileSystemNode := &localFileStructure.FilesystemNode{
+		Key:     "2019/shooting1/abc.jpg",
+		ModTime: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC),
+		Name:    "abc.jpg",
+		Path:    "2019/shooting1/abc.jpg",
+		IsDir:   false}
+
+	fileSystemNodes := map[string]*localFileStructure.FilesystemNode{}
+	fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
+
+	// execute the sync metadata based on the file system results
+	err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// check if data are saved
+	savedData, exist := db.savedMetadata[testFileSystemNode.Key]
+	if !exist {
+		t.Fatal("Could not find correct metadata!")
+	}
+	if savedData.RelativeImagePath != testFileSystemNode.Key {
+		t.Errorf("relativeImagePath %s on db image metadata is not set to %s!", savedData.RelativeImagePath, testFileSystemNode.Key)
+	}
+	if savedData.LastChange != testFileSystemNode.ModTime {
+		t.Error("lastChange on db image metadata is not set to the right date!")
+	}
+	if savedData.Filename != "abc.jpg" {
+		t.Error("filename on db image metadata is not set to abc.jpg!")
+	}
+	if savedData.Md5Sum != testFileSystemNode.Key {
+		t.Errorf("md5sum %s on db image metadata is not set to %s!", savedData.Md5Sum, testFileSystemNode.Key)
+	}
+	if savedData.UploadRequired != true {
+		t.Errorf("uploadRequired on db image metadata is not set to true!")
+	}
+}
+
+func TestSynchronizeLocalImageMetadataShouldMarkChangedEntriesAsUploads(t *testing.T) {
+	db := NewtestStore()
+	db.savedMetadata["2019/shooting1/abc.jpg"] = ImageMetaData{
+		Md5Sum:            "2019/shooting1/abc.jpg",
+		RelativeImagePath: "2019/shooting1/abc.jpg",
+		UploadRequired:    false,
+		LastChange:        time.Date(2019, 01, 01, 00, 0, 0, 0, time.UTC),
+		Filename:          "abc.jpg",
+	}
+
+	testFileSystemNode := &localFileStructure.FilesystemNode{
+		Key:     "2019/shooting1/abc.jpg",
+		ModTime: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC),
+		Name:    "abc.jpg",
+		Path:    "2019/shooting1/abc.jpg",
+		IsDir:   false}
+
+	fileSystemNodes := map[string]*localFileStructure.FilesystemNode{}
+	fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
+
+	// execute the sync metadata based on the file system results
+	err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// check if data are saved
+	savedData, exist := db.savedMetadata[testFileSystemNode.Key]
+	if !exist {
+		t.Fatal("Could not find correct metadata!")
+	}
+	if savedData.LastChange != testFileSystemNode.ModTime {
+		t.Error("lastChange on db image metadata is not set to the right date!")
+	}
+	if savedData.UploadRequired != true {
+		t.Errorf("uploadRequired on db image metadata is not set to true!")
+	}
+}
+
+func TestSynchronizeLocalImageMetadataShouldNotMarkUnchangedFilesToUpload(t *testing.T) {
+	db := NewtestStore()
+	db.savedMetadata["2019/shooting1/abc.jpg"] = ImageMetaData{
+		Md5Sum:            "2019/shooting1/abc.jpg",
+		RelativeImagePath: "2019/shooting1/abc.jpg",
+		UploadRequired:    false,
+		LastChange:        time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC),
+		Filename:          "abc.jpg",
+	}
+
+	testFileSystemNode := &localFileStructure.FilesystemNode{
+		Key:     "2019/shooting1/abc.jpg",
+		ModTime: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC),
+		Name:    "abc.jpg",
+		Path:    "2019/shooting1/abc.jpg",
+		IsDir:   false}
+
+	fileSystemNodes := map[string]*localFileStructure.FilesystemNode{}
+	fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
+
+	// execute the sync metadata based on the file system results
+	err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// check if data are saved
+	savedData, exist := db.savedMetadata[testFileSystemNode.Key]
+	if !exist {
+		t.Fatal("Could not find correct metadata!")
+	}
+	if savedData.UploadRequired {
+		t.Errorf("uploadRequired on db image metadata is set to true, but should not be on unchanged items!")
+	}
+}
+
+func TestSynchronizeLocalImageMetadataShouldNotProcessDirectories(t *testing.T) {
+	db := NewtestStore()
+
+	testFileSystemNode := &localFileStructure.FilesystemNode{
+		Key:     "2019/shooting1",
+		ModTime: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC),
+		Name:    "shooting1",
+		Path:    "2019/shooting1/",
+		IsDir:   true}
+
+	fileSystemNodes := map[string]*localFileStructure.FilesystemNode{}
+	fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
+
+	// execute the sync metadata based on the file system results
+	err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(db.savedMetadata) > 0 {
+		t.Error("There were metadata records saved but non expected!")
+	}
+}
+
+// test metadata store to store save the metadat and simulate the database
+type testStore struct {
+	savedMetadata map[string]ImageMetaData
+}
+
+func NewtestStore() *testStore {
+	return &testStore{savedMetadata: make(map[string]ImageMetaData)}
+}
+
+func (s *testStore) GetImageMetadata(relativePath string) (ImageMetaData, error) {
+	metadata, exist := s.savedMetadata[relativePath]
+	if !exist {
+		return ImageMetaData{}, ErrorRecordNotFound
+	}
+	return metadata, nil
+}
+
+func (s *testStore) SaveImageMetadata(m ImageMetaData) error {
+	s.savedMetadata[m.RelativeImagePath] = m
+	return nil
+}
+
+// to make the sync testable, we pass in a simple mock that returns the filepath as checksum
+func testChecksumCalculator(file string) (string, error) {
+	return file, nil
+}