From 798b74d07178dd11e9d0e65b7555c6658dc1688c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20H=C3=A4felfinger?= Date: Sun, 17 Mar 2019 23:05:17 +0100 Subject: [PATCH] Implemented synchronizeLocalImageMetadata to build the local image database --- internal/app/app.go | 7 +- internal/app/images.go | 183 +++++++++++++++++++++-------------- internal/app/images_test.go | 188 ++++++++++++++++++++++++++++++++++++ 3 files changed, 306 insertions(+), 72 deletions(-) create mode 100644 internal/app/images_test.go diff --git a/internal/app/app.go b/internal/app/app.go index 58f7dae..cd7d373 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -43,11 +43,16 @@ func Run() { logErrorAndExit(err, 5) } - err = synchronizeImages(context, filesystemNodes, categories) + err = synchronizeLocalImageMetadata(context.dataStore, filesystemNodes, localFileStructure.CalculateFileCheckSums) if err != nil { logErrorAndExit(err, 6) } + err = synchronizeImages(context.piwigo, context.dataStore, categories) + if err != nil { + logErrorAndExit(err, 7) + } + _ = piwigo.Logout(context.piwigo) } diff --git a/internal/app/images.go b/internal/app/images.go index 409debe..8de9041 100644 --- a/internal/app/images.go +++ b/internal/app/images.go @@ -4,95 +4,136 @@ import ( "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure" "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo" "github.com/sirupsen/logrus" - "sort" + "path/filepath" ) -func synchronizeImages(context *appContext, fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*piwigo.PiwigoCategory) error { - - // to make use of the new local data store, we have to rethink and refactor the whole local detection process - - // extend the storage of the images to keep track of upload state - - // TBD: How to deal with updates -> delete / upload all based on md5 sums +type fileChecksumCalculator func(filePath string) (string, error) +// to make use of the new local data store, we have to rethink and refactor the whole local detection process +// extend the storage of the images to keep track of upload state +// TBD: How to deal with updates -> delete / upload all based on md5 sums +func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error { // STEP 1 - update and sync local datastore with filesystem // - walk through all files of the fileSystem map // - get file metadata from filesystem (date, filename, dir, modtime etc.) // - recalculate md5 sum if file changed referring to the stored record (reduces load after first calculation a lot) // - mark metadata as upload required if changed or new - // STEP 2 - get file states from piwigo (pwg.images.checkFiles) - // - get upload status of md5 sum from piwigo for all marked to upload - // - check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) + logrus.Info("Synchronizing local image metadata database with local available images") - // STEP 3: Upload missing images - // - upload file in chunks - // - assign image to category + for _, file := range fileSystemNodes { + if file.IsDir { + // we are only interested in files not directories + continue + } - imageFiles, err := localFileStructure.GetImageList(fileSystem) - if err != nil { - return err - } + metadata, err := metadataStorage.GetImageMetadata(file.Key) + if err == ErrorRecordNotFound { + logrus.Debugf("No metadata for %s found. Creating new entry.", file.Key) + metadata = ImageMetaData{} + metadata.Filename = file.Name + metadata.RelativeImagePath = file.Key + metadata.CategoryPath = filepath.Dir(file.Key) + } else if err != nil { + logrus.Errorf("Could not get metadata due to trouble. Cancelling - %s", err) + return err + } - missingFiles, err := findMissingImages(context, imageFiles) - if err != nil { - return err - } + if metadata.LastChange.Equal(file.ModTime) { + logrus.Infof("No changed detected on file %s -> keeping current state", file.Key) + continue + } - err = uploadImages(context, missingFiles, existingCategories) - if err != nil { - return err - } + metadata.LastChange = file.ModTime + metadata.UploadRequired = true + metadata.Md5Sum, err = checksumCalculator(file.Path) + if err != nil { + logrus.Warnf("Could not calculate checksum for file %s. Skipping...", file.Path) + continue + } - logrus.Infof("Synchronized %d files.", len(missingFiles)) - - return nil -} - -func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) { - - logrus.Debugln("Preparing lookuplist for missing files...") - - files := make([]string, 0, len(imageFiles)) - md5map := make(map[string]*localFileStructure.ImageNode, len(imageFiles)) - for _, file := range imageFiles { - md5map[file.Md5Sum] = file - files = append(files, file.Md5Sum) - } - - missingSums, err := piwigo.ImageUploadRequired(context.piwigo, files) - if err != nil { - return nil, err - } - - missingFiles := make([]*localFileStructure.ImageNode, 0, len(missingSums)) - for _, sum := range missingSums { - file := md5map[sum] - logrus.Infof("Found missing file %s", file.Path) - missingFiles = append(missingFiles, file) - } - - logrus.Infof("Found %d missing files", len(missingFiles)) - - return missingFiles, nil -} - -func uploadImages(context *appContext, missingFiles []*localFileStructure.ImageNode, existingCategories map[string]*piwigo.PiwigoCategory) error { - - // We sort the files by path to populate per category and not random by file - sort.Slice(missingFiles, func(i, j int) bool { - return missingFiles[i].Path < missingFiles[j].Path - }) - - for _, file := range missingFiles { - categoryId := existingCategories[file.CategoryName].Id - - imageId, err := piwigo.UploadImage(context.piwigo, file.Path, file.Md5Sum, categoryId) + err = metadataStorage.SaveImageMetadata(metadata) if err != nil { return err } - file.ImageId = imageId } return nil } + +// STEP 2 - get file states from piwigo (pwg.images.checkFiles) +// - get upload status of md5 sum from piwigo for all marked to upload +// - check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) + +// STEP 3: Upload missing images +// - upload file in chunks +// - assign image to category + +func synchronizeImages(piwigo *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider, existingCategories map[string]*piwigo.PiwigoCategory) error { + //imageFiles, err := localFileStructure.GetImageList(fileSystem) + //if err != nil { + // return err + //} + // + //missingFiles, err := findMissingImages(context, imageFiles) + //if err != nil { + // return err + //} + // + //err = uploadImages(context, missingFiles, existingCategories) + //if err != nil { + // return err + //} + // + //logrus.Infof("Synchronized %d files.", len(missingFiles)) + + return nil +} + +//func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) { +// +// logrus.Debugln("Preparing lookuplist for missing files...") +// +// files := make([]string, 0, len(imageFiles)) +// md5map := make(map[string]*localFileStructure.ImageNode, len(imageFiles)) +// for _, file := range imageFiles { +// md5map[file.Md5Sum] = file +// files = append(files, file.Md5Sum) +// } +// +// missingSums, err := piwigo.ImageUploadRequired(context.piwigo, files) +// if err != nil { +// return nil, err +// } +// +// missingFiles := make([]*localFileStructure.ImageNode, 0, len(missingSums)) +// for _, sum := range missingSums { +// file := md5map[sum] +// logrus.Infof("Found missing file %s", file.Path) +// missingFiles = append(missingFiles, file) +// } +// +// logrus.Infof("Found %d missing files", len(missingFiles)) +// +// return missingFiles, nil +//} +// +//func uploadImages(context *appContext, missingFiles []*localFileStructure.ImageNode, existingCategories map[string]*piwigo.PiwigoCategory) error { +// +// // We sort the files by path to populate per category and not random by file +// sort.Slice(missingFiles, func(i, j int) bool { +// return missingFiles[i].Path < missingFiles[j].Path +// }) +// +// for _, file := range missingFiles { +// categoryId := existingCategories[file.CategoryName].Id +// +// imageId, err := piwigo.UploadImage(context.piwigo, file.Path, file.Md5Sum, categoryId) +// if err != nil { +// return err +// } +// file.ImageId = imageId +// } +// +// return nil +//} diff --git a/internal/app/images_test.go b/internal/app/images_test.go new file mode 100644 index 0000000..d76e263 --- /dev/null +++ b/internal/app/images_test.go @@ -0,0 +1,188 @@ +package app + +import ( + "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure" + "testing" + "time" +) + +func TestSynchronizeLocalImageMetadataShouldDoNothingIfEmpty(t *testing.T) { + db := NewtestStore() + fileSystemNodes := map[string]*localFileStructure.FilesystemNode{} + + err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator) + if err != nil { + t.Error(err) + } + + if len(db.savedMetadata) > 0 { + t.Error("There were metadata records saved but non expected!") + } +} + +func TestSynchronizeLocalImageMetadataShouldAddNewMetadata(t *testing.T) { + db := NewtestStore() + + testFileSystemNode := &localFileStructure.FilesystemNode{ + Key: "2019/shooting1/abc.jpg", + ModTime: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC), + Name: "abc.jpg", + Path: "2019/shooting1/abc.jpg", + IsDir: false} + + fileSystemNodes := map[string]*localFileStructure.FilesystemNode{} + fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode + + // execute the sync metadata based on the file system results + err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator) + if err != nil { + t.Error(err) + } + + // check if data are saved + savedData, exist := db.savedMetadata[testFileSystemNode.Key] + if !exist { + t.Fatal("Could not find correct metadata!") + } + if savedData.RelativeImagePath != testFileSystemNode.Key { + t.Errorf("relativeImagePath %s on db image metadata is not set to %s!", savedData.RelativeImagePath, testFileSystemNode.Key) + } + if savedData.LastChange != testFileSystemNode.ModTime { + t.Error("lastChange on db image metadata is not set to the right date!") + } + if savedData.Filename != "abc.jpg" { + t.Error("filename on db image metadata is not set to abc.jpg!") + } + if savedData.Md5Sum != testFileSystemNode.Key { + t.Errorf("md5sum %s on db image metadata is not set to %s!", savedData.Md5Sum, testFileSystemNode.Key) + } + if savedData.UploadRequired != true { + t.Errorf("uploadRequired on db image metadata is not set to true!") + } +} + +func TestSynchronizeLocalImageMetadataShouldMarkChangedEntriesAsUploads(t *testing.T) { + db := NewtestStore() + db.savedMetadata["2019/shooting1/abc.jpg"] = ImageMetaData{ + Md5Sum: "2019/shooting1/abc.jpg", + RelativeImagePath: "2019/shooting1/abc.jpg", + UploadRequired: false, + LastChange: time.Date(2019, 01, 01, 00, 0, 0, 0, time.UTC), + Filename: "abc.jpg", + } + + testFileSystemNode := &localFileStructure.FilesystemNode{ + Key: "2019/shooting1/abc.jpg", + ModTime: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC), + Name: "abc.jpg", + Path: "2019/shooting1/abc.jpg", + IsDir: false} + + fileSystemNodes := map[string]*localFileStructure.FilesystemNode{} + fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode + + // execute the sync metadata based on the file system results + err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator) + if err != nil { + t.Error(err) + } + + // check if data are saved + savedData, exist := db.savedMetadata[testFileSystemNode.Key] + if !exist { + t.Fatal("Could not find correct metadata!") + } + if savedData.LastChange != testFileSystemNode.ModTime { + t.Error("lastChange on db image metadata is not set to the right date!") + } + if savedData.UploadRequired != true { + t.Errorf("uploadRequired on db image metadata is not set to true!") + } +} + +func TestSynchronizeLocalImageMetadataShouldNotMarkUnchangedFilesToUpload(t *testing.T) { + db := NewtestStore() + db.savedMetadata["2019/shooting1/abc.jpg"] = ImageMetaData{ + Md5Sum: "2019/shooting1/abc.jpg", + RelativeImagePath: "2019/shooting1/abc.jpg", + UploadRequired: false, + LastChange: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC), + Filename: "abc.jpg", + } + + testFileSystemNode := &localFileStructure.FilesystemNode{ + Key: "2019/shooting1/abc.jpg", + ModTime: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC), + Name: "abc.jpg", + Path: "2019/shooting1/abc.jpg", + IsDir: false} + + fileSystemNodes := map[string]*localFileStructure.FilesystemNode{} + fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode + + // execute the sync metadata based on the file system results + err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator) + if err != nil { + t.Error(err) + } + + // check if data are saved + savedData, exist := db.savedMetadata[testFileSystemNode.Key] + if !exist { + t.Fatal("Could not find correct metadata!") + } + if savedData.UploadRequired { + t.Errorf("uploadRequired on db image metadata is set to true, but should not be on unchanged items!") + } +} + +func TestSynchronizeLocalImageMetadataShouldNotProcessDirectories(t *testing.T) { + db := NewtestStore() + + testFileSystemNode := &localFileStructure.FilesystemNode{ + Key: "2019/shooting1", + ModTime: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC), + Name: "shooting1", + Path: "2019/shooting1/", + IsDir: true} + + fileSystemNodes := map[string]*localFileStructure.FilesystemNode{} + fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode + + // execute the sync metadata based on the file system results + err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator) + if err != nil { + t.Error(err) + } + + if len(db.savedMetadata) > 0 { + t.Error("There were metadata records saved but non expected!") + } +} + +// test metadata store to store save the metadat and simulate the database +type testStore struct { + savedMetadata map[string]ImageMetaData +} + +func NewtestStore() *testStore { + return &testStore{savedMetadata: make(map[string]ImageMetaData)} +} + +func (s *testStore) GetImageMetadata(relativePath string) (ImageMetaData, error) { + metadata, exist := s.savedMetadata[relativePath] + if !exist { + return ImageMetaData{}, ErrorRecordNotFound + } + return metadata, nil +} + +func (s *testStore) SaveImageMetadata(m ImageMetaData) error { + s.savedMetadata[m.RelativeImagePath] = m + return nil +} + +// to make the sync testable, we pass in a simple mock that returns the filepath as checksum +func testChecksumCalculator(file string) (string, error) { + return file, nil +}