From 2808c85147e704ef54ea6905a9fb57dabaec9b53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20H=C3=A4felfinger?= Date: Wed, 27 Feb 2019 22:11:47 +0100 Subject: [PATCH] added md5 sums to the local file upload indexing --- internal/app/app.go | 2 +- internal/app/images.go | 19 +++--------- .../localFileStructure/checksumCalculator.go | 30 ++++++++++++++++++ .../localFileStructure/filesystemScanner.go | 16 +++++----- internal/pkg/localFileStructure/imageList.go | 31 +++++++++++++++++++ internal/pkg/localFileStructure/types.go | 21 ++++++++++--- internal/pkg/piwigo/picture/query.go | 10 +++--- internal/pkg/piwigo/picture/upload.go | 3 +- 8 files changed, 98 insertions(+), 34 deletions(-) create mode 100644 internal/pkg/localFileStructure/checksumCalculator.go create mode 100644 internal/pkg/localFileStructure/imageList.go diff --git a/internal/app/app.go b/internal/app/app.go index 07cc54a..f74af7e 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -44,7 +44,7 @@ func Run() { logErrorAndExit(err, 5) } - err = synchronizeImages(filesystemNodes, categories) + err = synchronizeImages(context, filesystemNodes, categories) if err != nil { logErrorAndExit(err, 6) } diff --git a/internal/app/images.go b/internal/app/images.go index 8a79cf9..c528e0e 100644 --- a/internal/app/images.go +++ b/internal/app/images.go @@ -9,7 +9,10 @@ import ( func synchronizeImages(context *AppContext, fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*category.PiwigoCategory) error { - imageFiles := getImageList(fileSystem) + imageFiles, err := localFileStructure.GetImageList(fileSystem) + if err != nil { + return err + } missingFiles := findMissingImages(imageFiles) uploadImages(missingFiles) @@ -17,7 +20,7 @@ func synchronizeImages(context *AppContext, fileSystem map[string]*localFileStru return errors.New("synchronizeImages: NOT IMPLEMENTED") } -func findMissingImages(imageFiles []string) []string { +func findMissingImages(imageFiles []*localFileStructure.ImageNode) []string { logrus.Warnln("Finding missing images (NotImplemented)") @@ -27,15 +30,3 @@ func findMissingImages(imageFiles []string) []string { func uploadImages(missingFiles []string) { logrus.Warnln("Uploading missing images (NotImplemented)") } - -func getImageList(fileSystem map[string]*localFileStructure.FilesystemNode) []string { - imageFiles := []string{} - - for _, file := range fileSystem { - if !file.IsDir { - imageFiles = append(imageFiles, file.Key) - } - } - - return imageFiles -} diff --git a/internal/pkg/localFileStructure/checksumCalculator.go b/internal/pkg/localFileStructure/checksumCalculator.go new file mode 100644 index 0000000..c271470 --- /dev/null +++ b/internal/pkg/localFileStructure/checksumCalculator.go @@ -0,0 +1,30 @@ +package localFileStructure + +import ( + "crypto/md5" + "fmt" + "github.com/sirupsen/logrus" + "io" + "os" +) + +func calculateFileCheckSums(filePath string) (string, error) { + f, err := os.Open(filePath) + if err != nil { + logrus.Errorf("Could not open file %s", filePath) + return "", err + } + defer f.Close() + + h := md5.New() + if _, err := io.Copy(h, f); err != nil { + logrus.Errorf("Could calculate md5 sum of file %s", filePath) + return "", err + } + + md5sum := fmt.Sprintf("%x", md5.Sum(nil)) + + logrus.Tracef("Calculated md5 sum of %s - %s", filePath, md5sum) + + return md5sum, nil +} diff --git a/internal/pkg/localFileStructure/filesystemScanner.go b/internal/pkg/localFileStructure/filesystemScanner.go index da6f22a..d623059 100644 --- a/internal/pkg/localFileStructure/filesystemScanner.go +++ b/internal/pkg/localFileStructure/filesystemScanner.go @@ -21,19 +21,21 @@ func ScanLocalFileStructure(path string) (map[string]*FilesystemNode, error) { numberOfDirectories := 0 numberOfImages := 0 - err = filepath.Walk(fullPathRoot, func(p string, info os.FileInfo, err error) error { - if fullPathRoot == p { + err = filepath.Walk(fullPathRoot, func(path string, info os.FileInfo, err error) error { + if fullPathRoot == path { return nil } //TODO: Only allow jpg and png files here - key := strings.Replace(p, fullPathReplace, "", 1) + key := strings.Replace(path, fullPathReplace, "", 1) - fileMap[p] = &FilesystemNode{ - Key: key, - Name: info.Name(), - IsDir: info.IsDir(), + fileMap[path] = &FilesystemNode{ + Key: key, + Path: path, + Name: info.Name(), + IsDir: info.IsDir(), + ModTime: info.ModTime(), } if info.IsDir() { diff --git a/internal/pkg/localFileStructure/imageList.go b/internal/pkg/localFileStructure/imageList.go new file mode 100644 index 0000000..9d9eb7c --- /dev/null +++ b/internal/pkg/localFileStructure/imageList.go @@ -0,0 +1,31 @@ +package localFileStructure + +import ( + "github.com/sirupsen/logrus" + "time" +) + +func GetImageList(fileSystem map[string]*FilesystemNode) ([]*ImageNode, error) { + imageFiles := []*ImageNode{} + + for _, file := range fileSystem { + if file.IsDir { + continue + } + + md5sum, err := calculateFileCheckSums(file.Path) + if err != nil { + return nil, err + } + + logrus.Debugf("Image %s - %s - %s", md5sum, file.ModTime.Format(time.RFC3339), file.Path) + + imageFiles = append(imageFiles, &ImageNode{ + Path: file.Path, + ModTime: file.ModTime, + Md5Sum: md5sum, + }) + } + + return imageFiles, nil +} diff --git a/internal/pkg/localFileStructure/types.go b/internal/pkg/localFileStructure/types.go index 8f6b1fa..abcfae4 100644 --- a/internal/pkg/localFileStructure/types.go +++ b/internal/pkg/localFileStructure/types.go @@ -1,13 +1,24 @@ package localFileStructure -import "fmt" +import ( + "fmt" + "time" +) type FilesystemNode struct { - Key string - Name string - IsDir bool + Key string + Path string + Name string + IsDir bool + ModTime time.Time } func (n *FilesystemNode) String() string { - return fmt.Sprintf("FilesystemNode: %s", n.Key) + return fmt.Sprintf("FilesystemNode: %s", n.Path) +} + +type ImageNode struct { + Path string + ModTime time.Time + Md5Sum string } diff --git a/internal/pkg/piwigo/picture/query.go b/internal/pkg/piwigo/picture/query.go index a07290e..7346bb2 100644 --- a/internal/pkg/piwigo/picture/query.go +++ b/internal/pkg/piwigo/picture/query.go @@ -12,11 +12,11 @@ func ImageUploadRequired(context *piwigo.PiwigoContext, files []string) (bool, e } /* - http://pictures.haefelfinger.net/ws.php?format=json -{ - "md5sum_list": "d327416a83452b91764ed2888a5630a3,6d5f122e2b98bc1a192850e89fc2ae8c,40bfe8dd8349ccdedd4a939f9191cafa" -} - */ + http://pictures.haefelfinger.net/ws.php?format=json + { + "md5sum_list": "d327416a83452b91764ed2888a5630a3,6d5f122e2b98bc1a192850e89fc2ae8c,40bfe8dd8349ccdedd4a939f9191cafa" + } + */ return false, nil } diff --git a/internal/pkg/piwigo/picture/upload.go b/internal/pkg/piwigo/picture/upload.go index 26eaa57..7a0ab99 100644 --- a/internal/pkg/piwigo/picture/upload.go +++ b/internal/pkg/piwigo/picture/upload.go @@ -1,6 +1,5 @@ package picture - /* http://pictures.haefelfinger.net/ws.php?format=json { @@ -8,4 +7,4 @@ http://pictures.haefelfinger.net/ws.php?format=json "original_sum": "1234567", "position": "1" } - */ +*/