From a4a6cce943c204eece90d4fa77756d1e65fe164f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20H=C3=A4felfinger?= Date: Wed, 27 Feb 2019 23:26:18 +0100 Subject: [PATCH] made finding missing files work --- internal/app/category.go | 2 +- internal/app/images.go | 36 +++++++++++-- .../localFileStructure/checksumCalculator.go | 10 ++-- internal/pkg/localFileStructure/imageList.go | 14 +++-- internal/pkg/localFileStructure/types.go | 7 +-- internal/pkg/piwigo/picture/query.go | 51 +++++++++++++++---- 6 files changed, 91 insertions(+), 29 deletions(-) diff --git a/internal/app/category.go b/internal/app/category.go index 2a9765f..1e6be7c 100644 --- a/internal/app/category.go +++ b/internal/app/category.go @@ -30,7 +30,7 @@ func synchronizeCategories(context *appContext, filesystemNodes map[string]*loca } func findMissingCategories(fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*category.PiwigoCategory) []string { - missingCategories := []string{} + missingCategories := make([]string, 0, len(fileSystem)) for _, file := range fileSystem { if !file.IsDir { diff --git a/internal/app/images.go b/internal/app/images.go index 75d3c5b..4a39343 100644 --- a/internal/app/images.go +++ b/internal/app/images.go @@ -5,6 +5,7 @@ import ( "github.com/sirupsen/logrus" "haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/localFileStructure" "haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/category" + "haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/picture" ) func synchronizeImages(context *appContext, fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*category.PiwigoCategory) error { @@ -14,19 +15,44 @@ func synchronizeImages(context *appContext, fileSystem map[string]*localFileStru return err } - missingFiles := findMissingImages(imageFiles) + missingFiles, err := findMissingImages(context, imageFiles) + if err != nil { + return err + } + uploadImages(missingFiles) return errors.New("synchronizeImages: NOT IMPLEMENTED") } -func findMissingImages(imageFiles []*localFileStructure.ImageNode) []string { +func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) { - logrus.Warnln("Finding missing images (NotImplemented)") + logrus.Debugln("Preparing lookuplist for missing files...") - return nil + files := make([]string, 0, len(imageFiles)) + md5map := make(map[string]*localFileStructure.ImageNode, len(imageFiles)) + for _, file := range imageFiles { + md5map[file.Md5Sum] = file + files = append(files, file.Md5Sum) + } + + misingSums, err := picture.ImageUploadRequired(context.Piwigo, files) + if err != nil { + return nil, err + } + + missingFiles := make([]*localFileStructure.ImageNode, 0, len(misingSums)) + for _, sum := range misingSums { + file := md5map[sum] + logrus.Infof("Found missing file %s", file.Path) + missingFiles = append(missingFiles, file) + } + + logrus.Infof("Found %d missing files", len(missingFiles)) + + return missingFiles, nil } -func uploadImages(missingFiles []string) { +func uploadImages(missingFiles []*localFileStructure.ImageNode) { logrus.Warnln("Uploading missing images (NotImplemented)") } diff --git a/internal/pkg/localFileStructure/checksumCalculator.go b/internal/pkg/localFileStructure/checksumCalculator.go index c271470..1b96dd2 100644 --- a/internal/pkg/localFileStructure/checksumCalculator.go +++ b/internal/pkg/localFileStructure/checksumCalculator.go @@ -9,20 +9,20 @@ import ( ) func calculateFileCheckSums(filePath string) (string, error) { - f, err := os.Open(filePath) + file, err := os.Open(filePath) if err != nil { logrus.Errorf("Could not open file %s", filePath) return "", err } - defer f.Close() + defer file.Close() - h := md5.New() - if _, err := io.Copy(h, f); err != nil { + hash := md5.New() + if _, err := io.Copy(hash, file); err != nil { logrus.Errorf("Could calculate md5 sum of file %s", filePath) return "", err } - md5sum := fmt.Sprintf("%x", md5.Sum(nil)) + md5sum := fmt.Sprintf("%x", hash.Sum(nil)) logrus.Tracef("Calculated md5 sum of %s - %s", filePath, md5sum) diff --git a/internal/pkg/localFileStructure/imageList.go b/internal/pkg/localFileStructure/imageList.go index 9d9eb7c..34c7eb3 100644 --- a/internal/pkg/localFileStructure/imageList.go +++ b/internal/pkg/localFileStructure/imageList.go @@ -2,11 +2,12 @@ package localFileStructure import ( "github.com/sirupsen/logrus" + "path/filepath" "time" ) func GetImageList(fileSystem map[string]*FilesystemNode) ([]*ImageNode, error) { - imageFiles := []*ImageNode{} + imageFiles := make([]*ImageNode, 0, len(fileSystem)) for _, file := range fileSystem { if file.IsDir { @@ -18,14 +19,17 @@ func GetImageList(fileSystem map[string]*FilesystemNode) ([]*ImageNode, error) { return nil, err } - logrus.Debugf("Image %s - %s - %s", md5sum, file.ModTime.Format(time.RFC3339), file.Path) + logrus.Debugf("Local Image %s - %s - %s", md5sum, file.ModTime.Format(time.RFC3339), file.Path) imageFiles = append(imageFiles, &ImageNode{ - Path: file.Path, - ModTime: file.ModTime, - Md5Sum: md5sum, + Path: file.Path, + Directory: filepath.Dir(file.Path), + ModTime: file.ModTime, + Md5Sum: md5sum, }) } + logrus.Infof("Found %d local images to process", len(imageFiles)) + return imageFiles, nil } diff --git a/internal/pkg/localFileStructure/types.go b/internal/pkg/localFileStructure/types.go index abcfae4..c422739 100644 --- a/internal/pkg/localFileStructure/types.go +++ b/internal/pkg/localFileStructure/types.go @@ -18,7 +18,8 @@ func (n *FilesystemNode) String() string { } type ImageNode struct { - Path string - ModTime time.Time - Md5Sum string + Path string + Directory string + ModTime time.Time + Md5Sum string } diff --git a/internal/pkg/piwigo/picture/query.go b/internal/pkg/piwigo/picture/query.go index 7346bb2..81706a5 100644 --- a/internal/pkg/piwigo/picture/query.go +++ b/internal/pkg/piwigo/picture/query.go @@ -1,22 +1,53 @@ package picture import ( + "encoding/json" + "errors" "github.com/sirupsen/logrus" "haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo" + "net/http" + "net/url" + "strings" ) -func ImageUploadRequired(context *piwigo.PiwigoContext, files []string) (bool, error) { +func ImageUploadRequired(context *piwigo.PiwigoContext, md5sums []string) ([]string, error) { - for file := range files { - logrus.Debug(file) + if context.Cookies == nil { + return nil, errors.New("Not logged in and no cookies found! Can not get the category list!") } - /* - http://pictures.haefelfinger.net/ws.php?format=json - { - "md5sum_list": "d327416a83452b91764ed2888a5630a3,6d5f122e2b98bc1a192850e89fc2ae8c,40bfe8dd8349ccdedd4a939f9191cafa" - } - */ + //TODO: make sure to split to multiple queries -> to honor max upload queries - return false, nil + md5sumList := strings.Join(md5sums, ",") + + formData := url.Values{} + formData.Set("method", "pwg.images.exist") + formData.Set("md5sum_list", md5sumList) + + logrus.Tracef("Looking up missing files: %s", md5sumList) + + client := http.Client{Jar: context.Cookies} + response, err := client.PostForm(context.Url, formData) + if err != nil { + logrus.Errorln("The HTTP request failed with error %s", err) + return nil, err + } + defer response.Body.Close() + + var imageExistResponse imageExistResponse + if err := json.NewDecoder(response.Body).Decode(&imageExistResponse); err != nil { + logrus.Errorln(err) + return nil, err + } + + missingFiles := make([]string, 0, len(imageExistResponse.Result)) + + for key, value := range imageExistResponse.Result { + if value == "" { + logrus.Tracef("Missing file with md5sum: %s", key) + missingFiles = append(missingFiles, key) + } + } + + return missingFiles, nil }