made finding missing files work

internal/app/category.go

@@ -30,7 +30,7 @@ func synchronizeCategories(context *appContext, filesystemNodes map[string]*loca
 }
 
 func findMissingCategories(fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*category.PiwigoCategory) []string {
-	missingCategories := []string{}
+	missingCategories := make([]string, 0, len(fileSystem))
 
 	for _, file := range fileSystem {
 		if !file.IsDir {

internal/app/images.go

@@ -5,6 +5,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/localFileStructure"
 	"haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/category"
+	"haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/picture"
 )
 
 func synchronizeImages(context *appContext, fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*category.PiwigoCategory) error {
@@ -14,19 +15,44 @@ func synchronizeImages(context *appContext, fileSystem map[string]*localFileStru
 		return err
 	}
 
-	missingFiles := findMissingImages(imageFiles)
+	missingFiles, err := findMissingImages(context, imageFiles)
+	if err != nil {
+		return err
+	}
+
 	uploadImages(missingFiles)
 
 	return errors.New("synchronizeImages: NOT IMPLEMENTED")
 }
 
-func findMissingImages(imageFiles []*localFileStructure.ImageNode) []string {
+func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) {
 
-	logrus.Warnln("Finding missing images (NotImplemented)")
+	logrus.Debugln("Preparing lookuplist for missing files...")
 
-	return nil
+	files := make([]string, 0, len(imageFiles))
+	md5map := make(map[string]*localFileStructure.ImageNode, len(imageFiles))
+	for _, file := range imageFiles  {
+		md5map[file.Md5Sum] = file
+		files = append(files, file.Md5Sum)
+	}
+
+	misingSums, err := picture.ImageUploadRequired(context.Piwigo, files)
+	if err != nil {
+		return nil, err
+	}
+
+	missingFiles := make([]*localFileStructure.ImageNode, 0, len(misingSums))
+	for _, sum := range misingSums  {
+		file := md5map[sum]
+		logrus.Infof("Found missing file %s", file.Path)
+		missingFiles = append(missingFiles, file)
+	}
+
+	logrus.Infof("Found %d missing files", len(missingFiles))
+
+	return missingFiles, nil
 }
 
-func uploadImages(missingFiles []string) {
+func uploadImages(missingFiles []*localFileStructure.ImageNode) {
 	logrus.Warnln("Uploading missing images (NotImplemented)")
 }

internal/pkg/localFileStructure/checksumCalculator.go

@@ -9,20 +9,20 @@ import (
 )
 
 func calculateFileCheckSums(filePath string) (string, error) {
-	f, err := os.Open(filePath)
+	file, err := os.Open(filePath)
 	if err != nil {
 		logrus.Errorf("Could not open file %s", filePath)
 		return "", err
 	}
-	defer f.Close()
+	defer file.Close()
 
-	h := md5.New()
+	hash := md5.New()
-	if _, err := io.Copy(h, f); err != nil {
+	if _, err := io.Copy(hash, file); err != nil {
 		logrus.Errorf("Could calculate md5 sum of file %s", filePath)
 		return "", err
 	}
 
-	md5sum := fmt.Sprintf("%x", md5.Sum(nil))
+	md5sum := fmt.Sprintf("%x", hash.Sum(nil))
 
 	logrus.Tracef("Calculated md5 sum of %s - %s", filePath, md5sum)
 

internal/pkg/localFileStructure/imageList.go

@@ -2,11 +2,12 @@ package localFileStructure
 
 import (
 	"github.com/sirupsen/logrus"
+	"path/filepath"
 	"time"
 )
 
 func GetImageList(fileSystem map[string]*FilesystemNode) ([]*ImageNode, error) {
-	imageFiles := []*ImageNode{}
+	imageFiles := make([]*ImageNode, 0, len(fileSystem))
 
 	for _, file := range fileSystem {
 		if file.IsDir {
@@ -18,14 +19,17 @@ func GetImageList(fileSystem map[string]*FilesystemNode) ([]*ImageNode, error) {
 			return nil, err
 		}
 
-		logrus.Debugf("Image %s - %s - %s", md5sum, file.ModTime.Format(time.RFC3339), file.Path)
+		logrus.Debugf("Local Image %s - %s - %s", md5sum, file.ModTime.Format(time.RFC3339), file.Path)
 
 		imageFiles = append(imageFiles, &ImageNode{
-			Path:    file.Path,
+			Path:      file.Path,
-			ModTime: file.ModTime,
+			Directory: filepath.Dir(file.Path),
-			Md5Sum:  md5sum,
+			ModTime:   file.ModTime,
+			Md5Sum:    md5sum,
 		})
 	}
 
+	logrus.Infof("Found %d local images to process", len(imageFiles))
+
 	return imageFiles, nil
 }

internal/pkg/localFileStructure/types.go

@@ -18,7 +18,8 @@ func (n *FilesystemNode) String() string {
 }
 
 type ImageNode struct {
-	Path    string
+	Path      string
-	ModTime time.Time
+	Directory string
-	Md5Sum  string
+	ModTime   time.Time
+	Md5Sum    string
 }

internal/pkg/piwigo/picture/query.go

@@ -1,22 +1,53 @@
 package picture
 
 import (
+	"encoding/json"
+	"errors"
 	"github.com/sirupsen/logrus"
 	"haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
+	"net/http"
+	"net/url"
+	"strings"
 )
 
-func ImageUploadRequired(context *piwigo.PiwigoContext, files []string) (bool, error) {
+func ImageUploadRequired(context *piwigo.PiwigoContext, md5sums []string) ([]string, error) {
 
-	for file := range files {
+	if context.Cookies == nil {
-		logrus.Debug(file)
+		return nil, errors.New("Not logged in and no cookies found! Can not get the category list!")
 	}
 
-	/*
+	//TODO: make sure to split to multiple queries -> to honor max upload queries
-			http://pictures.haefelfinger.net/ws.php?format=json
-		{
-		    "md5sum_list": "d327416a83452b91764ed2888a5630a3,6d5f122e2b98bc1a192850e89fc2ae8c,40bfe8dd8349ccdedd4a939f9191cafa"
-		}
-	*/
 
-	return false, nil
+	md5sumList := strings.Join(md5sums, ",")
+
+	formData := url.Values{}
+	formData.Set("method", "pwg.images.exist")
+	formData.Set("md5sum_list", md5sumList)
+
+	logrus.Tracef("Looking up missing files: %s", md5sumList)
+
+	client := http.Client{Jar: context.Cookies}
+	response, err := client.PostForm(context.Url, formData)
+	if err != nil {
+		logrus.Errorln("The HTTP request failed with error %s", err)
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	var imageExistResponse imageExistResponse
+	if err := json.NewDecoder(response.Body).Decode(&imageExistResponse); err != nil {
+		logrus.Errorln(err)
+		return nil, err
+	}
+
+	missingFiles := make([]string, 0, len(imageExistResponse.Result))
+
+	for key, value := range imageExistResponse.Result {
+		if value == "" {
+			logrus.Tracef("Missing file with md5sum: %s", key)
+			missingFiles = append(missingFiles, key)
+		}
+	}
+
+	return missingFiles, nil
 }