made upload of images work

cmd/DirectoriesToAlbums/main.go

@@ -1,9 +1,9 @@
 package main
 
 import (
+	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/app"
 	"github.com/sirupsen/logrus"
 	"github.com/vharitonsky/iniflags"
-	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/app"
 	"os"
 )
 

internal/app/app.go

@@ -4,18 +4,19 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/localFileStructure"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/authentication"
+	"github.com/sirupsen/logrus"
 	"os"
 )
 
 var (
-	imagesRootPath = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.")
+	imagesRootPath            = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.")
-	piwigoUrl      = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.")
+	piwigoUrl                 = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.")
-	piwigoUser     = flag.String("piwigoUser", "", "The username to use during sync.")
+	piwigoUser                = flag.String("piwigoUser", "", "The username to use during sync.")
-	piwigoPassword = flag.String("piwigoPassword", "", "This is password to the given username.")
+	piwigoPassword            = flag.String("piwigoPassword", "", "This is password to the given username.")
+	piwigoUploadChunkSizeInKB = flag.Int("piwigoUploadChunkSizeInKB", 512, "The chunksize used to upload an image to piwigo.")
 )
 
 func Run() {
@@ -73,6 +74,7 @@ func configureContext() (*appContext, error) {
 	context.Piwigo.Url = fmt.Sprintf("%s/ws.php?format=json", *piwigoUrl)
 	context.Piwigo.Username = *piwigoUser
 	context.Piwigo.Password = *piwigoPassword
+	context.Piwigo.ChunkSizeInKB = *piwigoUploadChunkSizeInKB
 
 	return context, nil
 }

internal/app/category.go

@@ -3,9 +3,9 @@ package app
 import (
 	"errors"
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/localFileStructure"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/category"
+	"github.com/sirupsen/logrus"
 	"path/filepath"
 	"sort"
 )

internal/app/images.go

@@ -1,11 +1,10 @@
 package app
 
 import (
-	"errors"
-	"github.com/sirupsen/logrus"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/localFileStructure"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/category"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/picture"
+	"github.com/sirupsen/logrus"
 )
 
 func synchronizeImages(context *appContext, fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*category.PiwigoCategory) error {
@@ -25,7 +24,9 @@ func synchronizeImages(context *appContext, fileSystem map[string]*localFileStru
 		return err
 	}
 
-	return errors.New("synchronizeImages: NOT IMPLEMENTED")
+	logrus.Infof("Synchronized %d files.", len(missingFiles))
+
+	return nil
 }
 
 func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) {
@@ -57,17 +58,15 @@ func findMissingImages(context *appContext, imageFiles []*localFileStructure.Ima
 }
 
 func uploadImages(context *appContext, missingFiles []*localFileStructure.ImageNode, existingCategories map[string]*category.PiwigoCategory) error {
-	logrus.Warnln("Uploading missing images (NotImplemented)")
-
 	for _, file := range missingFiles {
 		logrus.Infof("Uploading %s %s", file.CategoryName, file.Path)
 		categoryId := existingCategories[file.CategoryName].Id
 
-		//TODO handle added id
+		imageId, err := picture.UploadImage(context.Piwigo, file.Path, file.Md5Sum, categoryId)
-		_, err := picture.UploadImage(context.Piwigo, file.Path, file.Md5Sum, categoryId)
 		if err != nil {
 			return err
 		}
+		file.ImageId = imageId
 	}
 
 	return nil

internal/pkg/localFileStructure/types.go

@@ -22,4 +22,5 @@ type ImageNode struct {
 	CategoryName string
 	ModTime      time.Time
 	Md5Sum       string
+	ImageId      int
 }

internal/pkg/piwigo/PiwigoContext.go

@@ -8,10 +8,11 @@ import (
 )
 
 type PiwigoContext struct {
-	Url      string
+	Url           string
-	Username string
+	Username      string
-	Password string
+	Password      string
-	Cookies  *cookiejar.Jar
+	ChunkSizeInKB int
+	Cookies       *cookiejar.Jar
 }
 
 func (context *PiwigoContext) PostForm(formData url.Values) (resp *http.Response, err error) {

internal/pkg/piwigo/authentication/authentication.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
+	"github.com/sirupsen/logrus"
 	"net/url"
 	"strings"
 )

internal/pkg/piwigo/category/create.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
+	"github.com/sirupsen/logrus"
 	"net/url"
 )
 

internal/pkg/piwigo/category/query.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
+	"github.com/sirupsen/logrus"
 	"net/url"
 	"os"
 )

internal/pkg/piwigo/picture/query.go

@@ -2,14 +2,15 @@ package picture
 
 import (
 	"encoding/json"
-	"github.com/sirupsen/logrus"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
+	"github.com/sirupsen/logrus"
 	"net/url"
 	"strings"
 )
 
 func ImageUploadRequired(context *piwigo.PiwigoContext, md5sums []string) ([]string, error) {
 	//TODO: make sure to split to multiple queries -> to honor max upload queries
+	//TODO: Make sure to return the found imageIds of the found sums to update the local image nodes
 
 	md5sumList := strings.Join(md5sums, ",")
 

internal/pkg/piwigo/picture/types.go

@@ -7,7 +7,10 @@ type uploadChunkResponse struct {
 
 type fileAddResponse struct {
 	Status string `json:"stat"`
-	Result string `json:"result"`
+	Result struct {
+		ImageID int    `json:"image_id"`
+		URL     string `json:"url"`
+	} `json:"result"`
 }
 
 type imageExistResponse struct {

internal/pkg/piwigo/picture/upload.go

@@ -1,33 +1,89 @@
 package picture
 
 import (
+	"bufio"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
+	"github.com/sirupsen/logrus"
+	"io"
 	"net/url"
+	"os"
 	"strconv"
 )
 
 func UploadImage(context *piwigo.PiwigoContext, filePath string, md5sum string, category int) (int, error) {
-	logrus.Infof("Uploading %s", filePath)
+	if context.ChunkSizeInKB <= 0 {
+		return 0, errors.New("Uploadchunk size is less or equal to zero. 512 is a recommendet value to begin with.")
+	}
 
-	// split into chunks
+	fileInfo, err := os.Stat(filePath)
-	// upload chunks
+	if err != nil {
-	// finalize upload
+		return 0, err
+	}
 
-	return 0, nil
+	fileSizeInKB := fileInfo.Size() / 1024
+	logrus.Infof("Uploading %s using chunksize of %d KB and total size of %d", filePath, context.ChunkSizeInKB, fileSizeInKB)
+
+	err = uploadImageChunks(filePath, context, fileSizeInKB, md5sum)
+	if err != nil {
+		return 0, err
+	}
+
+	imageId, err := uploadImageFinal(context, fileInfo.Name(), md5sum, category)
+	if err != nil {
+		return 0, err
+	}
+
+	return imageId, nil
 }
 
-func uploadImageChunk(context *piwigo.PiwigoContext, base64chunk string, md5sum string, position int) error {
+func uploadImageChunks(filePath string, context *piwigo.PiwigoContext, fileSizeInKB int64, md5sum string) error {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	reader := bufio.NewReader(file)
+	buffer := make([]byte, context.ChunkSizeInKB*1024)
+	numberOfChunks := (fileSizeInKB / int64(context.ChunkSizeInKB)) + 1
+	currentChunk := int64(0)
+
+	for {
+		logrus.Tracef("Processing chunk %d of %d of %s", currentChunk, numberOfChunks, filePath)
+
+		readBytes, readError := reader.Read(buffer)
+		if readError == io.EOF && readBytes == 0 {
+			break
+		}
+		if readError != io.EOF && readError != nil {
+			return readError
+		}
+
+		encodedChunk := base64.StdEncoding.EncodeToString(buffer[:readBytes])
+
+		uploadError := uploadImageChunk(context, encodedChunk, md5sum, currentChunk)
+		if uploadError != nil {
+			return uploadError
+		}
+
+		currentChunk++
+	}
+
+	return nil
+}
+
+func uploadImageChunk(context *piwigo.PiwigoContext, base64chunk string, md5sum string, position int64) error {
 	formData := url.Values{}
 	formData.Set("method", "pwg.images.addChunk")
 	formData.Set("data", base64chunk)
 	formData.Set("original_sum", md5sum)
 	// required by the API for compatibility
 	formData.Set("type", "file")
-	formData.Set("position", strconv.Itoa(position))
+	formData.Set("position", strconv.FormatInt(position, 10))
 
 	logrus.Tracef("Uploading chunk %d of file with sum %s", position, md5sum)
 
@@ -51,31 +107,32 @@ func uploadImageChunk(context *piwigo.PiwigoContext, base64chunk string, md5sum
 	return nil
 }
 
-func uploadImageFinal(context *piwigo.PiwigoContext, originalFilename string, md5sum string, categoryId int) error {
+func uploadImageFinal(context *piwigo.PiwigoContext, originalFilename string, md5sum string, categoryId int) (int, error) {
 	formData := url.Values{}
 	formData.Set("method", "pwg.images.add")
 	formData.Set("original_sum", md5sum)
 	formData.Set("original_filename", originalFilename)
-	formData.Set("categoriesi", strconv.Itoa(categoryId))
+	formData.Set("name", originalFilename)
+	formData.Set("categories", strconv.Itoa(categoryId))
 
 	logrus.Debugf("Finalizing upload of file %s with sum %s to category %d", originalFilename, md5sum, categoryId)
 
 	response, err := context.PostForm(formData)
 	if err != nil {
-		return err
+		return 0, err
 	}
 	defer response.Body.Close()
 
 	var fileAddResponse fileAddResponse
 	if err := json.NewDecoder(response.Body).Decode(&fileAddResponse); err != nil {
 		logrus.Errorln(err)
-		return err
+		return 0, err
 	}
 
 	if fileAddResponse.Status != "ok" {
 		logrus.Errorf("Got state %s while adding image %s", fileAddResponse.Status, originalFilename)
-		return errors.New(fmt.Sprintf("Got state %s while adding image %s", fileAddResponse.Status, originalFilename))
+		return 0, errors.New(fmt.Sprintf("Got state %s while adding image %s", fileAddResponse.Status, originalFilename))
 	}
 
-	return nil
+	return fileAddResponse.Result.ImageID, nil
 }