piwigo/PiwigoDirectoryUploader
2
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

made upload of images work

Browse Source
This commit is contained in:
Philipp Häfelfinger 2019-03-02 00:21:01 +01:00
parent 515a3c6950
commit bfd90cf1b6
12 changed files with 101 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/DirectoriesToAlbums/main.go
View File

@ -1,9 +1,9 @@
package main
import (
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/app"
"github.com/sirupsen/logrus"
"github.com/vharitonsky/iniflags"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/app"
"os"
)

12
internal/app/app.go
View File

@ -4,18 +4,19 @@ import (
"errors"
"flag"
"fmt"
"github.com/sirupsen/logrus"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/localFileStructure"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/authentication"
"github.com/sirupsen/logrus"
"os"
)
var (
imagesRootPath = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.")
piwigoUrl = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.")
piwigoUser = flag.String("piwigoUser", "", "The username to use during sync.")
piwigoPassword = flag.String("piwigoPassword", "", "This is password to the given username.")
imagesRootPath = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.")
piwigoUrl = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.")
piwigoUser = flag.String("piwigoUser", "", "The username to use during sync.")
piwigoPassword = flag.String("piwigoPassword", "", "This is password to the given username.")
piwigoUploadChunkSizeInKB = flag.Int("piwigoUploadChunkSizeInKB", 512, "The chunksize used to upload an image to piwigo.")
)
func Run() {
@ -73,6 +74,7 @@ func configureContext() (*appContext, error) {
context.Piwigo.Url = fmt.Sprintf("%s/ws.php?format=json", *piwigoUrl)
context.Piwigo.Username = *piwigoUser
context.Piwigo.Password = *piwigoPassword
context.Piwigo.ChunkSizeInKB = *piwigoUploadChunkSizeInKB
return context, nil
}

2
internal/app/category.go
View File

@ -3,9 +3,9 @@ package app
import (
"errors"
"fmt"
"github.com/sirupsen/logrus"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/localFileStructure"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/category"
"github.com/sirupsen/logrus"
"path/filepath"
"sort"
)

13
internal/app/images.go
View File

@ -1,11 +1,10 @@
package app
import (
"errors"
"github.com/sirupsen/logrus"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/localFileStructure"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/category"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/picture"
"github.com/sirupsen/logrus"
)
func synchronizeImages(context *appContext, fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*category.PiwigoCategory) error {
@ -25,7 +24,9 @@ func synchronizeImages(context *appContext, fileSystem map[string]*localFileStru
return err
}
return errors.New("synchronizeImages: NOT IMPLEMENTED")
logrus.Infof("Synchronized %d files.", len(missingFiles))
return nil
}
func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) {
@ -57,17 +58,15 @@ func findMissingImages(context *appContext, imageFiles []*localFileStructure.Ima
}
func uploadImages(context *appContext, missingFiles []*localFileStructure.ImageNode, existingCategories map[string]*category.PiwigoCategory) error {
logrus.Warnln("Uploading missing images (NotImplemented)")
for _, file := range missingFiles {
logrus.Infof("Uploading %s %s", file.CategoryName, file.Path)
categoryId := existingCategories[file.CategoryName].Id
//TODO handle added id
_, err := picture.UploadImage(context.Piwigo, file.Path, file.Md5Sum, categoryId)
imageId, err := picture.UploadImage(context.Piwigo, file.Path, file.Md5Sum, categoryId)
if err != nil {
return err
}
file.ImageId = imageId
}
return nil

1
internal/pkg/localFileStructure/types.go
View File

@ -22,4 +22,5 @@ type ImageNode struct {
CategoryName string
ModTime time.Time
Md5Sum string
ImageId int
}

9
internal/pkg/piwigo/PiwigoContext.go
View File

@ -8,10 +8,11 @@ import (
)
type PiwigoContext struct {
Url string
Username string
Password string
Cookies *cookiejar.Jar
Url string
Username string
Password string
ChunkSizeInKB int
Cookies *cookiejar.Jar
}
func (context *PiwigoContext) PostForm(formData url.Values) (resp *http.Response, err error) {

2
internal/pkg/piwigo/authentication/authentication.go
View File

@ -4,8 +4,8 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/sirupsen/logrus"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
"github.com/sirupsen/logrus"
"net/url"
"strings"
)

2
internal/pkg/piwigo/category/create.go
View File

@ -4,8 +4,8 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/sirupsen/logrus"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
"github.com/sirupsen/logrus"
"net/url"
)

2
internal/pkg/piwigo/category/query.go
View File

@ -4,8 +4,8 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/sirupsen/logrus"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
"github.com/sirupsen/logrus"
"net/url"
"os"
)

3
internal/pkg/piwigo/picture/query.go
View File

@ -2,14 +2,15 @@ package picture
import (
"encoding/json"
"github.com/sirupsen/logrus"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
"github.com/sirupsen/logrus"
"net/url"
"strings"
)
func ImageUploadRequired(context *piwigo.PiwigoContext, md5sums []string) ([]string, error) {
//TODO: make sure to split to multiple queries -> to honor max upload queries
//TODO: Make sure to return the found imageIds of the found sums to update the local image nodes
md5sumList := strings.Join(md5sums, ",")

5
internal/pkg/piwigo/picture/types.go
View File

@ -7,7 +7,10 @@ type uploadChunkResponse struct {
type fileAddResponse struct {
Status string `json:"stat"`
Result string `json:"result"`
Result struct {
ImageID int `json:"image_id"`
URL string `json:"url"`
} `json:"result"`
}
type imageExistResponse struct {

85
internal/pkg/piwigo/picture/upload.go
View File

@ -1,33 +1,89 @@
package picture
import (
"bufio"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"github.com/sirupsen/logrus"
"git.haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
"github.com/sirupsen/logrus"
"io"
"net/url"
"os"
"strconv"
)
func UploadImage(context *piwigo.PiwigoContext, filePath string, md5sum string, category int) (int, error) {
logrus.Infof("Uploading %s", filePath)
if context.ChunkSizeInKB <= 0 {
return 0, errors.New("Uploadchunk size is less or equal to zero. 512 is a recommendet value to begin with.")
}
// split into chunks
// upload chunks
// finalize upload
fileInfo, err := os.Stat(filePath)
if err != nil {
return 0, err
}
return 0, nil
fileSizeInKB := fileInfo.Size() / 1024
logrus.Infof("Uploading %s using chunksize of %d KB and total size of %d", filePath, context.ChunkSizeInKB, fileSizeInKB)
err = uploadImageChunks(filePath, context, fileSizeInKB, md5sum)
if err != nil {
return 0, err
}
imageId, err := uploadImageFinal(context, fileInfo.Name(), md5sum, category)
if err != nil {
return 0, err
}
return imageId, nil
}
func uploadImageChunk(context *piwigo.PiwigoContext, base64chunk string, md5sum string, position int) error {
func uploadImageChunks(filePath string, context *piwigo.PiwigoContext, fileSizeInKB int64, md5sum string) error {
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
reader := bufio.NewReader(file)
buffer := make([]byte, context.ChunkSizeInKB*1024)
numberOfChunks := (fileSizeInKB / int64(context.ChunkSizeInKB)) + 1
currentChunk := int64(0)
for {
logrus.Tracef("Processing chunk %d of %d of %s", currentChunk, numberOfChunks, filePath)
readBytes, readError := reader.Read(buffer)
if readError == io.EOF && readBytes == 0 {
break
}
if readError != io.EOF && readError != nil {
return readError
}
encodedChunk := base64.StdEncoding.EncodeToString(buffer[:readBytes])
uploadError := uploadImageChunk(context, encodedChunk, md5sum, currentChunk)
if uploadError != nil {
return uploadError
}
currentChunk++
}
return nil
}
func uploadImageChunk(context *piwigo.PiwigoContext, base64chunk string, md5sum string, position int64) error {
formData := url.Values{}
formData.Set("method", "pwg.images.addChunk")
formData.Set("data", base64chunk)
formData.Set("original_sum", md5sum)
// required by the API for compatibility
formData.Set("type", "file")
formData.Set("position", strconv.Itoa(position))
formData.Set("position", strconv.FormatInt(position, 10))
logrus.Tracef("Uploading chunk %d of file with sum %s", position, md5sum)
@ -51,31 +107,32 @@ func uploadImageChunk(context *piwigo.PiwigoContext, base64chunk string, md5sum
return nil
}
func uploadImageFinal(context *piwigo.PiwigoContext, originalFilename string, md5sum string, categoryId int) error {
func uploadImageFinal(context *piwigo.PiwigoContext, originalFilename string, md5sum string, categoryId int) (int, error) {
formData := url.Values{}
formData.Set("method", "pwg.images.add")
formData.Set("original_sum", md5sum)
formData.Set("original_filename", originalFilename)
formData.Set("categoriesi", strconv.Itoa(categoryId))
formData.Set("name", originalFilename)
formData.Set("categories", strconv.Itoa(categoryId))
logrus.Debugf("Finalizing upload of file %s with sum %s to category %d", originalFilename, md5sum, categoryId)
response, err := context.PostForm(formData)
if err != nil {
return err
return 0, err
}
defer response.Body.Close()
var fileAddResponse fileAddResponse
if err := json.NewDecoder(response.Body).Decode(&fileAddResponse); err != nil {
logrus.Errorln(err)
return err
return 0, err
}
if fileAddResponse.Status != "ok" {
logrus.Errorf("Got state %s while adding image %s", fileAddResponse.Status, originalFilename)
return errors.New(fmt.Sprintf("Got state %s while adding image %s", fileAddResponse.Status, originalFilename))
return 0, errors.New(fmt.Sprintf("Got state %s while adding image %s", fileAddResponse.Status, originalFilename))
}
return nil
return fileAddResponse.Result.ImageID, nil
}