piwigo/PiwigoDirectoryUploader
2
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

made finding missing files work

Browse Source
This commit is contained in:
Philipp Häfelfinger 2019-02-27 23:26:18 +01:00
parent 45db2fe79b
commit a4a6cce943
6 changed files with 91 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/app/category.go
View File

@ -30,7 +30,7 @@ func synchronizeCategories(context *appContext, filesystemNodes map[string]*loca
}
func findMissingCategories(fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*category.PiwigoCategory) []string {
missingCategories := []string{}
missingCategories := make([]string, 0, len(fileSystem))
for _, file := range fileSystem {
if !file.IsDir {

36
internal/app/images.go
View File

@ -5,6 +5,7 @@ import (
"github.com/sirupsen/logrus"
"haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/localFileStructure"
"haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/category"
"haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo/picture"
)
func synchronizeImages(context *appContext, fileSystem map[string]*localFileStructure.FilesystemNode, existingCategories map[string]*category.PiwigoCategory) error {
@ -14,19 +15,44 @@ func synchronizeImages(context *appContext, fileSystem map[string]*localFileStru
return err
}
missingFiles := findMissingImages(imageFiles)
missingFiles, err := findMissingImages(context, imageFiles)
if err != nil {
return err
}
uploadImages(missingFiles)
return errors.New("synchronizeImages: NOT IMPLEMENTED")
}
func findMissingImages(imageFiles []*localFileStructure.ImageNode) []string {
func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) {
logrus.Warnln("Finding missing images (NotImplemented)")
logrus.Debugln("Preparing lookuplist for missing files...")
return nil
files := make([]string, 0, len(imageFiles))
md5map := make(map[string]*localFileStructure.ImageNode, len(imageFiles))
for _, file := range imageFiles {
md5map[file.Md5Sum] = file
files = append(files, file.Md5Sum)
}
misingSums, err := picture.ImageUploadRequired(context.Piwigo, files)
if err != nil {
return nil, err
}
missingFiles := make([]*localFileStructure.ImageNode, 0, len(misingSums))
for _, sum := range misingSums {
file := md5map[sum]
logrus.Infof("Found missing file %s", file.Path)
missingFiles = append(missingFiles, file)
}
logrus.Infof("Found %d missing files", len(missingFiles))
return missingFiles, nil
}
func uploadImages(missingFiles []string) {
func uploadImages(missingFiles []*localFileStructure.ImageNode) {
logrus.Warnln("Uploading missing images (NotImplemented)")
}

10
internal/pkg/localFileStructure/checksumCalculator.go
View File

@ -9,20 +9,20 @@ import (
)
func calculateFileCheckSums(filePath string) (string, error) {
f, err := os.Open(filePath)
file, err := os.Open(filePath)
if err != nil {
logrus.Errorf("Could not open file %s", filePath)
return "", err
}
defer f.Close()
defer file.Close()
h := md5.New()
if _, err := io.Copy(h, f); err != nil {
hash := md5.New()
if _, err := io.Copy(hash, file); err != nil {
logrus.Errorf("Could calculate md5 sum of file %s", filePath)
return "", err
}
md5sum := fmt.Sprintf("%x", md5.Sum(nil))
md5sum := fmt.Sprintf("%x", hash.Sum(nil))
logrus.Tracef("Calculated md5 sum of %s - %s", filePath, md5sum)

14
internal/pkg/localFileStructure/imageList.go
View File

@ -2,11 +2,12 @@ package localFileStructure
import (
"github.com/sirupsen/logrus"
"path/filepath"
"time"
)
func GetImageList(fileSystem map[string]*FilesystemNode) ([]*ImageNode, error) {
imageFiles := []*ImageNode{}
imageFiles := make([]*ImageNode, 0, len(fileSystem))
for _, file := range fileSystem {
if file.IsDir {
@ -18,14 +19,17 @@ func GetImageList(fileSystem map[string]*FilesystemNode) ([]*ImageNode, error) {
return nil, err
}
logrus.Debugf("Image %s - %s - %s", md5sum, file.ModTime.Format(time.RFC3339), file.Path)
logrus.Debugf("Local Image %s - %s - %s", md5sum, file.ModTime.Format(time.RFC3339), file.Path)
imageFiles = append(imageFiles, &ImageNode{
Path: file.Path,
ModTime: file.ModTime,
Md5Sum: md5sum,
Path: file.Path,
Directory: filepath.Dir(file.Path),
ModTime: file.ModTime,
Md5Sum: md5sum,
})
}
logrus.Infof("Found %d local images to process", len(imageFiles))
return imageFiles, nil
}

7
internal/pkg/localFileStructure/types.go
View File

@ -18,7 +18,8 @@ func (n *FilesystemNode) String() string {
}
type ImageNode struct {
Path string
ModTime time.Time
Md5Sum string
Path string
Directory string
ModTime time.Time
Md5Sum string
}

51
internal/pkg/piwigo/picture/query.go
View File

@ -1,22 +1,53 @@
package picture
import (
"encoding/json"
"errors"
"github.com/sirupsen/logrus"
"haefelfinger.net/piwigo/DirectoriesToAlbums/internal/pkg/piwigo"
"net/http"
"net/url"
"strings"
)
func ImageUploadRequired(context *piwigo.PiwigoContext, files []string) (bool, error) {
func ImageUploadRequired(context *piwigo.PiwigoContext, md5sums []string) ([]string, error) {
for file := range files {
logrus.Debug(file)
if context.Cookies == nil {
return nil, errors.New("Not logged in and no cookies found! Can not get the category list!")
}
/*
http://pictures.haefelfinger.net/ws.php?format=json
{
"md5sum_list": "d327416a83452b91764ed2888a5630a3,6d5f122e2b98bc1a192850e89fc2ae8c,40bfe8dd8349ccdedd4a939f9191cafa"
}
*/
//TODO: make sure to split to multiple queries -> to honor max upload queries
return false, nil
md5sumList := strings.Join(md5sums, ",")
formData := url.Values{}
formData.Set("method", "pwg.images.exist")
formData.Set("md5sum_list", md5sumList)
logrus.Tracef("Looking up missing files: %s", md5sumList)
client := http.Client{Jar: context.Cookies}
response, err := client.PostForm(context.Url, formData)
if err != nil {
logrus.Errorln("The HTTP request failed with error %s", err)
return nil, err
}
defer response.Body.Close()
var imageExistResponse imageExistResponse
if err := json.NewDecoder(response.Body).Decode(&imageExistResponse); err != nil {
logrus.Errorln(err)
return nil, err
}
missingFiles := make([]string, 0, len(imageExistResponse.Result))
for key, value := range imageExistResponse.Result {
if value == "" {
logrus.Tracef("Missing file with md5sum: %s", key)
missingFiles = append(missingFiles, key)
}
}
return missingFiles, nil
}