piwigo/PiwigoDirectoryUploader
2
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Removed old empty method

Browse Source 
added synchronizePiwigoMetadata implementations to update local metadata from piwigo
This commit is contained in:
Philipp Häfelfinger 2019-03-19 23:08:28 +01:00
parent 7bf14e5ca3
commit f3f864ec49
3 changed files with 85 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
internal/app/app.go
View File

@ -53,10 +53,10 @@ func Run() {
logErrorAndExit(err, 7)
}
err = synchronizeImages(context.piwigo, context.dataStore, categories)
if err != nil {
logErrorAndExit(err, 8)
}
//err = synchronizeImages(context.piwigo, context.dataStore, categories)
//if err != nil {
// logErrorAndExit(err, 8)
//}
_ = piwigo.Logout(context.piwigo)
}

150
internal/app/images.go
View File

@ -1,26 +1,22 @@
package app
import (
"errors"
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure"
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
"github.com/sirupsen/logrus"
"path/filepath"
)
type fileChecksumCalculator func(filePath string) (string, error)
// to make use of the new local data store, we have to rethink and refactor the whole local detection process
// extend the storage of the images to keep track of upload state
// TBD: How to deal with updates -> delete / upload all based on md5 sums
func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error {
// STEP 1 - update and sync local datastore with filesystem
// - walk through all files of the fileSystem map
// - get file metadata from filesystem (date, filename, dir, modtime etc.)
// - recalculate md5 sum if file changed referring to the stored record (reduces load after first calculation a lot)
// - mark metadata as upload required if changed or new
logrus.Debugf("Starting synchronizeLocalImageMetadata")
type fileChecksumCalculator func(filePath string) (string, error)
// Update the local image metadata by walking through all found files and check if the modification date has changed
// or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well.
func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error {
logrus.Debugf("Starting synchronizeLocalImageMetadata")
logrus.Info("Synchronizing local image metadata database with local available images")
for _, file := range fileSystemNodes {
@ -64,81 +60,89 @@ func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSy
return nil
}
func synchronizePiwigoMetadata(piwigo *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider) error {
// STEP 2 - get file states from piwigo (pwg.images.checkFiles)
// - make bulk query possible
// - get upload status of md5 sum from piwigo for all marked to upload
// - build method to update database with new piwigoId by md5 sum
// - set UploadRequired to false and the returned piwigoid
// - check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later
// This method agregates the check for files with missing piwigoids and if changed files need to be uploaded again.
func synchronizePiwigoMetadata(piwigoCtx *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider) error {
// TODO: check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later
logrus.Debugf("Starting synchronizePiwigoMetadata")
images, err := metadataStorage.ImageMetadataToUpload()
if err != nil {
err := updatePiwigoIdIfAlreadyUploaded(metadataStorage, piwigoCtx)
if err != nil {
return err
}
logrus.Infof("Synchronizing metadata with existing Piwigo data...")
for img := range images {
logrus.Debug(img)
err = checkPiwigoForChangedImages(metadataStorage, piwigoCtx)
if err != nil {
return err
}
logrus.Debugf("Finished synchronizePiwigoMetadata")
return errors.New("N/A")
return nil
}
// Check all images with upload required if they are really changed and need to be uploaded to the server.
func checkPiwigoForChangedImages(provider ImageMetadataProvider, piwigoCtx *piwigo.PiwigoContext) error {
logrus.Infof("checking for pending files that are already on piwigo and updating piwigoids...")
images, err := provider.ImageMetadataToUpload()
if err != nil {
return err
}
for _, img := range images {
if img.PiwigoId == 0 {
continue
}
state, err := piwigo.ImageCheckFile(piwigoCtx, img.PiwigoId, img.Md5Sum)
if err != nil {
logrus.Warnf("Error during file change check of file %s", img.RelativeImagePath)
continue
}
if state == piwigo.ImageStateUptodate {
logrus.Debugf("File %s - %d has not changed", img.RelativeImagePath, img.PiwigoId)
img.UploadRequired = false
err = provider.SaveImageMetadata(*img)
if err != nil {
logrus.Warnf("Could not save image data of image %s", img.RelativeImagePath)
}
}
}
return nil
}
// This function calls piwigo and checks if the given md5sum is already present.
// Only files without a piwigo id are used to query the server.
func updatePiwigoIdIfAlreadyUploaded(provider ImageMetadataProvider, piwiCtx *piwigo.PiwigoContext) error {
logrus.Infof("checking for pending files that are already on piwigo and updating piwigoids...")
images, err := provider.ImageMetadataToUpload()
if err != nil {
return err
}
logrus.Debugln("Preparing lookuplist for missing piwigo ids...")
files := make([]string, 0, len(images))
for _, img := range images {
if img.PiwigoId == 0 {
files = append(files, img.Md5Sum)
}
}
missingResults, err := piwigo.ImagesExistOnPiwigo(piwiCtx, files)
if err != nil {
return err
}
for md5sum, piwigoId := range missingResults {
logrus.Debugf("Setting piwigo id of %s to %d", md5sum, piwigoId)
err = provider.SavePiwigoIdAndUpdateUploadFlag(md5sum, piwigoId)
if err != nil {
logrus.Warnf("Could not save piwigo id %d for file %s", piwigoId, md5sum)
}
}
return nil
}
// STEP 3: Upload missing images
// - upload file in chunks
// - assign image to category
func synchronizeImages(piwigo *piwigo.PiwigoContext, metadataStorage ImageMetadataProvider, existingCategories map[string]*piwigo.PiwigoCategory) error {
//imageFiles, err := localFileStructure.GetImageList(fileSystem)
//if err != nil {
// return err
//}
//
//missingFiles, err := findMissingImages(context, imageFiles)
//if err != nil {
// return err
//}
//
//err = uploadImages(context, missingFiles, existingCategories)
//if err != nil {
// return err
//}
//
//logrus.Infof("Synchronized %d files.", len(missingFiles))
return nil
}
//func findMissingImages(context *appContext, imageFiles []*localFileStructure.ImageNode) ([]*localFileStructure.ImageNode, error) {
//
// logrus.Debugln("Preparing lookuplist for missing files...")
//
// files := make([]string, 0, len(imageFiles))
// md5map := make(map[string]*localFileStructure.ImageNode, len(imageFiles))
// for _, file := range imageFiles {
// md5map[file.Md5Sum] = file
// files = append(files, file.Md5Sum)
// }
//
// missingSums, err := piwigo.ImageUploadRequired(context.piwigo, files)
// if err != nil {
// return nil, err
// }
//
// missingFiles := make([]*localFileStructure.ImageNode, 0, len(missingSums))
// for _, sum := range missingSums {
// file := md5map[sum]
// logrus.Infof("Found missing file %s", file.Path)
// missingFiles = append(missingFiles, file)
// }
//
// logrus.Infof("Found %d missing files", len(missingFiles))
//
// return missingFiles, nil
//}
//
//func uploadImages(context *appContext, missingFiles []*localFileStructure.ImageNode, existingCategories map[string]*piwigo.PiwigoCategory) error {
//

4
internal/app/images_test.go
View File

@ -206,6 +206,10 @@ func (d *testStore) ImageMetadataToUpload() ([]*ImageMetaData, error) {
return nil, errors.New("N/A")
}
func (d *testStore) SavePiwigoIdAndUpdateUploadFlag(md5Sum string, piwigoId int) error {
return errors.New("N/A")
}
// to make the sync testable, we pass in a simple mock that returns the filepath as checksum
func testChecksumCalculator(file string) (string, error) {
return file, nil