split code into multiple files with each file dedicated to one purpose
This commit is contained in:
parent
2468bf3e16
commit
56e21b2d90
48
internal/pkg/images/delete.go
Normal file
48
internal/pkg/images/delete.go
Normal file
@ -0,0 +1,48 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2019 Philipp Haefelfinger (http://www.haefelfinger.ch/). All Rights Reserved.
|
||||||
|
* This application is licensed under GPLv2. See the LICENSE file in the root directory of the project.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package images
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
func DeleteImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider) error {
|
||||||
|
logrus.Debug("Starting deleteImages")
|
||||||
|
defer logrus.Debug("Finished deleteImages successfully")
|
||||||
|
|
||||||
|
images, err := metadataProvider.ImageMetadataToDelete()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(images) == 0 {
|
||||||
|
logrus.Info("There are not images scheduled for deletion.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
logrus.Infof("Deleting %d images from piwigo", len(images))
|
||||||
|
|
||||||
|
var piwigoIds []int = nil
|
||||||
|
for _, img := range images {
|
||||||
|
if img.PiwigoId > 0 {
|
||||||
|
logrus.Tracef("Adding %d to deletable list", img.PiwigoId)
|
||||||
|
piwigoIds = append(piwigoIds, img.PiwigoId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(piwigoIds) > 0 {
|
||||||
|
err = piwigoCtx.DeleteImages(piwigoIds)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
logrus.Info("Only local images found to delete. No call to piwigo is made.")
|
||||||
|
}
|
||||||
|
|
||||||
|
return metadataProvider.DeleteMarkedImages()
|
||||||
|
}
|
||||||
75
internal/pkg/images/delete_test.go
Normal file
75
internal/pkg/images/delete_test.go
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2019 Philipp Haefelfinger (http://www.haefelfinger.ch/). All Rights Reserved.
|
||||||
|
* This application is licensed under GPLv2. See the LICENSE file in the root directory of the project.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package images
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
|
||||||
|
"github.com/golang/mock/gomock"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func Test_deleteImages_should_call_piwigo_and_remove_metadata(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
img := createTestImageMetaData(5)
|
||||||
|
img.UploadRequired = false
|
||||||
|
img.DeleteRequired = true
|
||||||
|
images := []datastore.ImageMetaData{img}
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToDelete().Times(1).Return(images, nil)
|
||||||
|
dbmock.EXPECT().DeleteMarkedImages().Times(1).Return(nil)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().DeleteImages([]int{5}).Times(1).Return(nil)
|
||||||
|
|
||||||
|
err := DeleteImages(piwigomock, dbmock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_deleteImages_should_not_call_piwigo_for_not_uploaded_images_and_remove_metadata(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
img := createTestImageMetaData(0)
|
||||||
|
img.UploadRequired = false
|
||||||
|
img.DeleteRequired = true
|
||||||
|
images := []datastore.ImageMetaData{img}
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToDelete().Times(1).Return(images, nil)
|
||||||
|
dbmock.EXPECT().DeleteMarkedImages().Times(1).Return(nil)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().DeleteImages(gomock.Any()).Times(0)
|
||||||
|
|
||||||
|
err := DeleteImages(piwigomock, dbmock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_deleteImages_should_not_call_anything_if_no_images_are_marked_for_deletion(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
images := []datastore.ImageMetaData{}
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToDelete().Times(1).Return(images, nil)
|
||||||
|
dbmock.EXPECT().DeleteMarkedImages().Times(0)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().DeleteImages(gomock.Any()).Times(0)
|
||||||
|
|
||||||
|
err := DeleteImages(piwigomock, dbmock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,300 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (C) 2019 Philipp Haefelfinger (http://www.haefelfinger.ch/). All Rights Reserved.
|
|
||||||
* This application is licensed under GPLv2. See the LICENSE file in the root directory of the project.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package images
|
|
||||||
|
|
||||||
import (
|
|
||||||
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
|
|
||||||
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure"
|
|
||||||
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
)
|
|
||||||
|
|
||||||
// to make use of the new local data store, we have to rethink and refactor the whole local detection process
|
|
||||||
// extend the storage of the images to keep track of upload state
|
|
||||||
// TBD: How to deal with updates -> delete / upload all based on md5 sums
|
|
||||||
|
|
||||||
type fileChecksumCalculator func(filePath string) (string, error)
|
|
||||||
|
|
||||||
// Update the local image metadata by walking through all found files and check if the modification date has changed
|
|
||||||
// or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well.
|
|
||||||
func SynchronizeLocalImageMetadata(imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error {
|
|
||||||
logrus.Debug("Starting SynchronizeLocalImageMetadata")
|
|
||||||
defer logrus.Debug("Leaving SynchronizeLocalImageMetadata")
|
|
||||||
|
|
||||||
logrus.Info("Synchronizing local image metadata database with local available images")
|
|
||||||
|
|
||||||
err := synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes, imageDb, categoryDb, checksumCalculator)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = synchronizeLocalImageMetadataFindFilesToDelete(imageDb)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*localFileStructure.FilesystemNode, imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, checksumCalculator fileChecksumCalculator) error {
|
|
||||||
logrus.Debug("Entering synchronizeLocalImageMetadataScanNewFiles")
|
|
||||||
defer logrus.Debug("Leaving synchronizeLocalImageMetadataScanNewFiles")
|
|
||||||
|
|
||||||
for _, file := range fileSystemNodes {
|
|
||||||
if file.IsDir {
|
|
||||||
// we are only interested in files not directories
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
metadata, err := imageDb.ImageMetadata(file.Path)
|
|
||||||
if err == datastore.ErrorRecordNotFound {
|
|
||||||
logrus.Debugf("Creating new metadata entry for %s.", file.Path)
|
|
||||||
metadata = datastore.ImageMetaData{}
|
|
||||||
metadata.Filename = file.Name
|
|
||||||
metadata.FullImagePath = file.Path
|
|
||||||
metadata.CategoryPath = filepath.Dir(file.Key)
|
|
||||||
|
|
||||||
category, err := categoryDb.GetCategoryByKey(metadata.CategoryPath)
|
|
||||||
if err == nil {
|
|
||||||
metadata.CategoryPiwigoId = category.PiwigoId
|
|
||||||
} else {
|
|
||||||
logrus.Warnf("No category found for image %s - %s", file.Path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
} else if err != nil {
|
|
||||||
logrus.Errorf("Could not get metadata due to trouble. Cancelling - %s", err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if fileDidNotChange(&metadata, file) {
|
|
||||||
logrus.Debugf("No changes found for file %s", file.Path)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
metadata.UploadRequired = !metadata.LastChange.Equal(file.ModTime) || metadata.PiwigoId == 0
|
|
||||||
metadata.DeleteRequired = false
|
|
||||||
metadata.LastChange = file.ModTime
|
|
||||||
metadata.Md5Sum, err = checksumCalculator(file.Path)
|
|
||||||
if err != nil {
|
|
||||||
logrus.Warnf("Could not calculate checksum for file %s. Skipping...", file.Path)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err = imageDb.SaveImageMetadata(metadata)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func synchronizeLocalImageMetadataFindFilesToDelete(imageDb datastore.ImageMetadataProvider) error {
|
|
||||||
logrus.Debug("Entering SynchronizeLocalImageMetadataFindFilesToDelete")
|
|
||||||
defer logrus.Debug("Leaving SynchronizeLocalImageMetadataFindFilesToDelete")
|
|
||||||
|
|
||||||
images, err := imageDb.ImageMetadataAll()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, img := range images {
|
|
||||||
if _, err := os.Stat(img.FullImagePath); os.IsNotExist(err) {
|
|
||||||
img.UploadRequired = false
|
|
||||||
img.DeleteRequired = true
|
|
||||||
err := imageDb.SaveImageMetadata(img)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Uploads the pending images to the piwigo gallery and assign the category of to the image.
|
|
||||||
// Update local metadata and set upload flag to false. Also updates the piwigo image id if there was a difference.
|
|
||||||
func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider) error {
|
|
||||||
logrus.Debug("Starting uploadImages")
|
|
||||||
defer logrus.Debug("Finished uploadImages successfully")
|
|
||||||
|
|
||||||
images, err := metadataProvider.ImageMetadataToUpload()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
logrus.Infof("Uploading %d images to piwigo", len(images))
|
|
||||||
|
|
||||||
for _, img := range images {
|
|
||||||
|
|
||||||
imgId, err := piwigoCtx.UploadImage(img.PiwigoId, img.FullImagePath, img.Md5Sum, img.CategoryPiwigoId)
|
|
||||||
if err != nil {
|
|
||||||
logrus.Warnf("could not upload image %s. Continuing with the next image.", img.FullImagePath)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if imgId > 0 && imgId != img.PiwigoId {
|
|
||||||
img.PiwigoId = imgId
|
|
||||||
logrus.Debugf("Updating image %d with piwigo id %d", img.ImageId, img.PiwigoId)
|
|
||||||
}
|
|
||||||
|
|
||||||
logrus.Infof("Successfully uploaded %s", img.FullImagePath)
|
|
||||||
|
|
||||||
img.UploadRequired = false
|
|
||||||
err = metadataProvider.SaveImageMetadata(img)
|
|
||||||
if err != nil {
|
|
||||||
logrus.Warnf("could not save uploaded image %s. Continuing with the next image.", img.FullImagePath)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func DeleteImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider) error {
|
|
||||||
logrus.Debug("Starting deleteImages")
|
|
||||||
defer logrus.Debug("Finished deleteImages successfully")
|
|
||||||
|
|
||||||
images, err := metadataProvider.ImageMetadataToDelete()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(images) == 0 {
|
|
||||||
logrus.Info("There are not images scheduled for deletion.")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
logrus.Infof("Deleting %d images from piwigo", len(images))
|
|
||||||
|
|
||||||
var piwigoIds []int = nil
|
|
||||||
for _, img := range images {
|
|
||||||
if img.PiwigoId > 0 {
|
|
||||||
logrus.Tracef("Adding %d to deletable list", img.PiwigoId)
|
|
||||||
piwigoIds = append(piwigoIds, img.PiwigoId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(piwigoIds) > 0 {
|
|
||||||
err = piwigoCtx.DeleteImages(piwigoIds)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
logrus.Info("Only local images found to delete. No call to piwigo is made.")
|
|
||||||
}
|
|
||||||
|
|
||||||
return metadataProvider.DeleteMarkedImages()
|
|
||||||
}
|
|
||||||
|
|
||||||
// This method aggregates the check for files with missing piwigoids and if changed files need to be uploaded again.
|
|
||||||
func SynchronizePiwigoMetadata(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider) error {
|
|
||||||
logrus.Debug("Entering SynchronizePiwigoMetadata")
|
|
||||||
defer logrus.Debug("Leaving SynchronizePiwigoMetadata")
|
|
||||||
|
|
||||||
// TODO: check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later
|
|
||||||
err := updatePiwigoIdIfAlreadyUploaded(metadataProvider, piwigoCtx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = checkPiwigoForChangedImages(metadataProvider, piwigoCtx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check all images with upload required if they are really changed and need to be uploaded to the server.
|
|
||||||
func checkPiwigoForChangedImages(provider datastore.ImageMetadataProvider, piwigoCtx piwigo.PiwigoImageApi) error {
|
|
||||||
logrus.Info("Checking pending files if they really differ from the version in piwigo...")
|
|
||||||
defer logrus.Info("Finished checking pending files if they really differ from the version in piwigo...")
|
|
||||||
|
|
||||||
images, err := provider.ImageMetadataToUpload()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(images) == 0 {
|
|
||||||
logrus.Info("There are no existing images to check for modification on the server.")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, img := range images {
|
|
||||||
if img.PiwigoId == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
state, err := piwigoCtx.ImageCheckFile(img.PiwigoId, img.Md5Sum)
|
|
||||||
if err != nil {
|
|
||||||
logrus.Warnf("Error during file change check of file %s", img.FullImagePath)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if state == piwigo.ImageStateUptodate {
|
|
||||||
logrus.Debugf("File %s - %d has not changed", img.FullImagePath, img.PiwigoId)
|
|
||||||
img.UploadRequired = false
|
|
||||||
err = provider.SaveImageMetadata(img)
|
|
||||||
if err != nil {
|
|
||||||
logrus.Warnf("Could not save image data of image %s", img.FullImagePath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// This function calls piwigo and checks if the given md5sum is already present.
|
|
||||||
// Only files without a piwigo id are used to query the server.
|
|
||||||
func updatePiwigoIdIfAlreadyUploaded(provider datastore.ImageMetadataProvider, piwigoCtx piwigo.PiwigoImageApi) error {
|
|
||||||
logrus.Info("checking for pending files that are already on piwigo and updating piwigoids...")
|
|
||||||
defer logrus.Info("finshed checking for pending files that are already on piwigo and updating piwigoids...")
|
|
||||||
|
|
||||||
images, err := provider.ImageMetadataToUpload()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(images) == 0 {
|
|
||||||
logrus.Info("There are no existing images to check for modification on the server.")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
logrus.Debugln("Preparing lookuplist for missing piwigo ids...")
|
|
||||||
files := make([]string, 0, len(images))
|
|
||||||
for _, img := range images {
|
|
||||||
if img.PiwigoId == 0 {
|
|
||||||
files = append(files, img.Md5Sum)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(files) == 0 {
|
|
||||||
logrus.Info("There are no images without piwigo id to check for modification on the server.")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
missingResults, err := piwigoCtx.ImagesExistOnPiwigo(files)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for md5sum, piwigoId := range missingResults {
|
|
||||||
if piwigoId > 0 {
|
|
||||||
logrus.Debugf("Setting piwigo id of %s to %d", md5sum, piwigoId)
|
|
||||||
err = provider.SavePiwigoIdAndUpdateUploadFlag(md5sum, piwigoId)
|
|
||||||
if err != nil {
|
|
||||||
logrus.Warnf("Could not save piwigo id %d for file %s", piwigoId, md5sum)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
logrus.Tracef("Image %s not found on server", md5sum)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func fileDidNotChange(metadata *datastore.ImageMetaData, file *localFileStructure.FilesystemNode) bool {
|
|
||||||
return metadata.LastChange.Equal(file.ModTime) && !metadata.DeleteRequired
|
|
||||||
}
|
|
||||||
114
internal/pkg/images/synchronizeLocalFiles.go
Normal file
114
internal/pkg/images/synchronizeLocalFiles.go
Normal file
@ -0,0 +1,114 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2019 Philipp Haefelfinger (http://www.haefelfinger.ch/). All Rights Reserved.
|
||||||
|
* This application is licensed under GPLv2. See the LICENSE file in the root directory of the project.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package images
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fileChecksumCalculator func(filePath string) (string, error)
|
||||||
|
|
||||||
|
// Update the local image metadata by walking through all found files and check if the modification date has changed
|
||||||
|
// or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well.
|
||||||
|
func SynchronizeLocalImageMetadata(imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error {
|
||||||
|
logrus.Debug("Starting SynchronizeLocalImageMetadata")
|
||||||
|
defer logrus.Debug("Leaving SynchronizeLocalImageMetadata")
|
||||||
|
|
||||||
|
logrus.Info("Synchronizing local image metadata database with local available images")
|
||||||
|
|
||||||
|
err := synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes, imageDb, categoryDb, checksumCalculator)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = synchronizeLocalImageMetadataFindFilesToDelete(imageDb)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*localFileStructure.FilesystemNode, imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, checksumCalculator fileChecksumCalculator) error {
|
||||||
|
logrus.Debug("Entering synchronizeLocalImageMetadataScanNewFiles")
|
||||||
|
defer logrus.Debug("Leaving synchronizeLocalImageMetadataScanNewFiles")
|
||||||
|
|
||||||
|
for _, file := range fileSystemNodes {
|
||||||
|
if file.IsDir {
|
||||||
|
// we are only interested in files not directories
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata, err := imageDb.ImageMetadata(file.Path)
|
||||||
|
if err == datastore.ErrorRecordNotFound {
|
||||||
|
logrus.Debugf("Creating new metadata entry for %s.", file.Path)
|
||||||
|
metadata = datastore.ImageMetaData{}
|
||||||
|
metadata.Filename = file.Name
|
||||||
|
metadata.FullImagePath = file.Path
|
||||||
|
metadata.CategoryPath = filepath.Dir(file.Key)
|
||||||
|
|
||||||
|
category, err := categoryDb.GetCategoryByKey(metadata.CategoryPath)
|
||||||
|
if err == nil {
|
||||||
|
metadata.CategoryPiwigoId = category.PiwigoId
|
||||||
|
} else {
|
||||||
|
logrus.Warnf("No category found for image %s - %s", file.Path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
} else if err != nil {
|
||||||
|
logrus.Errorf("Could not get metadata due to trouble. Cancelling - %s", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if fileDidNotChange(&metadata, file) {
|
||||||
|
logrus.Debugf("No changes found for file %s", file.Path)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata.UploadRequired = !metadata.LastChange.Equal(file.ModTime) || metadata.PiwigoId == 0
|
||||||
|
metadata.DeleteRequired = false
|
||||||
|
metadata.LastChange = file.ModTime
|
||||||
|
metadata.Md5Sum, err = checksumCalculator(file.Path)
|
||||||
|
if err != nil {
|
||||||
|
logrus.Warnf("Could not calculate checksum for file %s. Skipping...", file.Path)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
err = imageDb.SaveImageMetadata(metadata)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func synchronizeLocalImageMetadataFindFilesToDelete(imageDb datastore.ImageMetadataProvider) error {
|
||||||
|
logrus.Debug("Entering SynchronizeLocalImageMetadataFindFilesToDelete")
|
||||||
|
defer logrus.Debug("Leaving SynchronizeLocalImageMetadataFindFilesToDelete")
|
||||||
|
|
||||||
|
images, err := imageDb.ImageMetadataAll()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, img := range images {
|
||||||
|
if _, err := os.Stat(img.FullImagePath); os.IsNotExist(err) {
|
||||||
|
img.UploadRequired = false
|
||||||
|
img.DeleteRequired = true
|
||||||
|
err := imageDb.SaveImageMetadata(img)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileDidNotChange(metadata *datastore.ImageMetaData, file *localFileStructure.FilesystemNode) bool {
|
||||||
|
return metadata.LastChange.Equal(file.ModTime) && !metadata.DeleteRequired
|
||||||
|
}
|
||||||
@ -11,7 +11,6 @@ package images
|
|||||||
import (
|
import (
|
||||||
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
|
||||||
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure"
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure"
|
||||||
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
|
|
||||||
"github.com/golang/mock/gomock"
|
"github.com/golang/mock/gomock"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@ -209,242 +208,6 @@ func Test_synchronize_local_image_metadata_should_not_process_directories(t *tes
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func Test_checkPiwigoForChangedImages_none_with_piwigoId(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
img := datastore.ImageMetaData{ImageId: 1, UploadRequired: true}
|
|
||||||
images := []datastore.ImageMetaData{img}
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(0)
|
|
||||||
piwigomock.EXPECT().ImageCheckFile(gomock.Any(), gomock.Any()).Times(0)
|
|
||||||
|
|
||||||
err := checkPiwigoForChangedImages(dbmock, piwigomock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_checkPiwigoForChangedImages_with_empty_list(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
images := []datastore.ImageMetaData{}
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(0)
|
|
||||||
piwigomock.EXPECT().ImageCheckFile(gomock.Any(), gomock.Any()).Times(0)
|
|
||||||
|
|
||||||
err := checkPiwigoForChangedImages(dbmock, piwigomock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_checkPiwigoForChangedImages_should_call_piwigo_set_uploadRequired_to_false(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
img := datastore.ImageMetaData{
|
|
||||||
ImageId: 1,
|
|
||||||
PiwigoId: 1,
|
|
||||||
UploadRequired: true,
|
|
||||||
Md5Sum: "1234",
|
|
||||||
}
|
|
||||||
images := []datastore.ImageMetaData{img}
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
|
||||||
|
|
||||||
imgExpected := img
|
|
||||||
imgExpected.UploadRequired = false
|
|
||||||
dbmock.EXPECT().SaveImageMetadata(imgExpected).Times(1)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().ImageCheckFile(1, "1234").Return(piwigo.ImageStateUptodate, nil)
|
|
||||||
|
|
||||||
err := checkPiwigoForChangedImages(dbmock, piwigomock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_checkPiwigoForChangedImages_return_image_differs(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
img := datastore.ImageMetaData{
|
|
||||||
ImageId: 1,
|
|
||||||
PiwigoId: 1,
|
|
||||||
UploadRequired: true,
|
|
||||||
Md5Sum: "1234",
|
|
||||||
}
|
|
||||||
images := []datastore.ImageMetaData{img}
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
|
||||||
dbmock.EXPECT().SaveImageMetadata(gomock.Any()).Times(0)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().ImageCheckFile(1, "1234").Return(piwigo.ImageStateDifferent, nil)
|
|
||||||
|
|
||||||
err := checkPiwigoForChangedImages(dbmock, piwigomock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_updatePiwigoIdIfAlreadyUploaded_without_images_to_upload(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
images := []datastore.ImageMetaData{}
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
|
||||||
dbmock.EXPECT().SavePiwigoIdAndUpdateUploadFlag(gomock.Any(), gomock.Any()).Times(0)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(0)
|
|
||||||
|
|
||||||
err := updatePiwigoIdIfAlreadyUploaded(dbmock, piwigomock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_updatePiwigoIdIfAlreadyUploaded_without_image_to_check(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
img := datastore.ImageMetaData{
|
|
||||||
ImageId: 1,
|
|
||||||
PiwigoId: 1,
|
|
||||||
UploadRequired: true,
|
|
||||||
Md5Sum: "1234",
|
|
||||||
}
|
|
||||||
images := []datastore.ImageMetaData{img}
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
|
||||||
dbmock.EXPECT().SavePiwigoIdAndUpdateUploadFlag(gomock.Any(), gomock.Any()).Times(0)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(0)
|
|
||||||
|
|
||||||
err := updatePiwigoIdIfAlreadyUploaded(dbmock, piwigomock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_updatePiwigoIdIfAlreadyUploaded_with_image_to_check(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
img := datastore.ImageMetaData{
|
|
||||||
ImageId: 1,
|
|
||||||
PiwigoId: 0,
|
|
||||||
UploadRequired: true,
|
|
||||||
Md5Sum: "1234",
|
|
||||||
}
|
|
||||||
images := []datastore.ImageMetaData{img}
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
|
||||||
dbmock.EXPECT().SavePiwigoIdAndUpdateUploadFlag("1234", 1).Times(1)
|
|
||||||
|
|
||||||
piwigoResponose := make(map[string]int)
|
|
||||||
piwigoResponose["1234"] = 1
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(1).Return(piwigoResponose, nil)
|
|
||||||
|
|
||||||
err := updatePiwigoIdIfAlreadyUploaded(dbmock, piwigomock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_updatePiwigoIdIfAlreadyUploaded_with_image_to_check_missing_on_server(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
img := datastore.ImageMetaData{
|
|
||||||
ImageId: 1,
|
|
||||||
PiwigoId: 0,
|
|
||||||
UploadRequired: true,
|
|
||||||
Md5Sum: "1234",
|
|
||||||
}
|
|
||||||
images := []datastore.ImageMetaData{img}
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
|
||||||
dbmock.EXPECT().SavePiwigoIdAndUpdateUploadFlag("1234", 1).Times(0)
|
|
||||||
|
|
||||||
piwigoResponose := make(map[string]int)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(1).Return(piwigoResponose, nil)
|
|
||||||
|
|
||||||
err := updatePiwigoIdIfAlreadyUploaded(dbmock, piwigomock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_uploadImages_saves_new_id_to_db(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
img := createTestImageMetaData(0)
|
|
||||||
images := []datastore.ImageMetaData{img}
|
|
||||||
|
|
||||||
imgToSave := img
|
|
||||||
imgToSave.PiwigoId = 5
|
|
||||||
imgToSave.UploadRequired = false
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToUpload().Times(1).Return(images, nil)
|
|
||||||
dbmock.EXPECT().SaveImageMetadata(imgToSave).Times(1)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().UploadImage(0, "/nonexisting/file.jpg", "1234", 2).Times(1).Return(5, nil)
|
|
||||||
|
|
||||||
err := UploadImages(piwigomock, dbmock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_uploadImages_saves_same_id_to_db(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
img := createTestImageMetaData(5)
|
|
||||||
images := []datastore.ImageMetaData{img}
|
|
||||||
|
|
||||||
imgToSave := img
|
|
||||||
imgToSave.UploadRequired = false
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToUpload().Times(1).Return(images, nil)
|
|
||||||
dbmock.EXPECT().SaveImageMetadata(imgToSave).Times(1)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().UploadImage(5, "/nonexisting/file.jpg", "1234", 2).Times(1).Return(5, nil)
|
|
||||||
|
|
||||||
err := UploadImages(piwigomock, dbmock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_synchronizeLocalImageMetadataFindFilesToDelete(t *testing.T) {
|
func Test_synchronizeLocalImageMetadataFindFilesToDelete(t *testing.T) {
|
||||||
mockCtrl := gomock.NewController(t)
|
mockCtrl := gomock.NewController(t)
|
||||||
defer mockCtrl.Finish()
|
defer mockCtrl.Finish()
|
||||||
@ -466,69 +229,6 @@ func Test_synchronizeLocalImageMetadataFindFilesToDelete(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func Test_deleteImages_should_call_piwigo_and_remove_metadata(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
img := createTestImageMetaData(5)
|
|
||||||
img.UploadRequired = false
|
|
||||||
img.DeleteRequired = true
|
|
||||||
images := []datastore.ImageMetaData{img}
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToDelete().Times(1).Return(images, nil)
|
|
||||||
dbmock.EXPECT().DeleteMarkedImages().Times(1).Return(nil)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().DeleteImages([]int{5}).Times(1).Return(nil)
|
|
||||||
|
|
||||||
err := DeleteImages(piwigomock, dbmock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_deleteImages_should_not_call_piwigo_for_not_uploaded_images_and_remove_metadata(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
img := createTestImageMetaData(0)
|
|
||||||
img.UploadRequired = false
|
|
||||||
img.DeleteRequired = true
|
|
||||||
images := []datastore.ImageMetaData{img}
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToDelete().Times(1).Return(images, nil)
|
|
||||||
dbmock.EXPECT().DeleteMarkedImages().Times(1).Return(nil)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().DeleteImages(gomock.Any()).Times(0)
|
|
||||||
|
|
||||||
err := DeleteImages(piwigomock, dbmock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_deleteImages_should_not_call_anything_if_no_images_are_marked_for_deletion(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
images := []datastore.ImageMetaData{}
|
|
||||||
|
|
||||||
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
|
||||||
dbmock.EXPECT().ImageMetadataToDelete().Times(1).Return(images, nil)
|
|
||||||
dbmock.EXPECT().DeleteMarkedImages().Times(0)
|
|
||||||
|
|
||||||
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
|
||||||
piwigomock.EXPECT().DeleteImages(gomock.Any()).Times(0)
|
|
||||||
|
|
||||||
err := DeleteImages(piwigomock, dbmock)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// to make the sync testable, we pass in a simple mock that returns the filepath as checksum
|
// to make the sync testable, we pass in a simple mock that returns the filepath as checksum
|
||||||
func testChecksumCalculator(file string) (string, error) {
|
func testChecksumCalculator(file string) (string, error) {
|
||||||
return file, nil
|
return file, nil
|
||||||
118
internal/pkg/images/synchronizePiwigo.go
Normal file
118
internal/pkg/images/synchronizePiwigo.go
Normal file
@ -0,0 +1,118 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2019 Philipp Haefelfinger (http://www.haefelfinger.ch/). All Rights Reserved.
|
||||||
|
* This application is licensed under GPLv2. See the LICENSE file in the root directory of the project.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package images
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This method aggregates the check for files with missing piwigoids and if changed files need to be uploaded again.
|
||||||
|
func SynchronizePiwigoMetadata(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider) error {
|
||||||
|
logrus.Debug("Entering SynchronizePiwigoMetadata")
|
||||||
|
defer logrus.Debug("Leaving SynchronizePiwigoMetadata")
|
||||||
|
|
||||||
|
// TODO: check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later
|
||||||
|
err := updatePiwigoIdIfAlreadyUploaded(metadataProvider, piwigoCtx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = checkPiwigoForChangedImages(metadataProvider, piwigoCtx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// This function calls piwigo and checks if the given md5sum is already present.
|
||||||
|
// Only files without a piwigo id are used to query the server.
|
||||||
|
func updatePiwigoIdIfAlreadyUploaded(provider datastore.ImageMetadataProvider, piwigoCtx piwigo.PiwigoImageApi) error {
|
||||||
|
logrus.Info("checking for pending files that are already on piwigo and updating piwigoids...")
|
||||||
|
defer logrus.Info("finshed checking for pending files that are already on piwigo and updating piwigoids...")
|
||||||
|
|
||||||
|
images, err := provider.ImageMetadataToUpload()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(images) == 0 {
|
||||||
|
logrus.Info("There are no existing images to check for modification on the server.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
logrus.Debugln("Preparing lookuplist for missing piwigo ids...")
|
||||||
|
files := make([]string, 0, len(images))
|
||||||
|
for _, img := range images {
|
||||||
|
if img.PiwigoId == 0 {
|
||||||
|
files = append(files, img.Md5Sum)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(files) == 0 {
|
||||||
|
logrus.Info("There are no images without piwigo id to check for modification on the server.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
missingResults, err := piwigoCtx.ImagesExistOnPiwigo(files)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for md5sum, piwigoId := range missingResults {
|
||||||
|
if piwigoId > 0 {
|
||||||
|
logrus.Debugf("Setting piwigo id of %s to %d", md5sum, piwigoId)
|
||||||
|
err = provider.SavePiwigoIdAndUpdateUploadFlag(md5sum, piwigoId)
|
||||||
|
if err != nil {
|
||||||
|
logrus.Warnf("Could not save piwigo id %d for file %s", piwigoId, md5sum)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
logrus.Tracef("Image %s not found on server", md5sum)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check all images with upload required if they are really changed and need to be uploaded to the server.
|
||||||
|
func checkPiwigoForChangedImages(provider datastore.ImageMetadataProvider, piwigoCtx piwigo.PiwigoImageApi) error {
|
||||||
|
logrus.Info("Checking pending files if they really differ from the version in piwigo...")
|
||||||
|
defer logrus.Info("Finished checking pending files if they really differ from the version in piwigo...")
|
||||||
|
|
||||||
|
images, err := provider.ImageMetadataToUpload()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(images) == 0 {
|
||||||
|
logrus.Info("There are no existing images to check for modification on the server.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, img := range images {
|
||||||
|
if img.PiwigoId == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
state, err := piwigoCtx.ImageCheckFile(img.PiwigoId, img.Md5Sum)
|
||||||
|
if err != nil {
|
||||||
|
logrus.Warnf("Error during file change check of file %s", img.FullImagePath)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if state == piwigo.ImageStateUptodate {
|
||||||
|
logrus.Debugf("File %s - %d has not changed", img.FullImagePath, img.PiwigoId)
|
||||||
|
img.UploadRequired = false
|
||||||
|
err = provider.SaveImageMetadata(img)
|
||||||
|
if err != nil {
|
||||||
|
logrus.Warnf("Could not save image data of image %s", img.FullImagePath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
202
internal/pkg/images/synchronizePiwigo_test.go
Normal file
202
internal/pkg/images/synchronizePiwigo_test.go
Normal file
@ -0,0 +1,202 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2019 Philipp Haefelfinger (http://www.haefelfinger.ch/). All Rights Reserved.
|
||||||
|
* This application is licensed under GPLv2. See the LICENSE file in the root directory of the project.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package images
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
|
||||||
|
"github.com/golang/mock/gomock"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func Test_checkPiwigoForChangedImages_none_with_piwigoId(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
img := datastore.ImageMetaData{ImageId: 1, UploadRequired: true}
|
||||||
|
images := []datastore.ImageMetaData{img}
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(0)
|
||||||
|
piwigomock.EXPECT().ImageCheckFile(gomock.Any(), gomock.Any()).Times(0)
|
||||||
|
|
||||||
|
err := checkPiwigoForChangedImages(dbmock, piwigomock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_checkPiwigoForChangedImages_with_empty_list(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
images := []datastore.ImageMetaData{}
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(0)
|
||||||
|
piwigomock.EXPECT().ImageCheckFile(gomock.Any(), gomock.Any()).Times(0)
|
||||||
|
|
||||||
|
err := checkPiwigoForChangedImages(dbmock, piwigomock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_checkPiwigoForChangedImages_should_call_piwigo_set_uploadRequired_to_false(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
img := datastore.ImageMetaData{
|
||||||
|
ImageId: 1,
|
||||||
|
PiwigoId: 1,
|
||||||
|
UploadRequired: true,
|
||||||
|
Md5Sum: "1234",
|
||||||
|
}
|
||||||
|
images := []datastore.ImageMetaData{img}
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
||||||
|
|
||||||
|
imgExpected := img
|
||||||
|
imgExpected.UploadRequired = false
|
||||||
|
dbmock.EXPECT().SaveImageMetadata(imgExpected).Times(1)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().ImageCheckFile(1, "1234").Return(piwigo.ImageStateUptodate, nil)
|
||||||
|
|
||||||
|
err := checkPiwigoForChangedImages(dbmock, piwigomock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_checkPiwigoForChangedImages_return_image_differs(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
img := datastore.ImageMetaData{
|
||||||
|
ImageId: 1,
|
||||||
|
PiwigoId: 1,
|
||||||
|
UploadRequired: true,
|
||||||
|
Md5Sum: "1234",
|
||||||
|
}
|
||||||
|
images := []datastore.ImageMetaData{img}
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
||||||
|
dbmock.EXPECT().SaveImageMetadata(gomock.Any()).Times(0)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().ImageCheckFile(1, "1234").Return(piwigo.ImageStateDifferent, nil)
|
||||||
|
|
||||||
|
err := checkPiwigoForChangedImages(dbmock, piwigomock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_updatePiwigoIdIfAlreadyUploaded_without_images_to_upload(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
images := []datastore.ImageMetaData{}
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
||||||
|
dbmock.EXPECT().SavePiwigoIdAndUpdateUploadFlag(gomock.Any(), gomock.Any()).Times(0)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(0)
|
||||||
|
|
||||||
|
err := updatePiwigoIdIfAlreadyUploaded(dbmock, piwigomock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_updatePiwigoIdIfAlreadyUploaded_without_image_to_check(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
img := datastore.ImageMetaData{
|
||||||
|
ImageId: 1,
|
||||||
|
PiwigoId: 1,
|
||||||
|
UploadRequired: true,
|
||||||
|
Md5Sum: "1234",
|
||||||
|
}
|
||||||
|
images := []datastore.ImageMetaData{img}
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
||||||
|
dbmock.EXPECT().SavePiwigoIdAndUpdateUploadFlag(gomock.Any(), gomock.Any()).Times(0)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(0)
|
||||||
|
|
||||||
|
err := updatePiwigoIdIfAlreadyUploaded(dbmock, piwigomock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_updatePiwigoIdIfAlreadyUploaded_with_image_to_check(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
img := datastore.ImageMetaData{
|
||||||
|
ImageId: 1,
|
||||||
|
PiwigoId: 0,
|
||||||
|
UploadRequired: true,
|
||||||
|
Md5Sum: "1234",
|
||||||
|
}
|
||||||
|
images := []datastore.ImageMetaData{img}
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
||||||
|
dbmock.EXPECT().SavePiwigoIdAndUpdateUploadFlag("1234", 1).Times(1)
|
||||||
|
|
||||||
|
piwigoResponose := make(map[string]int)
|
||||||
|
piwigoResponose["1234"] = 1
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(1).Return(piwigoResponose, nil)
|
||||||
|
|
||||||
|
err := updatePiwigoIdIfAlreadyUploaded(dbmock, piwigomock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_updatePiwigoIdIfAlreadyUploaded_with_image_to_check_missing_on_server(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
img := datastore.ImageMetaData{
|
||||||
|
ImageId: 1,
|
||||||
|
PiwigoId: 0,
|
||||||
|
UploadRequired: true,
|
||||||
|
Md5Sum: "1234",
|
||||||
|
}
|
||||||
|
images := []datastore.ImageMetaData{img}
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToUpload().Return(images, nil)
|
||||||
|
dbmock.EXPECT().SavePiwigoIdAndUpdateUploadFlag("1234", 1).Times(0)
|
||||||
|
|
||||||
|
piwigoResponose := make(map[string]int)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().ImagesExistOnPiwigo(gomock.Any()).Times(1).Return(piwigoResponose, nil)
|
||||||
|
|
||||||
|
err := updatePiwigoIdIfAlreadyUploaded(dbmock, piwigomock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
51
internal/pkg/images/upload.go
Normal file
51
internal/pkg/images/upload.go
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2019 Philipp Haefelfinger (http://www.haefelfinger.ch/). All Rights Reserved.
|
||||||
|
* This application is licensed under GPLv2. See the LICENSE file in the root directory of the project.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package images
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Uploads the pending images to the piwigo gallery and assign the category of to the image.
|
||||||
|
// Update local metadata and set upload flag to false. Also updates the piwigo image id if there was a difference.
|
||||||
|
func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider) error {
|
||||||
|
logrus.Debug("Starting uploadImages")
|
||||||
|
defer logrus.Debug("Finished uploadImages successfully")
|
||||||
|
|
||||||
|
images, err := metadataProvider.ImageMetadataToUpload()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
logrus.Infof("Uploading %d images to piwigo", len(images))
|
||||||
|
|
||||||
|
for _, img := range images {
|
||||||
|
|
||||||
|
imgId, err := piwigoCtx.UploadImage(img.PiwigoId, img.FullImagePath, img.Md5Sum, img.CategoryPiwigoId)
|
||||||
|
if err != nil {
|
||||||
|
logrus.Warnf("could not upload image %s. Continuing with the next image.", img.FullImagePath)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if imgId > 0 && imgId != img.PiwigoId {
|
||||||
|
img.PiwigoId = imgId
|
||||||
|
logrus.Debugf("Updating image %d with piwigo id %d", img.ImageId, img.PiwigoId)
|
||||||
|
}
|
||||||
|
|
||||||
|
logrus.Infof("Successfully uploaded %s", img.FullImagePath)
|
||||||
|
|
||||||
|
img.UploadRequired = false
|
||||||
|
err = metadataProvider.SaveImageMetadata(img)
|
||||||
|
if err != nil {
|
||||||
|
logrus.Warnf("could not save uploaded image %s. Continuing with the next image.", img.FullImagePath)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
59
internal/pkg/images/upload_test.go
Normal file
59
internal/pkg/images/upload_test.go
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2019 Philipp Haefelfinger (http://www.haefelfinger.ch/). All Rights Reserved.
|
||||||
|
* This application is licensed under GPLv2. See the LICENSE file in the root directory of the project.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package images
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
|
||||||
|
"github.com/golang/mock/gomock"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func Test_uploadImages_saves_new_id_to_db(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
img := createTestImageMetaData(0)
|
||||||
|
images := []datastore.ImageMetaData{img}
|
||||||
|
|
||||||
|
imgToSave := img
|
||||||
|
imgToSave.PiwigoId = 5
|
||||||
|
imgToSave.UploadRequired = false
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToUpload().Times(1).Return(images, nil)
|
||||||
|
dbmock.EXPECT().SaveImageMetadata(imgToSave).Times(1)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().UploadImage(0, "/nonexisting/file.jpg", "1234", 2).Times(1).Return(5, nil)
|
||||||
|
|
||||||
|
err := UploadImages(piwigomock, dbmock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_uploadImages_saves_same_id_to_db(t *testing.T) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
img := createTestImageMetaData(5)
|
||||||
|
images := []datastore.ImageMetaData{img}
|
||||||
|
|
||||||
|
imgToSave := img
|
||||||
|
imgToSave.UploadRequired = false
|
||||||
|
|
||||||
|
dbmock := NewMockImageMetadataProvider(mockCtrl)
|
||||||
|
dbmock.EXPECT().ImageMetadataToUpload().Times(1).Return(images, nil)
|
||||||
|
dbmock.EXPECT().SaveImageMetadata(imgToSave).Times(1)
|
||||||
|
|
||||||
|
piwigomock := NewMockPiwigoImageApi(mockCtrl)
|
||||||
|
piwigomock.EXPECT().UploadImage(5, "/nonexisting/file.jpg", "1234", 2).Times(1).Return(5, nil)
|
||||||
|
|
||||||
|
err := UploadImages(piwigomock, dbmock)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user