PiwigoDirectoryUploader/internal/app/images.go
Philipp Häfelfinger b326b0da84 added uploadImages
added piwigo image id to upload to support file updates
added noUpload flag to do all but skip file upload
changed metadata filepath to full file path
moved category detection during local file metadata generation
2019-03-20 23:15:41 +01:00

200 lines
6.5 KiB
Go

package app
import (
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure"
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
"github.com/sirupsen/logrus"
"path/filepath"
)
// to make use of the new local data store, we have to rethink and refactor the whole local detection process
// extend the storage of the images to keep track of upload state
// TBD: How to deal with updates -> delete / upload all based on md5 sums
type fileChecksumCalculator func(filePath string) (string, error)
// Update the local image metadata by walking through all found files and check if the modification date has changed
// or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well.
func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, categories map[string]*piwigo.PiwigoCategory, checksumCalculator fileChecksumCalculator) error {
logrus.Debugf("Starting synchronizeLocalImageMetadata")
logrus.Info("Synchronizing local image metadata database with local available images")
for _, file := range fileSystemNodes {
if file.IsDir {
// we are only interested in files not directories
continue
}
metadata, err := metadataStorage.ImageMetadata(file.Path)
if err == ErrorRecordNotFound {
logrus.Debugf("No metadata for %s found. Creating new entry.", file.Key)
metadata = ImageMetaData{}
metadata.Filename = file.Name
metadata.FullImagePath = file.Path
metadata.CategoryPath = filepath.Dir(file.Key)
category, exist := categories[metadata.CategoryPath]
if exist {
metadata.CategoryId = category.Id
} else {
logrus.Warnf("No category found for image %s", file.Path)
}
} else if err != nil {
logrus.Errorf("Could not get metadata due to trouble. Cancelling - %s", err)
return err
}
if metadata.LastChange.Equal(file.ModTime) {
logrus.Infof("No changed detected on file %s -> keeping current state", file.Key)
continue
}
metadata.LastChange = file.ModTime
metadata.UploadRequired = true
metadata.Md5Sum, err = checksumCalculator(file.Path)
if err != nil {
logrus.Warnf("Could not calculate checksum for file %s. Skipping...", file.Path)
continue
}
err = metadataStorage.SaveImageMetadata(metadata)
if err != nil {
return err
}
}
logrus.Debugf("Finished synchronizeLocalImageMetadata")
return nil
}
// Uploads the pending images to the piwigo gallery and assign the category of to the image.
// Update local metadata and set upload flag to false. Also updates the piwigo image id if there was a difference.
func uploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider ImageMetadataProvider) error {
logrus.Debugf("Starting uploadImages")
images, err := metadataProvider.ImageMetadataToUpload()
if err != nil {
return err
}
logrus.Infof("Uploading %d images to piwigo", len(images))
for _, img := range images {
imgId, err := piwigoCtx.UploadImage(img.PiwigoId, img.FullImagePath, img.Md5Sum, img.CategoryId)
if err != nil {
logrus.Warnf("could not upload image %s. Continuing with the next image.", img.FullImagePath)
continue
}
if imgId > 0 && imgId != img.PiwigoId {
img.PiwigoId = imgId
logrus.Debugf("Updating image %d with piwigo id %d", img.ImageId, img.PiwigoId)
}
logrus.Infof("Successfully uploaded %s", img.FullImagePath)
img.UploadRequired = false
err = metadataProvider.SaveImageMetadata(img)
if err != nil {
logrus.Warnf("could not save uploaded image %s. Continuing with the next image.", img.FullImagePath)
continue
}
}
logrus.Debugf("Finished uploadImages successfully")
return nil
}
// This method aggregates the check for files with missing piwigoids and if changed files need to be uploaded again.
func synchronizePiwigoMetadata(piwigoCtx piwigo.PiwigoImageApi, metadataProvider ImageMetadataProvider) error {
// TODO: check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later
logrus.Debugf("Starting synchronizePiwigoMetadata")
err := updatePiwigoIdIfAlreadyUploaded(metadataProvider, piwigoCtx)
if err != nil {
return err
}
err = checkPiwigoForChangedImages(metadataProvider, piwigoCtx)
if err != nil {
return err
}
return nil
}
// Check all images with upload required if they are really changed and need to be uploaded to the server.
func checkPiwigoForChangedImages(provider ImageMetadataProvider, piwigoCtx piwigo.PiwigoImageApi) error {
logrus.Infof("Checking pending files if they really differ from the version in piwigo...")
images, err := provider.ImageMetadataToUpload()
if err != nil {
return err
}
if len(images) == 0 {
logrus.Info("There are no existing images to check for modification on the server.")
return nil
}
for _, img := range images {
if img.PiwigoId == 0 {
continue
}
state, err := piwigoCtx.ImageCheckFile(img.PiwigoId, img.Md5Sum)
if err != nil {
logrus.Warnf("Error during file change check of file %s", img.FullImagePath)
continue
}
if state == piwigo.ImageStateUptodate {
logrus.Debugf("File %s - %d has not changed", img.FullImagePath, img.PiwigoId)
img.UploadRequired = false
err = provider.SaveImageMetadata(img)
if err != nil {
logrus.Warnf("Could not save image data of image %s", img.FullImagePath)
}
}
}
return nil
}
// This function calls piwigo and checks if the given md5sum is already present.
// Only files without a piwigo id are used to query the server.
func updatePiwigoIdIfAlreadyUploaded(provider ImageMetadataProvider, piwigoCtx piwigo.PiwigoImageApi) error {
logrus.Infof("checking for pending files that are already on piwigo and updating piwigoids...")
images, err := provider.ImageMetadataToUpload()
if err != nil {
return err
}
if len(images) == 0 {
logrus.Info("There are no existing images to check for modification on the server.")
return nil
}
logrus.Debugln("Preparing lookuplist for missing piwigo ids...")
files := make([]string, 0, len(images))
for _, img := range images {
if img.PiwigoId == 0 {
files = append(files, img.Md5Sum)
}
}
missingResults, err := piwigoCtx.ImagesExistOnPiwigo(files)
if err != nil {
return err
}
for md5sum, piwigoId := range missingResults {
logrus.Debugf("Setting piwigo id of %s to %d", md5sum, piwigoId)
err = provider.SavePiwigoIdAndUpdateUploadFlag(md5sum, piwigoId)
if err != nil {
logrus.Warnf("Could not save piwigo id %d for file %s", piwigoId, md5sum)
}
}
return nil
}