package app import ( "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure" "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo" "github.com/sirupsen/logrus" "path/filepath" ) // to make use of the new local data store, we have to rethink and refactor the whole local detection process // extend the storage of the images to keep track of upload state // TBD: How to deal with updates -> delete / upload all based on md5 sums type fileChecksumCalculator func(filePath string) (string, error) // Update the local image metadata by walking through all found files and check if the modification date has changed // or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well. func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, categories map[string]*piwigo.PiwigoCategory, checksumCalculator fileChecksumCalculator) error { logrus.Debugf("Starting synchronizeLocalImageMetadata") logrus.Info("Synchronizing local image metadata database with local available images") for _, file := range fileSystemNodes { if file.IsDir { // we are only interested in files not directories continue } metadata, err := metadataStorage.ImageMetadata(file.Path) if err == ErrorRecordNotFound { logrus.Debugf("No metadata for %s found. Creating new entry.", file.Key) metadata = ImageMetaData{} metadata.Filename = file.Name metadata.FullImagePath = file.Path metadata.CategoryPath = filepath.Dir(file.Key) category, exist := categories[metadata.CategoryPath] if exist { metadata.CategoryId = category.Id } else { logrus.Warnf("No category found for image %s", file.Path) } } else if err != nil { logrus.Errorf("Could not get metadata due to trouble. Cancelling - %s", err) return err } if metadata.LastChange.Equal(file.ModTime) { logrus.Infof("No changed detected on file %s -> keeping current state", file.Key) continue } metadata.LastChange = file.ModTime metadata.UploadRequired = true metadata.Md5Sum, err = checksumCalculator(file.Path) if err != nil { logrus.Warnf("Could not calculate checksum for file %s. Skipping...", file.Path) continue } err = metadataStorage.SaveImageMetadata(metadata) if err != nil { return err } } logrus.Debugf("Finished synchronizeLocalImageMetadata") return nil } // Uploads the pending images to the piwigo gallery and assign the category of to the image. // Update local metadata and set upload flag to false. Also updates the piwigo image id if there was a difference. func uploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider ImageMetadataProvider) error { logrus.Debugf("Starting uploadImages") images, err := metadataProvider.ImageMetadataToUpload() if err != nil { return err } logrus.Infof("Uploading %d images to piwigo", len(images)) for _, img := range images { imgId, err := piwigoCtx.UploadImage(img.PiwigoId, img.FullImagePath, img.Md5Sum, img.CategoryId) if err != nil { logrus.Warnf("could not upload image %s. Continuing with the next image.", img.FullImagePath) continue } if imgId > 0 && imgId != img.PiwigoId { img.PiwigoId = imgId logrus.Debugf("Updating image %d with piwigo id %d", img.ImageId, img.PiwigoId) } logrus.Infof("Successfully uploaded %s", img.FullImagePath) img.UploadRequired = false err = metadataProvider.SaveImageMetadata(img) if err != nil { logrus.Warnf("could not save uploaded image %s. Continuing with the next image.", img.FullImagePath) continue } } logrus.Debugf("Finished uploadImages successfully") return nil } // This method aggregates the check for files with missing piwigoids and if changed files need to be uploaded again. func synchronizePiwigoMetadata(piwigoCtx piwigo.PiwigoImageApi, metadataProvider ImageMetadataProvider) error { // TODO: check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later logrus.Debugf("Starting synchronizePiwigoMetadata") err := updatePiwigoIdIfAlreadyUploaded(metadataProvider, piwigoCtx) if err != nil { return err } err = checkPiwigoForChangedImages(metadataProvider, piwigoCtx) if err != nil { return err } return nil } // Check all images with upload required if they are really changed and need to be uploaded to the server. func checkPiwigoForChangedImages(provider ImageMetadataProvider, piwigoCtx piwigo.PiwigoImageApi) error { logrus.Infof("Checking pending files if they really differ from the version in piwigo...") images, err := provider.ImageMetadataToUpload() if err != nil { return err } if len(images) == 0 { logrus.Info("There are no existing images to check for modification on the server.") return nil } for _, img := range images { if img.PiwigoId == 0 { continue } state, err := piwigoCtx.ImageCheckFile(img.PiwigoId, img.Md5Sum) if err != nil { logrus.Warnf("Error during file change check of file %s", img.FullImagePath) continue } if state == piwigo.ImageStateUptodate { logrus.Debugf("File %s - %d has not changed", img.FullImagePath, img.PiwigoId) img.UploadRequired = false err = provider.SaveImageMetadata(img) if err != nil { logrus.Warnf("Could not save image data of image %s", img.FullImagePath) } } } return nil } // This function calls piwigo and checks if the given md5sum is already present. // Only files without a piwigo id are used to query the server. func updatePiwigoIdIfAlreadyUploaded(provider ImageMetadataProvider, piwigoCtx piwigo.PiwigoImageApi) error { logrus.Infof("checking for pending files that are already on piwigo and updating piwigoids...") images, err := provider.ImageMetadataToUpload() if err != nil { return err } if len(images) == 0 { logrus.Info("There are no existing images to check for modification on the server.") return nil } logrus.Debugln("Preparing lookuplist for missing piwigo ids...") files := make([]string, 0, len(images)) for _, img := range images { if img.PiwigoId == 0 { files = append(files, img.Md5Sum) } } missingResults, err := piwigoCtx.ImagesExistOnPiwigo(files) if err != nil { return err } for md5sum, piwigoId := range missingResults { logrus.Debugf("Setting piwigo id of %s to %d", md5sum, piwigoId) err = provider.SavePiwigoIdAndUpdateUploadFlag(md5sum, piwigoId) if err != nil { logrus.Warnf("Could not save piwigo id %d for file %s", piwigoId, md5sum) } } return nil }