/* * Copyright (C) 2019 Philipp Haefelfinger (http://www.haefelfinger.ch/). All Rights Reserved. * This application is licensed under GPLv2. See the LICENSE file in the root directory of the project. */ package images import ( "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore" "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure" "github.com/sirupsen/logrus" "os" "path/filepath" "runtime" "sync" ) type fileChecksumCalculator func(filePath string) (string, error) // Update the local image metadata by walking through all found files and check if the modification date has changed // or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well. func SynchronizeLocalImageMetadata(imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error { logrus.Debug("Starting SynchronizeLocalImageMetadata") defer logrus.Debug("Leaving SynchronizeLocalImageMetadata") logrus.Info("Synchronizing local image metadata database with local available images") err := synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes, imageDb, categoryDb, checksumCalculator) if err != nil { return err } err = synchronizeLocalImageMetadataFindFilesToDelete(imageDb) if err != nil { return err } return nil } func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*localFileStructure.FilesystemNode, imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, checksumCalculator fileChecksumCalculator) error { logrus.Debug("Entering synchronizeLocalImageMetadataScanNewFiles") defer logrus.Debug("Leaving synchronizeLocalImageMetadataScanNewFiles") workQueue := make(chan localFileStructure.FilesystemNode, 128) wg := sync.WaitGroup{} wg.Add(1) logrus.Debug("Starting change detection producer") go checkFileForChangesProducer(fileSystemNodes, workQueue, &wg) for i := 0; i < runtime.NumCPU(); i++ { logrus.Debugf("Starting image change detection worker %d", i) wg.Add(1) go checkFileForChangesWorker(workQueue, &wg, imageDb, categoryDb, checksumCalculator) } wg.Wait() return nil } func checkFileForChangesProducer(fileSystemNodes map[string]*localFileStructure.FilesystemNode, workQueue chan<- localFileStructure.FilesystemNode, waitGroup *sync.WaitGroup) { for _, file := range fileSystemNodes { workQueue <- *file } waitGroup.Done() close(workQueue) } func checkFileForChangesWorker(workQueue <-chan localFileStructure.FilesystemNode, waitGroup *sync.WaitGroup, imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, checksumCalculator fileChecksumCalculator) { for file := range workQueue { if file.IsDir { // we are only interested in files not directories logrus.Tracef("Skipping file check as %s is a directory", file.Path) continue } metadata, err := imageDb.ImageMetadata(file.Path) if err == datastore.ErrorRecordNotFound { logrus.Debugf("Creating new metadata entry for %s.", file.Path) metadata = datastore.ImageMetaData{} metadata.Filename = file.Name metadata.FullImagePath = file.Path metadata.CategoryPath = filepath.Dir(file.Key) var category datastore.CategoryData category, err = categoryDb.GetCategoryByKey(metadata.CategoryPath) if err == nil { metadata.CategoryPiwigoId = category.PiwigoId } else { logrus.Warnf("No category found for image %s - %s", file.Path, err) } } else if err != nil { logrus.Errorf("Could not get metadata due to trouble. Cancelling - %s", err) continue } if fileDidNotChange(&metadata, &file) { logrus.Debugf("No changes found for file %s", file.Path) continue } metadata.UploadRequired = !metadata.LastChange.Equal(file.ModTime) || metadata.PiwigoId == 0 metadata.DeleteRequired = false metadata.LastChange = file.ModTime metadata.Md5Sum, err = checksumCalculator(file.Path) if err != nil { logrus.Warnf("Could not calculate checksum for file %s. Skipping...", file.Path) continue } err = imageDb.SaveImageMetadata(metadata) if err != nil { logrus.Errorf("Error during save of metadata of %s - %s", file.Path, err) } } waitGroup.Done() } func synchronizeLocalImageMetadataFindFilesToDelete(imageDb datastore.ImageMetadataProvider) error { logrus.Debug("Entering SynchronizeLocalImageMetadataFindFilesToDelete") defer logrus.Debug("Leaving SynchronizeLocalImageMetadataFindFilesToDelete") images, err := imageDb.ImageMetadataAll() if err != nil { return err } for _, img := range images { if _, err = os.Stat(img.FullImagePath); os.IsNotExist(err) { img.UploadRequired = false img.DeleteRequired = true err = imageDb.SaveImageMetadata(img) if err != nil { return err } } } return nil } func fileDidNotChange(metadata *datastore.ImageMetaData, file *localFileStructure.FilesystemNode) bool { return metadata.LastChange.Equal(file.ModTime) && !metadata.DeleteRequired }