diff --git a/README.md b/README.md index 9977df0..9b311dc 100644 --- a/README.md +++ b/README.md @@ -11,17 +11,17 @@ Currently the following features are supported - Check if an image needs to be uploaded (only md5sum variant currently supported) - Upload image and assign it to the album based on the directory structure - Upload updated images that changed locally -- Local metadata storage using sqlite to make change detection easier +- Local imagemetadata / category storage using sqlite to make change detection easier - Rebuild the local metadata database without uploading any pictures. Though, The categories get created! - Remove images no longer present (configurable) +- Uses all CPU Cores to calculate initial metadata +- Upload 4 files in parallel by default (configurable) There are some features planned but not ready yet: -- Optimize performance on initial matadata build up. -- Upload more than one file at a time - Fully support files within multiple albums - Specify more than one root path to gather images on the local system -- Storing categories in the local database +- Rework source to use go modules ## Build and run the application @@ -102,6 +102,8 @@ Usage of ./PiwigoDirectoryUploader: The minimum log level required to write out a log message. (panic,fatal,error,warn,info,debug,trace) (default "info") -noUpload If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90 + -parallelUploads int + Set the number of images that get uploaded in parallel. (default 4) -piwigoPassword string This is password to the given username. -piwigoUrl string diff --git a/configs/defaultConfig.ini b/configs/defaultConfig.ini index 718d34b..28d45c2 100644 --- a/configs/defaultConfig.ini +++ b/configs/defaultConfig.ini @@ -4,6 +4,7 @@ configUpdateInterval = 0s # Update interval for re-reading config file set via imagesRootPath = # This is the images root path that should be mirrored to piwigo. logLevel = info # The minimum log level required to write out a log message. (panic,fatal,error,warn,info,debug,trace) noUpload = false # If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90 +parallelUploads = 4 # Set the number of images that get uploaded in parallel. piwigoPassword = # This is password to the given username. piwigoUrl = # The root url without tailing slash to your piwigo installation. piwigoUser = # The username to use during sync. diff --git a/internal/app/app.go b/internal/app/app.go index bbdfa1c..7db7e6e 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -15,13 +15,14 @@ import ( ) var ( - imagesRootPath = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.") - sqliteDb = flag.String("sqliteDb", "./localstate.db", "The connection string to the sql lite database file.") - noUpload = flag.Bool("noUpload", false, "If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90") - piwigoUrl = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.") - piwigoUser = flag.String("piwigoUser", "", "The username to use during sync.") - piwigoPassword = flag.String("piwigoPassword", "", "This is password to the given username.") - removeImages = flag.Bool("removeImages", false, "If set to true, images scheduled to delete will be removed from the piwigo server. Be sure you want to delete images before enabling this flag.") + imagesRootPath = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.") + sqliteDb = flag.String("sqliteDb", "./localstate.db", "The connection string to the sql lite database file.") + noUpload = flag.Bool("noUpload", false, "If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90") + piwigoUrl = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.") + piwigoUser = flag.String("piwigoUser", "", "The username to use during sync.") + piwigoPassword = flag.String("piwigoPassword", "", "This is password to the given username.") + removeImages = flag.Bool("removeImages", false, "If set to true, images scheduled to delete will be removed from the piwigo server. Be sure you want to delete images before enabling this flag.") + parallelUploads = flag.Int("parallelUploads", 4, "Set the number of images that get uploaded in parallel.") ) func Run() { @@ -65,7 +66,7 @@ func Run() { } if !(*noUpload) { - err = images.UploadImages(context.piwigo, context.dataStore) + err = images.UploadImages(context.piwigo, context.dataStore, *parallelUploads) if err != nil { logErrorAndExit(err, 8) } diff --git a/internal/pkg/images/upload.go b/internal/pkg/images/upload.go index 3ffbab0..8e27cc2 100644 --- a/internal/pkg/images/upload.go +++ b/internal/pkg/images/upload.go @@ -9,11 +9,12 @@ import ( "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore" "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo" "github.com/sirupsen/logrus" + "sync" ) // Uploads the pending images to the piwigo gallery and assign the category of to the image. // Update local metadata and set upload flag to false. Also updates the piwigo image id if there was a difference. -func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider) error { +func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider, numberOfWorkers int) error { logrus.Debug("Starting uploadImages") defer logrus.Debug("Finished uploadImages successfully") @@ -22,30 +23,65 @@ func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.Im return err } - logrus.Infof("Uploading %d images to piwigo", len(images)) + if len(images) == 0 { + logrus.Info("No images to upload.") + return nil + } - for _, img := range images { + if numberOfWorkers <= 0 { + logrus.Warnf("Invalid numbers of worker set: %d falling back to default of 4", numberOfWorkers) + numberOfWorkers = 4 + } + + logrus.Infof("Uploading %d images to piwigo using %d workers", len(images), numberOfWorkers) + workQueue := make(chan datastore.ImageMetaData, numberOfWorkers) + + wg := sync.WaitGroup{} + + wg.Add(1) + go uploadQueueProducer(images, workQueue, &wg) + + for i := 0; i < numberOfWorkers; i++ { + logrus.Debugf("Starting image upload worker %d", i) + wg.Add(1) + go uploadQueueWorker(workQueue, piwigoCtx, metadataProvider, &wg) + } + + wg.Wait() + return nil +} + +func uploadQueueWorker(workQueue <-chan datastore.ImageMetaData, piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider, waitGroup *sync.WaitGroup) { + for img := range workQueue { + logrus.Debugf("%s: uploading image to piwigo", img.FullImagePath) imgId, err := piwigoCtx.UploadImage(img.PiwigoId, img.FullImagePath, img.Md5Sum, img.CategoryPiwigoId) if err != nil { - logrus.Warnf("could not upload image %s. Continuing with the next image.", img.FullImagePath) + logrus.Warnf("%s: could not upload image. Continuing with the next image.", img.FullImagePath) continue } if imgId > 0 && imgId != img.PiwigoId { img.PiwigoId = imgId - logrus.Debugf("Updating image %d with piwigo id %d", img.ImageId, img.PiwigoId) + logrus.Debugf("%s: Updating image %d with piwigo id %d", img.FullImagePath, img.ImageId, img.PiwigoId) } - - logrus.Infof("Successfully uploaded %s", img.FullImagePath) + logrus.Infof("%s: Successfully uploaded", img.FullImagePath) img.UploadRequired = false err = metadataProvider.SaveImageMetadata(img) if err != nil { - logrus.Warnf("could not save uploaded image %s. Continuing with the next image.", img.FullImagePath) + logrus.Warnf("%s: could not save uploaded image. Continuing with the next image.", img.FullImagePath) continue } } - - return nil + waitGroup.Done() +} + +func uploadQueueProducer(imagesToUpload []datastore.ImageMetaData, workQueue chan<- datastore.ImageMetaData, waitGroup *sync.WaitGroup) { + for _, img := range imagesToUpload { + logrus.Debugf("%s: Adding image to queue", img.FullImagePath) + workQueue <- img + } + waitGroup.Done() + close(workQueue) } diff --git a/internal/pkg/images/upload_test.go b/internal/pkg/images/upload_test.go index 22a140d..e8364dd 100644 --- a/internal/pkg/images/upload_test.go +++ b/internal/pkg/images/upload_test.go @@ -29,7 +29,7 @@ func Test_uploadImages_saves_new_id_to_db(t *testing.T) { piwigomock := NewMockPiwigoImageApi(mockCtrl) piwigomock.EXPECT().UploadImage(0, "/nonexisting/file.jpg", "1234", 2).Times(1).Return(5, nil) - err := UploadImages(piwigomock, dbmock) + err := UploadImages(piwigomock, dbmock, 1) if err != nil { t.Error(err) } @@ -52,7 +52,7 @@ func Test_uploadImages_saves_same_id_to_db(t *testing.T) { piwigomock := NewMockPiwigoImageApi(mockCtrl) piwigomock.EXPECT().UploadImage(5, "/nonexisting/file.jpg", "1234", 2).Times(1).Return(5, nil) - err := UploadImages(piwigomock, dbmock) + err := UploadImages(piwigomock, dbmock, 1) if err != nil { t.Error(err) }