made upload run in parallel with 4 as default

This commit is contained in:
Philipp Häfelfinger 2019-04-08 23:08:27 +02:00
parent d62fc55c9e
commit 059b2df215
5 changed files with 64 additions and 24 deletions

View File

@ -11,17 +11,17 @@ Currently the following features are supported
- Check if an image needs to be uploaded (only md5sum variant currently supported)
- Upload image and assign it to the album based on the directory structure
- Upload updated images that changed locally
- Local metadata storage using sqlite to make change detection easier
- Local imagemetadata / category storage using sqlite to make change detection easier
- Rebuild the local metadata database without uploading any pictures. Though, The categories get created!
- Remove images no longer present (configurable)
- Uses all CPU Cores to calculate initial metadata
- Upload 4 files in parallel by default (configurable)
There are some features planned but not ready yet:
- Optimize performance on initial matadata build up.
- Upload more than one file at a time
- Fully support files within multiple albums
- Specify more than one root path to gather images on the local system
- Storing categories in the local database
- Rework source to use go modules
## Build and run the application
@ -102,6 +102,8 @@ Usage of ./PiwigoDirectoryUploader:
The minimum log level required to write out a log message. (panic,fatal,error,warn,info,debug,trace) (default "info")
-noUpload
If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90
-parallelUploads int
Set the number of images that get uploaded in parallel. (default 4)
-piwigoPassword string
This is password to the given username.
-piwigoUrl string

View File

@ -4,6 +4,7 @@ configUpdateInterval = 0s # Update interval for re-reading config file set via
imagesRootPath = # This is the images root path that should be mirrored to piwigo.
logLevel = info # The minimum log level required to write out a log message. (panic,fatal,error,warn,info,debug,trace)
noUpload = false # If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90
parallelUploads = 4 # Set the number of images that get uploaded in parallel.
piwigoPassword = # This is password to the given username.
piwigoUrl = # The root url without tailing slash to your piwigo installation.
piwigoUser = # The username to use during sync.

View File

@ -15,13 +15,14 @@ import (
)
var (
imagesRootPath = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.")
sqliteDb = flag.String("sqliteDb", "./localstate.db", "The connection string to the sql lite database file.")
noUpload = flag.Bool("noUpload", false, "If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90")
piwigoUrl = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.")
piwigoUser = flag.String("piwigoUser", "", "The username to use during sync.")
piwigoPassword = flag.String("piwigoPassword", "", "This is password to the given username.")
removeImages = flag.Bool("removeImages", false, "If set to true, images scheduled to delete will be removed from the piwigo server. Be sure you want to delete images before enabling this flag.")
imagesRootPath = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.")
sqliteDb = flag.String("sqliteDb", "./localstate.db", "The connection string to the sql lite database file.")
noUpload = flag.Bool("noUpload", false, "If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90")
piwigoUrl = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.")
piwigoUser = flag.String("piwigoUser", "", "The username to use during sync.")
piwigoPassword = flag.String("piwigoPassword", "", "This is password to the given username.")
removeImages = flag.Bool("removeImages", false, "If set to true, images scheduled to delete will be removed from the piwigo server. Be sure you want to delete images before enabling this flag.")
parallelUploads = flag.Int("parallelUploads", 4, "Set the number of images that get uploaded in parallel.")
)
func Run() {
@ -65,7 +66,7 @@ func Run() {
}
if !(*noUpload) {
err = images.UploadImages(context.piwigo, context.dataStore)
err = images.UploadImages(context.piwigo, context.dataStore, *parallelUploads)
if err != nil {
logErrorAndExit(err, 8)
}

View File

@ -9,11 +9,12 @@ import (
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
"github.com/sirupsen/logrus"
"sync"
)
// Uploads the pending images to the piwigo gallery and assign the category of to the image.
// Update local metadata and set upload flag to false. Also updates the piwigo image id if there was a difference.
func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider) error {
func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider, numberOfWorkers int) error {
logrus.Debug("Starting uploadImages")
defer logrus.Debug("Finished uploadImages successfully")
@ -22,30 +23,65 @@ func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.Im
return err
}
logrus.Infof("Uploading %d images to piwigo", len(images))
if len(images) == 0 {
logrus.Info("No images to upload.")
return nil
}
for _, img := range images {
if numberOfWorkers <= 0 {
logrus.Warnf("Invalid numbers of worker set: %d falling back to default of 4", numberOfWorkers)
numberOfWorkers = 4
}
logrus.Infof("Uploading %d images to piwigo using %d workers", len(images), numberOfWorkers)
workQueue := make(chan datastore.ImageMetaData, numberOfWorkers)
wg := sync.WaitGroup{}
wg.Add(1)
go uploadQueueProducer(images, workQueue, &wg)
for i := 0; i < numberOfWorkers; i++ {
logrus.Debugf("Starting image upload worker %d", i)
wg.Add(1)
go uploadQueueWorker(workQueue, piwigoCtx, metadataProvider, &wg)
}
wg.Wait()
return nil
}
func uploadQueueWorker(workQueue <-chan datastore.ImageMetaData, piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider, waitGroup *sync.WaitGroup) {
for img := range workQueue {
logrus.Debugf("%s: uploading image to piwigo", img.FullImagePath)
imgId, err := piwigoCtx.UploadImage(img.PiwigoId, img.FullImagePath, img.Md5Sum, img.CategoryPiwigoId)
if err != nil {
logrus.Warnf("could not upload image %s. Continuing with the next image.", img.FullImagePath)
logrus.Warnf("%s: could not upload image. Continuing with the next image.", img.FullImagePath)
continue
}
if imgId > 0 && imgId != img.PiwigoId {
img.PiwigoId = imgId
logrus.Debugf("Updating image %d with piwigo id %d", img.ImageId, img.PiwigoId)
logrus.Debugf("%s: Updating image %d with piwigo id %d", img.FullImagePath, img.ImageId, img.PiwigoId)
}
logrus.Infof("Successfully uploaded %s", img.FullImagePath)
logrus.Infof("%s: Successfully uploaded", img.FullImagePath)
img.UploadRequired = false
err = metadataProvider.SaveImageMetadata(img)
if err != nil {
logrus.Warnf("could not save uploaded image %s. Continuing with the next image.", img.FullImagePath)
logrus.Warnf("%s: could not save uploaded image. Continuing with the next image.", img.FullImagePath)
continue
}
}
return nil
waitGroup.Done()
}
func uploadQueueProducer(imagesToUpload []datastore.ImageMetaData, workQueue chan<- datastore.ImageMetaData, waitGroup *sync.WaitGroup) {
for _, img := range imagesToUpload {
logrus.Debugf("%s: Adding image to queue", img.FullImagePath)
workQueue <- img
}
waitGroup.Done()
close(workQueue)
}

View File

@ -29,7 +29,7 @@ func Test_uploadImages_saves_new_id_to_db(t *testing.T) {
piwigomock := NewMockPiwigoImageApi(mockCtrl)
piwigomock.EXPECT().UploadImage(0, "/nonexisting/file.jpg", "1234", 2).Times(1).Return(5, nil)
err := UploadImages(piwigomock, dbmock)
err := UploadImages(piwigomock, dbmock, 1)
if err != nil {
t.Error(err)
}
@ -52,7 +52,7 @@ func Test_uploadImages_saves_same_id_to_db(t *testing.T) {
piwigomock := NewMockPiwigoImageApi(mockCtrl)
piwigomock.EXPECT().UploadImage(5, "/nonexisting/file.jpg", "1234", 2).Times(1).Return(5, nil)
err := UploadImages(piwigomock, dbmock)
err := UploadImages(piwigomock, dbmock, 1)
if err != nil {
t.Error(err)
}