made upload run in parallel with 4 as default

README.md

@@ -11,17 +11,17 @@ Currently the following features are supported
 - Check if an image needs to be uploaded (only md5sum variant currently supported)
 - Upload image and assign it to the album based on the directory structure
 - Upload updated images that changed locally
-- Local metadata storage using sqlite to make change detection easier
+- Local imagemetadata / category storage using sqlite to make change detection easier
 - Rebuild the local metadata database without uploading any pictures. Though, The categories get created!
 - Remove images no longer present (configurable)
+- Uses all CPU Cores to calculate initial metadata
+- Upload 4 files in parallel by default (configurable)
 
 There are some features planned but not ready yet:
 
-- Optimize performance on initial matadata build up.
-- Upload more than one file at a time
 - Fully support files within multiple albums
 - Specify more than one root path to gather images on the local system
-- Storing categories in the local database
+- Rework source to use go modules
 
 ## Build and run the application
 
@@ -102,6 +102,8 @@ Usage of ./PiwigoDirectoryUploader:
         The minimum log level required to write out a log message. (panic,fatal,error,warn,info,debug,trace) (default "info")
   -noUpload
         If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90
+  -parallelUploads int
+        Set the number of images that get uploaded in parallel. (default 4)
   -piwigoPassword string
         This is password to the given username.
   -piwigoUrl string

configs/defaultConfig.ini

@@ -4,6 +4,7 @@ configUpdateInterval = 0s  # Update interval for re-reading config file set via
 imagesRootPath =   # This is the images root path that should be mirrored to piwigo.
 logLevel = info  # The minimum log level required to write out a log message. (panic,fatal,error,warn,info,debug,trace)
 noUpload = false  # If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90
+parallelUploads = 4  # Set the number of images that get uploaded in parallel.
 piwigoPassword =   # This is password to the given username.
 piwigoUrl =   # The root url without tailing slash to your piwigo installation.
 piwigoUser =   # The username to use during sync.

internal/app/app.go

@@ -15,13 +15,14 @@ import (
 )
 
 var (
-	imagesRootPath = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.")
-	sqliteDb       = flag.String("sqliteDb", "./localstate.db", "The connection string to the sql lite database file.")
-	noUpload       = flag.Bool("noUpload", false, "If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90")
-	piwigoUrl      = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.")
-	piwigoUser     = flag.String("piwigoUser", "", "The username to use during sync.")
-	piwigoPassword = flag.String("piwigoPassword", "", "This is password to the given username.")
-	removeImages   = flag.Bool("removeImages", false, "If set to true, images scheduled to delete will be removed from the piwigo server. Be sure you want to delete images before enabling this flag.")
+	imagesRootPath  = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.")
+	sqliteDb        = flag.String("sqliteDb", "./localstate.db", "The connection string to the sql lite database file.")
+	noUpload        = flag.Bool("noUpload", false, "If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90")
+	piwigoUrl       = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.")
+	piwigoUser      = flag.String("piwigoUser", "", "The username to use during sync.")
+	piwigoPassword  = flag.String("piwigoPassword", "", "This is password to the given username.")
+	removeImages    = flag.Bool("removeImages", false, "If set to true, images scheduled to delete will be removed from the piwigo server. Be sure you want to delete images before enabling this flag.")
+	parallelUploads = flag.Int("parallelUploads", 4, "Set the number of images that get uploaded in parallel.")
 )
 
 func Run() {
@@ -65,7 +66,7 @@ func Run() {
 	}
 
 	if !(*noUpload) {
-		err = images.UploadImages(context.piwigo, context.dataStore)
+		err = images.UploadImages(context.piwigo, context.dataStore, *parallelUploads)
 		if err != nil {
 			logErrorAndExit(err, 8)
 		}

internal/pkg/images/upload.go

@@ -9,11 +9,12 @@ import (
 	"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore"
 	"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
 	"github.com/sirupsen/logrus"
+	"sync"
 )
 
 // Uploads the pending images to the piwigo gallery and assign the category of to the image.
 // Update local metadata and set upload flag to false. Also updates the piwigo image id if there was a difference.
-func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider) error {
+func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider, numberOfWorkers int) error {
 	logrus.Debug("Starting uploadImages")
 	defer logrus.Debug("Finished uploadImages successfully")
 
@@ -22,30 +23,65 @@ func UploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.Im
 		return err
 	}
 
-	logrus.Infof("Uploading %d images to piwigo", len(images))
+	if len(images) == 0 {
+		logrus.Info("No images to upload.")
+		return nil
+	}
 
-	for _, img := range images {
+	if numberOfWorkers <= 0 {
+		logrus.Warnf("Invalid numbers of worker set: %d falling back to default of 4", numberOfWorkers)
+		numberOfWorkers = 4
+	}
+
+	logrus.Infof("Uploading %d images to piwigo using %d workers", len(images), numberOfWorkers)
+	workQueue := make(chan datastore.ImageMetaData, numberOfWorkers)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go uploadQueueProducer(images, workQueue, &wg)
+
+	for i := 0; i < numberOfWorkers; i++ {
+		logrus.Debugf("Starting image upload worker %d", i)
+		wg.Add(1)
+		go uploadQueueWorker(workQueue, piwigoCtx, metadataProvider, &wg)
+	}
+
+	wg.Wait()
+	return nil
+}
+
+func uploadQueueWorker(workQueue <-chan datastore.ImageMetaData, piwigoCtx piwigo.PiwigoImageApi, metadataProvider datastore.ImageMetadataProvider, waitGroup *sync.WaitGroup) {
+	for img := range workQueue {
+		logrus.Debugf("%s: uploading image to piwigo", img.FullImagePath)
 
 		imgId, err := piwigoCtx.UploadImage(img.PiwigoId, img.FullImagePath, img.Md5Sum, img.CategoryPiwigoId)
 		if err != nil {
-			logrus.Warnf("could not upload image %s. Continuing with the next image.", img.FullImagePath)
+			logrus.Warnf("%s: could not upload image. Continuing with the next image.", img.FullImagePath)
 			continue
 		}
 
 		if imgId > 0 && imgId != img.PiwigoId {
 			img.PiwigoId = imgId
-			logrus.Debugf("Updating image %d with piwigo id %d", img.ImageId, img.PiwigoId)
+			logrus.Debugf("%s: Updating image %d with piwigo id %d", img.FullImagePath, img.ImageId, img.PiwigoId)
 		}
-
-		logrus.Infof("Successfully uploaded %s", img.FullImagePath)
+		logrus.Infof("%s: Successfully uploaded", img.FullImagePath)
 
 		img.UploadRequired = false
 		err = metadataProvider.SaveImageMetadata(img)
 		if err != nil {
-			logrus.Warnf("could not save uploaded image %s. Continuing with the next image.", img.FullImagePath)
+			logrus.Warnf("%s: could not save uploaded image. Continuing with the next image.", img.FullImagePath)
 			continue
 		}
 	}
-
-	return nil
+	waitGroup.Done()
+}
+
+func uploadQueueProducer(imagesToUpload []datastore.ImageMetaData, workQueue chan<- datastore.ImageMetaData, waitGroup *sync.WaitGroup) {
+	for _, img := range imagesToUpload {
+		logrus.Debugf("%s: Adding image to queue", img.FullImagePath)
+		workQueue <- img
+	}
+	waitGroup.Done()
+	close(workQueue)
 }

internal/pkg/images/upload_test.go

@@ -29,7 +29,7 @@ func Test_uploadImages_saves_new_id_to_db(t *testing.T) {
 	piwigomock := NewMockPiwigoImageApi(mockCtrl)
 	piwigomock.EXPECT().UploadImage(0, "/nonexisting/file.jpg", "1234", 2).Times(1).Return(5, nil)
 
-	err := UploadImages(piwigomock, dbmock)
+	err := UploadImages(piwigomock, dbmock, 1)
 	if err != nil {
 		t.Error(err)
 	}
@@ -52,7 +52,7 @@ func Test_uploadImages_saves_same_id_to_db(t *testing.T) {
 	piwigomock := NewMockPiwigoImageApi(mockCtrl)
 	piwigomock.EXPECT().UploadImage(5, "/nonexisting/file.jpg", "1234", 2).Times(1).Return(5, nil)
 
-	err := UploadImages(piwigomock, dbmock)
+	err := UploadImages(piwigomock, dbmock, 1)
 	if err != nil {
 		t.Error(err)
 	}