piwigo/PiwigoDirectoryUploader
2
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

added uploadImages

Browse Source 
added piwigo image id to upload to support file updates
added noUpload flag to do all but skip file upload
changed metadata filepath to full file path
moved category detection during local file metadata generation
This commit is contained in:
Philipp Häfelfinger 2019-03-20 23:15:41 +01:00
parent 8e3c736bf9
commit b326b0da84
7 changed files with 161 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
internal/app/app.go
View File

@ -10,6 +10,7 @@ import (
var ( var (
imagesRootPath = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.") imagesRootPath = flag.String("imagesRootPath", "", "This is the images root path that should be mirrored to piwigo.")
sqliteDb = flag.String("sqliteDb", "", "The connection string to the sql lite database file.") sqliteDb = flag.String("sqliteDb", "", "The connection string to the sql lite database file.")
noUpload = flag.Bool("noUpload", false, "If set to true, the metadata gets prepared but the upload is not called and the application is exited with code 90")
piwigoUrl = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.") piwigoUrl = flag.String("piwigoUrl", "", "The root url without tailing slash to your piwigo installation.")
piwigoUser = flag.String("piwigoUser", "", "The username to use during sync.") piwigoUser = flag.String("piwigoUser", "", "The username to use during sync.")
piwigoPassword = flag.String("piwigoPassword", "", "This is password to the given username.") piwigoPassword = flag.String("piwigoPassword", "", "This is password to the given username.")
@ -42,7 +43,7 @@ func Run() {
logErrorAndExit(err, 5) logErrorAndExit(err, 5)
} }
err = synchronizeLocalImageMetadata(context.dataStore, filesystemNodes, localFileStructure.CalculateFileCheckSums) err = synchronizeLocalImageMetadata(context.dataStore, filesystemNodes, categories, localFileStructure.CalculateFileCheckSums)
if err != nil { if err != nil {
logErrorAndExit(err, 6) logErrorAndExit(err, 6)
} }
@ -52,10 +53,16 @@ func Run() {
logErrorAndExit(err, 7) logErrorAndExit(err, 7)
} }
//err = synchronizeImages(context.piwigo, context.dataStore, categories) if *noUpload {
//if err != nil { logrus.Warnln("Skipping upload of images as flag noUpload is set to true!")
// logErrorAndExit(err, 8) _ = context.piwigo.Logout()
//} os.Exit(90)
}
err = uploadImages(context.piwigo, context.dataStore)
if err != nil {
logErrorAndExit(err, 8)
}
_ = context.piwigo.Logout() _ = context.piwigo.Logout()
} }

50
internal/app/datastore.go
View File

@ -12,23 +12,23 @@ import (
var ErrorRecordNotFound = errors.New("Record not found") var ErrorRecordNotFound = errors.New("Record not found")
type ImageMetaData struct { type ImageMetaData struct {
ImageId int ImageId int
PiwigoId int PiwigoId int
RelativeImagePath string FullImagePath string
Filename string Filename string
Md5Sum string Md5Sum string
LastChange time.Time LastChange time.Time
CategoryPath string CategoryPath string
CategoryId int CategoryId int
UploadRequired bool UploadRequired bool
} }
func (img *ImageMetaData) String() string { func (img *ImageMetaData) String() string {
return fmt.Sprintf("ImageMetaData{ImageId:%d, PiwigoId:%d, CategoryId:%d, RelPath:%s, File:%s, Md5:%s, Change:%sS, catpath:%s, UploadRequired: %t}", img.ImageId, img.PiwigoId, img.CategoryId, img.RelativeImagePath, img.Filename, img.Md5Sum, img.LastChange.String(), img.CategoryPath, img.UploadRequired) return fmt.Sprintf("ImageMetaData{ImageId:%d, PiwigoId:%d, CategoryId:%d, RelPath:%s, File:%s, Md5:%s, Change:%sS, catpath:%s, UploadRequired: %t}", img.ImageId, img.PiwigoId, img.CategoryId, img.FullImagePath, img.Filename, img.Md5Sum, img.LastChange.String(), img.CategoryPath, img.UploadRequired)
} }
type ImageMetadataProvider interface { type ImageMetadataProvider interface {
ImageMetadata(relativePath string) (ImageMetaData, error) ImageMetadata(fullImagePath string) (ImageMetaData, error)
ImageMetadataToUpload() ([]ImageMetaData, error) ImageMetadataToUpload() ([]ImageMetaData, error)
SaveImageMetadata(m ImageMetaData) error SaveImageMetadata(m ImageMetaData) error
SavePiwigoIdAndUpdateUploadFlag(md5Sum string, piwigoId int) error SavePiwigoIdAndUpdateUploadFlag(md5Sum string, piwigoId int) error
@ -56,8 +56,8 @@ func (d *localDataStore) Initialize(connectionString string) error {
return err return err
} }
func (d *localDataStore) ImageMetadata(relativePath string) (ImageMetaData, error) { func (d *localDataStore) ImageMetadata(fullImagePath string) (ImageMetaData, error) {
logrus.Tracef("Query image metadata for file %s", relativePath) logrus.Tracef("Query image metadata for file %s", fullImagePath)
img := ImageMetaData{} img := ImageMetaData{}
db, err := d.openDatabase() db, err := d.openDatabase()
@ -66,12 +66,12 @@ func (d *localDataStore) ImageMetadata(relativePath string) (ImageMetaData, erro
} }
defer db.Close() defer db.Close()
stmt, err := db.Prepare("SELECT imageId, piwigoId, relativePath, fileName, md5sum, lastChanged, categoryPath, categoryId, uploadRequired FROM image WHERE relativePath = ?") stmt, err := db.Prepare("SELECT imageId, piwigoId, fullImagePath, fileName, md5sum, lastChanged, categoryPath, categoryId, uploadRequired FROM image WHERE fullImagePath = ?")
if err != nil { if err != nil {
return img, err return img, err
} }
rows, err := stmt.Query(relativePath) rows, err := stmt.Query(fullImagePath)
if err != nil { if err != nil {
return img, err return img, err
} }
@ -99,7 +99,7 @@ func (d *localDataStore) ImageMetadataToUpload() ([]ImageMetaData, error) {
} }
defer db.Close() defer db.Close()
rows, err := db.Query("SELECT imageId, piwigoId, relativePath, fileName, md5sum, lastChanged, categoryPath, categoryId, uploadRequired FROM image WHERE uploadRequired = 1") rows, err := db.Query("SELECT imageId, piwigoId, fullImagePath, fileName, md5sum, lastChanged, categoryPath, categoryId, uploadRequired FROM image WHERE uploadRequired = 1")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -120,7 +120,7 @@ func (d *localDataStore) ImageMetadataToUpload() ([]ImageMetaData, error) {
} }
func ReadImageMetadataFromRow(rows *sql.Rows, img *ImageMetaData) error { func ReadImageMetadataFromRow(rows *sql.Rows, img *ImageMetaData) error {
err := rows.Scan(&img.ImageId, &img.PiwigoId, &img.RelativeImagePath, &img.Filename, &img.Md5Sum, &img.LastChange, &img.CategoryPath, &img.CategoryId, &img.UploadRequired) err := rows.Scan(&img.ImageId, &img.PiwigoId, &img.FullImagePath, &img.Filename, &img.Md5Sum, &img.LastChange, &img.CategoryPath, &img.CategoryId, &img.UploadRequired)
return err return err
} }
@ -144,10 +144,10 @@ func (d *localDataStore) SaveImageMetadata(img ImageMetaData) error {
} }
if err != nil { if err != nil {
logrus.Errorf("Rolling back transaction for metadata of %s", img.RelativeImagePath) logrus.Errorf("Rolling back transaction for metadata of %s", img.FullImagePath)
errTx := tx.Rollback() errTx := tx.Rollback()
if errTx != nil { if errTx != nil {
logrus.Errorf("Rollback of transaction for metadata of %s failed!", img.RelativeImagePath) logrus.Errorf("Rollback of transaction for metadata of %s failed!", img.FullImagePath)
} }
return err return err
} }
@ -198,11 +198,11 @@ func (d *localDataStore) SavePiwigoIdAndUpdateUploadFlag(md5Sum string, piwigoId
} }
func (d *localDataStore) insertImageMetaData(tx *sql.Tx, data ImageMetaData) error { func (d *localDataStore) insertImageMetaData(tx *sql.Tx, data ImageMetaData) error {
stmt, err := tx.Prepare("INSERT INTO image (piwigoId, relativePath, fileName, md5sum, lastChanged, categoryPath, categoryId, uploadRequired) VALUES (?,?,?,?,?,?,?,?)") stmt, err := tx.Prepare("INSERT INTO image (piwigoId, fullImagePath, fileName, md5sum, lastChanged, categoryPath, categoryId, uploadRequired) VALUES (?,?,?,?,?,?,?,?)")
if err != nil { if err != nil {
return err return err
} }
_, err = stmt.Exec(data.PiwigoId, data.RelativeImagePath, data.Filename, data.Md5Sum, data.LastChange, data.CategoryPath, data.CategoryId, data.UploadRequired) _, err = stmt.Exec(data.PiwigoId, data.FullImagePath, data.Filename, data.Md5Sum, data.LastChange, data.CategoryPath, data.CategoryId, data.UploadRequired)
return err return err
} }
@ -221,7 +221,7 @@ func (d *localDataStore) createTablesIfNeeded(db *sql.DB) error {
_, err := db.Exec("CREATE TABLE IF NOT EXISTS image (" + _, err := db.Exec("CREATE TABLE IF NOT EXISTS image (" +
"imageId INTEGER PRIMARY KEY," + "imageId INTEGER PRIMARY KEY," +
"piwigoId INTEGER NULL," + "piwigoId INTEGER NULL," +
"relativePath NVARCHAR(1000) NOT NULL," + "fullImagePath NVARCHAR(1000) NOT NULL," +
"fileName NVARCHAR(255) NOT NULL," + "fileName NVARCHAR(255) NOT NULL," +
"md5sum NVARCHAR(50) NOT NULL," + "md5sum NVARCHAR(50) NOT NULL," +
"lastChanged DATETIME NOT NULL," + "lastChanged DATETIME NOT NULL," +
@ -233,15 +233,15 @@ func (d *localDataStore) createTablesIfNeeded(db *sql.DB) error {
return err return err
} }
_, err = db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS UX_ImageRelativePath ON image (relativePath);") _, err = db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS UX_ImageFullImagePath ON image (fullImagePath);")
return err return err
} }
func (d *localDataStore) updateImageMetaData(tx *sql.Tx, data ImageMetaData) error { func (d *localDataStore) updateImageMetaData(tx *sql.Tx, data ImageMetaData) error {
stmt, err := tx.Prepare("UPDATE image SET piwigoId = ?, relativePath = ?, fileName = ?, md5sum = ?, lastChanged = ?, categoryPath = ?, categoryId = ?, uploadRequired = ? WHERE imageId = ?") stmt, err := tx.Prepare("UPDATE image SET piwigoId = ?, fullImagePath = ?, fileName = ?, md5sum = ?, lastChanged = ?, categoryPath = ?, categoryId = ?, uploadRequired = ? WHERE imageId = ?")
if err != nil { if err != nil {
return err return err
} }
_, err = stmt.Exec(data.PiwigoId, data.RelativeImagePath, data.Filename, data.Md5Sum, data.LastChange, data.CategoryPath, data.CategoryId, data.UploadRequired, data.ImageId) _, err = stmt.Exec(data.PiwigoId, data.FullImagePath, data.Filename, data.Md5Sum, data.LastChange, data.CategoryPath, data.CategoryId, data.UploadRequired, data.ImageId)
return err return err
} }

18
internal/app/datastore_test.go
View File

@ -103,7 +103,7 @@ func TestUniqueIndexOnRelativeFilePath(t *testing.T) {
// check if the error contains the expected column as name. If not, this indicates another problem than // check if the error contains the expected column as name. If not, this indicates another problem than
// the expected duplicated insert error. // the expected duplicated insert error.
if !strings.Contains(err.Error(), "relativePath") { if !strings.Contains(err.Error(), "fullImagePath") {
t.Errorf("Got a unexpected error on saving duplicate records: %s", err) t.Errorf("Got a unexpected error on saving duplicate records: %s", err)
} }
@ -184,14 +184,14 @@ func EnsureMetadataAreEqual(action string, img ImageMetaData, imgLoad ImageMetaD
func getExampleImageMetadata(filePath string) ImageMetaData { func getExampleImageMetadata(filePath string) ImageMetaData {
return ImageMetaData{ return ImageMetaData{
RelativeImagePath: filePath, FullImagePath: filePath,
PiwigoId: 1, PiwigoId: 1,
Md5Sum: "aabbccddeeff", Md5Sum: "aabbccddeeff",
LastChange: time.Now().UTC(), LastChange: time.Now().UTC(),
Filename: "bar.jpg", Filename: "bar.jpg",
CategoryPath: "blah/foo", CategoryPath: "blah/foo",
CategoryId: 100, CategoryId: 100,
UploadRequired: true, UploadRequired: true,
} }
} }

97
internal/app/images.go
View File

@ -15,7 +15,7 @@ type fileChecksumCalculator func(filePath string) (string, error)
// Update the local image metadata by walking through all found files and check if the modification date has changed // Update the local image metadata by walking through all found files and check if the modification date has changed
// or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well. // or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well.
func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error { func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, categories map[string]*piwigo.PiwigoCategory, checksumCalculator fileChecksumCalculator) error {
logrus.Debugf("Starting synchronizeLocalImageMetadata") logrus.Debugf("Starting synchronizeLocalImageMetadata")
logrus.Info("Synchronizing local image metadata database with local available images") logrus.Info("Synchronizing local image metadata database with local available images")
@ -25,13 +25,21 @@ func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSy
continue continue
} }
metadata, err := metadataStorage.ImageMetadata(file.Key) metadata, err := metadataStorage.ImageMetadata(file.Path)
if err == ErrorRecordNotFound { if err == ErrorRecordNotFound {
logrus.Debugf("No metadata for %s found. Creating new entry.", file.Key) logrus.Debugf("No metadata for %s found. Creating new entry.", file.Key)
metadata = ImageMetaData{} metadata = ImageMetaData{}
metadata.Filename = file.Name metadata.Filename = file.Name
metadata.RelativeImagePath = file.Key metadata.FullImagePath = file.Path
metadata.CategoryPath = filepath.Dir(file.Key) metadata.CategoryPath = filepath.Dir(file.Key)
category, exist := categories[metadata.CategoryPath]
if exist {
metadata.CategoryId = category.Id
} else {
logrus.Warnf("No category found for image %s", file.Path)
}
} else if err != nil { } else if err != nil {
logrus.Errorf("Could not get metadata due to trouble. Cancelling - %s", err) logrus.Errorf("Could not get metadata due to trouble. Cancelling - %s", err)
return err return err
@ -60,16 +68,56 @@ func synchronizeLocalImageMetadata(metadataStorage ImageMetadataProvider, fileSy
return nil return nil
} }
// This method agregates the check for files with missing piwigoids and if changed files need to be uploaded again. // Uploads the pending images to the piwigo gallery and assign the category of to the image.
func synchronizePiwigoMetadata(piwigoCtx piwigo.PiwigoImageApi, metadataStorage ImageMetadataProvider) error { // Update local metadata and set upload flag to false. Also updates the piwigo image id if there was a difference.
// TODO: check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later func uploadImages(piwigoCtx piwigo.PiwigoImageApi, metadataProvider ImageMetadataProvider) error {
logrus.Debugf("Starting synchronizePiwigoMetadata") logrus.Debugf("Starting uploadImages")
err := updatePiwigoIdIfAlreadyUploaded(metadataStorage, piwigoCtx)
images, err := metadataProvider.ImageMetadataToUpload()
if err != nil { if err != nil {
return err return err
} }
err = checkPiwigoForChangedImages(metadataStorage, piwigoCtx) logrus.Infof("Uploading %d images to piwigo", len(images))
for _, img := range images {
imgId, err := piwigoCtx.UploadImage(img.PiwigoId, img.FullImagePath, img.Md5Sum, img.CategoryId)
if err != nil {
logrus.Warnf("could not upload image %s. Continuing with the next image.", img.FullImagePath)
continue
}
if imgId > 0 && imgId != img.PiwigoId {
img.PiwigoId = imgId
logrus.Debugf("Updating image %d with piwigo id %d", img.ImageId, img.PiwigoId)
}
logrus.Infof("Successfully uploaded %s", img.FullImagePath)
img.UploadRequired = false
err = metadataProvider.SaveImageMetadata(img)
if err != nil {
logrus.Warnf("could not save uploaded image %s. Continuing with the next image.", img.FullImagePath)
continue
}
}
logrus.Debugf("Finished uploadImages successfully")
return nil
}
// This method aggregates the check for files with missing piwigoids and if changed files need to be uploaded again.
func synchronizePiwigoMetadata(piwigoCtx piwigo.PiwigoImageApi, metadataProvider ImageMetadataProvider) error {
// TODO: check if category has to be assigned (image possibly added to two albums -> only uploaded once but assigned multiple times) -> implement later
logrus.Debugf("Starting synchronizePiwigoMetadata")
err := updatePiwigoIdIfAlreadyUploaded(metadataProvider, piwigoCtx)
if err != nil {
return err
}
err = checkPiwigoForChangedImages(metadataProvider, piwigoCtx)
if err != nil { if err != nil {
return err return err
} }
@ -97,16 +145,16 @@ func checkPiwigoForChangedImages(provider ImageMetadataProvider, piwigoCtx piwig
} }
state, err := piwigoCtx.ImageCheckFile(img.PiwigoId, img.Md5Sum) state, err := piwigoCtx.ImageCheckFile(img.PiwigoId, img.Md5Sum)
if err != nil { if err != nil {
logrus.Warnf("Error during file change check of file %s", img.RelativeImagePath) logrus.Warnf("Error during file change check of file %s", img.FullImagePath)
continue continue
} }
if state == piwigo.ImageStateUptodate { if state == piwigo.ImageStateUptodate {
logrus.Debugf("File %s - %d has not changed", img.RelativeImagePath, img.PiwigoId) logrus.Debugf("File %s - %d has not changed", img.FullImagePath, img.PiwigoId)
img.UploadRequired = false img.UploadRequired = false
err = provider.SaveImageMetadata(img) err = provider.SaveImageMetadata(img)
if err != nil { if err != nil {
logrus.Warnf("Could not save image data of image %s", img.RelativeImagePath) logrus.Warnf("Could not save image data of image %s", img.FullImagePath)
} }
} }
} }
@ -149,28 +197,3 @@ func updatePiwigoIdIfAlreadyUploaded(provider ImageMetadataProvider, piwigoCtx p
} }
return nil return nil
} }
// STEP 3: Upload missing images
// - upload file in chunks
// - assign image to category
//
//func uploadImages(context *appContext, missingFiles []*localFileStructure.ImageNode, existingCategories map[string]*piwigo.PiwigoCategory) error {
//
// // We sort the files by path to populate per category and not random by file
// sort.Slice(missingFiles, func(i, j int) bool {
// return missingFiles[i].Path < missingFiles[j].Path
// })
//
// for _, file := range missingFiles {
// categoryId := existingCategories[file.CategoryName].Id
//
// imageId, err := piwigo.UploadImage(context.piwigo, file.Path, file.Md5Sum, categoryId)
// if err != nil {
// return err
// }
// file.ImageId = imageId
// }
//
// return nil
//}

71
internal/app/images_test.go
View File

@ -3,15 +3,19 @@ package app
import ( import (
"errors" "errors"
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure" "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure"
"git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo"
"testing" "testing"
"time" "time"
) )
func TestSynchronizeLocalImageMetadataShouldDoNothingIfEmpty(t *testing.T) { func TestSynchronizeLocalImageMetadataShouldDoNothingIfEmpty(t *testing.T) {
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
db := NewtestStore() db := NewtestStore()
fileSystemNodes := map[string]*localFileStructure.FilesystemNode{} fileSystemNodes := map[string]*localFileStructure.FilesystemNode{}
err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator) err := synchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -22,6 +26,10 @@ func TestSynchronizeLocalImageMetadataShouldDoNothingIfEmpty(t *testing.T) {
} }
func TestSynchronizeLocalImageMetadataShouldAddNewMetadata(t *testing.T) { func TestSynchronizeLocalImageMetadataShouldAddNewMetadata(t *testing.T) {
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
db := NewtestStore() db := NewtestStore()
testFileSystemNode := &localFileStructure.FilesystemNode{ testFileSystemNode := &localFileStructure.FilesystemNode{
@ -35,7 +43,7 @@ func TestSynchronizeLocalImageMetadataShouldAddNewMetadata(t *testing.T) {
fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
// execute the sync metadata based on the file system results // execute the sync metadata based on the file system results
err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator) err := synchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -45,8 +53,8 @@ func TestSynchronizeLocalImageMetadataShouldAddNewMetadata(t *testing.T) {
if !exist { if !exist {
t.Fatal("Could not find correct metadata!") t.Fatal("Could not find correct metadata!")
} }
if savedData.RelativeImagePath != testFileSystemNode.Key { if savedData.FullImagePath != testFileSystemNode.Key {
t.Errorf("relativeImagePath %s on db image metadata is not set to %s!", savedData.RelativeImagePath, testFileSystemNode.Key) t.Errorf("fullImagePath %s on db image metadata is not set to %s!", savedData.FullImagePath, testFileSystemNode.Key)
} }
if savedData.LastChange != testFileSystemNode.ModTime { if savedData.LastChange != testFileSystemNode.ModTime {
t.Error("lastChange on db image metadata is not set to the right date!") t.Error("lastChange on db image metadata is not set to the right date!")
@ -63,13 +71,17 @@ func TestSynchronizeLocalImageMetadataShouldAddNewMetadata(t *testing.T) {
} }
func TestSynchronizeLocalImageMetadataShouldMarkChangedEntriesAsUploads(t *testing.T) { func TestSynchronizeLocalImageMetadataShouldMarkChangedEntriesAsUploads(t *testing.T) {
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
db := NewtestStore() db := NewtestStore()
db.savedMetadata["2019/shooting1/abc.jpg"] = ImageMetaData{ db.savedMetadata["2019/shooting1/abc.jpg"] = ImageMetaData{
Md5Sum: "2019/shooting1/abc.jpg", Md5Sum: "2019/shooting1/abc.jpg",
RelativeImagePath: "2019/shooting1/abc.jpg", FullImagePath: "2019/shooting1/abc.jpg",
UploadRequired: false, UploadRequired: false,
LastChange: time.Date(2019, 01, 01, 00, 0, 0, 0, time.UTC), LastChange: time.Date(2019, 01, 01, 00, 0, 0, 0, time.UTC),
Filename: "abc.jpg", Filename: "abc.jpg",
} }
testFileSystemNode := &localFileStructure.FilesystemNode{ testFileSystemNode := &localFileStructure.FilesystemNode{
@ -83,7 +95,7 @@ func TestSynchronizeLocalImageMetadataShouldMarkChangedEntriesAsUploads(t *testi
fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
// execute the sync metadata based on the file system results // execute the sync metadata based on the file system results
err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator) err := synchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -103,12 +115,16 @@ func TestSynchronizeLocalImageMetadataShouldMarkChangedEntriesAsUploads(t *testi
func TestSynchronizeLocalImageMetadataShouldNotMarkUnchangedFilesToUpload(t *testing.T) { func TestSynchronizeLocalImageMetadataShouldNotMarkUnchangedFilesToUpload(t *testing.T) {
db := NewtestStore() db := NewtestStore()
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
db.savedMetadata["2019/shooting1/abc.jpg"] = ImageMetaData{ db.savedMetadata["2019/shooting1/abc.jpg"] = ImageMetaData{
Md5Sum: "2019/shooting1/abc.jpg", Md5Sum: "2019/shooting1/abc.jpg",
RelativeImagePath: "2019/shooting1/abc.jpg", FullImagePath: "2019/shooting1/abc.jpg",
UploadRequired: false, UploadRequired: false,
LastChange: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC), LastChange: time.Date(2019, 01, 01, 01, 0, 0, 0, time.UTC),
Filename: "abc.jpg", Filename: "abc.jpg",
} }
testFileSystemNode := &localFileStructure.FilesystemNode{ testFileSystemNode := &localFileStructure.FilesystemNode{
@ -122,7 +138,7 @@ func TestSynchronizeLocalImageMetadataShouldNotMarkUnchangedFilesToUpload(t *tes
fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
// execute the sync metadata based on the file system results // execute the sync metadata based on the file system results
err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator) err := synchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -138,6 +154,9 @@ func TestSynchronizeLocalImageMetadataShouldNotMarkUnchangedFilesToUpload(t *tes
} }
func TestSynchronizeLocalImageMetadataShouldNotProcessDirectories(t *testing.T) { func TestSynchronizeLocalImageMetadataShouldNotProcessDirectories(t *testing.T) {
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
db := NewtestStore() db := NewtestStore()
testFileSystemNode := &localFileStructure.FilesystemNode{ testFileSystemNode := &localFileStructure.FilesystemNode{
@ -151,7 +170,7 @@ func TestSynchronizeLocalImageMetadataShouldNotProcessDirectories(t *testing.T)
fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
// execute the sync metadata based on the file system results // execute the sync metadata based on the file system results
err := synchronizeLocalImageMetadata(db, fileSystemNodes, testChecksumCalculator) err := synchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -164,11 +183,11 @@ func TestSynchronizeLocalImageMetadataShouldNotProcessDirectories(t *testing.T)
func TestSynchronizePiwigoMetadata(t *testing.T) { func TestSynchronizePiwigoMetadata(t *testing.T) {
db := NewtestStore() db := NewtestStore()
db.savedMetadata["2019/shooting1/abc.jpg"] = ImageMetaData{ db.savedMetadata["2019/shooting1/abc.jpg"] = ImageMetaData{
Md5Sum: "2019/shooting1/abc.jpg", Md5Sum: "2019/shooting1/abc.jpg",
RelativeImagePath: "2019/shooting1/abc.jpg", FullImagePath: "2019/shooting1/abc.jpg",
UploadRequired: false, UploadRequired: false,
LastChange: time.Date(2019, 01, 01, 00, 0, 0, 0, time.UTC), LastChange: time.Date(2019, 01, 01, 00, 0, 0, 0, time.UTC),
Filename: "abc.jpg", Filename: "abc.jpg",
} }
// execute the sync metadata based on the file system results // execute the sync metadata based on the file system results
@ -176,7 +195,7 @@ func TestSynchronizePiwigoMetadata(t *testing.T) {
//if err != nil { //if err != nil {
// t.Error(err) // t.Error(err)
//} //}
t.FailNow() t.Skip("Not yet implemented!")
} }
// test metadata store to store save the metadat and simulate the database // test metadata store to store save the metadat and simulate the database
@ -188,8 +207,8 @@ func NewtestStore() *testStore {
return &testStore{savedMetadata: make(map[string]ImageMetaData)} return &testStore{savedMetadata: make(map[string]ImageMetaData)}
} }
func (s *testStore) ImageMetadata(relativePath string) (ImageMetaData, error) { func (s *testStore) ImageMetadata(fullImagePath string) (ImageMetaData, error) {
metadata, exist := s.savedMetadata[relativePath] metadata, exist := s.savedMetadata[fullImagePath]
if !exist { if !exist {
return ImageMetaData{}, ErrorRecordNotFound return ImageMetaData{}, ErrorRecordNotFound
} }
@ -197,7 +216,7 @@ func (s *testStore) ImageMetadata(relativePath string) (ImageMetaData, error) {
} }
func (s *testStore) SaveImageMetadata(m ImageMetaData) error { func (s *testStore) SaveImageMetadata(m ImageMetaData) error {
s.savedMetadata[m.RelativeImagePath] = m s.savedMetadata[m.FullImagePath] = m
return nil return nil
} }

6
internal/pkg/piwigo/PiwigoContext.go
View File

@ -28,7 +28,7 @@ type PiwigoCategoryApi interface {
type PiwigoImageApi interface { type PiwigoImageApi interface {
ImageCheckFile(piwigoId int, md5sum string) (int, error) ImageCheckFile(piwigoId int, md5sum string) (int, error)
ImagesExistOnPiwigo(md5sums []string) (map[string]int, error) ImagesExistOnPiwigo(md5sums []string) (map[string]int, error)
UploadImage(filePath string, md5sum string, category int) (int, error) UploadImage(piwigoId int, filePath string, md5sum string, category int) (int, error)
} }
type PiwigoContext struct { type PiwigoContext struct {
@ -220,7 +220,7 @@ func (context *PiwigoContext) ImagesExistOnPiwigo(md5sums []string) (map[string]
return existResults, nil return existResults, nil
} }
func (context *PiwigoContext) UploadImage(filePath string, md5sum string, category int) (int, error) { func (context *PiwigoContext) UploadImage(piwigoId int, filePath string, md5sum string, category int) (int, error) {
if context.chunkSizeInKB <= 0 { if context.chunkSizeInKB <= 0 {
return 0, errors.New("Uploadchunk size is less or equal to zero. 512 is a recommendet value to begin with.") return 0, errors.New("Uploadchunk size is less or equal to zero. 512 is a recommendet value to begin with.")
} }
@ -238,7 +238,7 @@ func (context *PiwigoContext) UploadImage(filePath string, md5sum string, catego
return 0, err return 0, err
} }
imageId, err := uploadImageFinal(context, fileInfo.Name(), md5sum, category) imageId, err := uploadImageFinal(context, piwigoId, fileInfo.Name(), md5sum, category)
if err != nil { if err != nil {
return 0, err return 0, err
} }

8
internal/pkg/piwigo/picture.go
View File

@ -76,7 +76,7 @@ func uploadImageChunk(context *PiwigoContext, base64chunk string, md5sum string,
return nil return nil
} }
func uploadImageFinal(context *PiwigoContext, originalFilename string, md5sum string, categoryId int) (int, error) { func uploadImageFinal(context *PiwigoContext, piwigoId int, originalFilename string, md5sum string, categoryId int) (int, error) {
formData := url.Values{} formData := url.Values{}
formData.Set("method", "pwg.images.add") formData.Set("method", "pwg.images.add")
formData.Set("original_sum", md5sum) formData.Set("original_sum", md5sum)
@ -84,6 +84,12 @@ func uploadImageFinal(context *PiwigoContext, originalFilename string, md5sum st
formData.Set("name", originalFilename) formData.Set("name", originalFilename)
formData.Set("categories", strconv.Itoa(categoryId)) formData.Set("categories", strconv.Itoa(categoryId))
// when there is a image id, we are updating an existing image and need to specify the piwigo image id.
// if we skip the image id, a new id will be generated
if piwigoId > 0 {
formData.Set("image_id", strconv.Itoa(piwigoId))
}
logrus.Debugf("Finalizing upload of file %s with sum %s to category %d", originalFilename, md5sum, categoryId) logrus.Debugf("Finalizing upload of file %s with sum %s to category %d", originalFilename, md5sum, categoryId)
var fileAddResponse fileAddResponse var fileAddResponse fileAddResponse