piwigo/PiwigoDirectoryUploader
2
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

refactored image handling to use new local data store

Browse Source
This commit is contained in:
Philipp Häfelfinger 2019-04-06 22:14:17 +02:00
parent e0bf273e35
commit 8ed570728e
3 changed files with 137 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
internal/pkg/images/datastore_mock_test.go
View File

@ -1,5 +1,5 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore (interfaces: ImageMetadataProvider)
// Source: git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore (interfaces: ImageMetadataProvider,CategoryProvider)
// Package images is a generated GoMock package.
package images
@ -134,3 +134,85 @@ func (mr *MockImageMetadataProviderMockRecorder) SavePiwigoIdAndUpdateUploadFlag
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SavePiwigoIdAndUpdateUploadFlag", reflect.TypeOf((*MockImageMetadataProvider)(nil).SavePiwigoIdAndUpdateUploadFlag), arg0, arg1)
}
// MockCategoryProvider is a mock of CategoryProvider interface
type MockCategoryProvider struct {
ctrl *gomock.Controller
recorder *MockCategoryProviderMockRecorder
}
// MockCategoryProviderMockRecorder is the mock recorder for MockCategoryProvider
type MockCategoryProviderMockRecorder struct {
mock *MockCategoryProvider
}
// NewMockCategoryProvider creates a new mock instance
func NewMockCategoryProvider(ctrl *gomock.Controller) *MockCategoryProvider {
mock := &MockCategoryProvider{ctrl: ctrl}
mock.recorder = &MockCategoryProviderMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockCategoryProvider) EXPECT() *MockCategoryProviderMockRecorder {
return m.recorder
}
// GetCategoriesToCreate mocks base method
func (m *MockCategoryProvider) GetCategoriesToCreate() ([]datastore.CategoryData, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetCategoriesToCreate")
ret0, _ := ret[0].([]datastore.CategoryData)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetCategoriesToCreate indicates an expected call of GetCategoriesToCreate
func (mr *MockCategoryProviderMockRecorder) GetCategoriesToCreate() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCategoriesToCreate", reflect.TypeOf((*MockCategoryProvider)(nil).GetCategoriesToCreate))
}
// GetCategoryByKey mocks base method
func (m *MockCategoryProvider) GetCategoryByKey(arg0 string) (datastore.CategoryData, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetCategoryByKey", arg0)
ret0, _ := ret[0].(datastore.CategoryData)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetCategoryByKey indicates an expected call of GetCategoryByKey
func (mr *MockCategoryProviderMockRecorder) GetCategoryByKey(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCategoryByKey", reflect.TypeOf((*MockCategoryProvider)(nil).GetCategoryByKey), arg0)
}
// GetCategoryByPiwigoId mocks base method
func (m *MockCategoryProvider) GetCategoryByPiwigoId(arg0 int) (datastore.CategoryData, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetCategoryByPiwigoId", arg0)
ret0, _ := ret[0].(datastore.CategoryData)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetCategoryByPiwigoId indicates an expected call of GetCategoryByPiwigoId
func (mr *MockCategoryProviderMockRecorder) GetCategoryByPiwigoId(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCategoryByPiwigoId", reflect.TypeOf((*MockCategoryProvider)(nil).GetCategoryByPiwigoId), arg0)
}
// SaveCategory mocks base method
func (m *MockCategoryProvider) SaveCategory(arg0 datastore.CategoryData) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SaveCategory", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// SaveCategory indicates an expected call of SaveCategory
func (mr *MockCategoryProviderMockRecorder) SaveCategory(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveCategory", reflect.TypeOf((*MockCategoryProvider)(nil).SaveCategory), arg0)
}

26
internal/pkg/images/images.go
View File

@ -22,25 +22,25 @@ type fileChecksumCalculator func(filePath string) (string, error)
// Update the local image metadata by walking through all found files and check if the modification date has changed
// or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well.
func SynchronizeLocalImageMetadata(metadataStorage datastore.ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, categories map[string]*piwigo.PiwigoCategory, checksumCalculator fileChecksumCalculator) error {
func SynchronizeLocalImageMetadata(imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error {
logrus.Debug("Starting SynchronizeLocalImageMetadata")
defer logrus.Debug("Leaving SynchronizeLocalImageMetadata")
logrus.Info("Synchronizing local image metadata database with local available images")
err := synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes, metadataStorage, categories, checksumCalculator)
err := synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes, imageDb, categoryDb, checksumCalculator)
if err != nil {
return err
}
err = synchronizeLocalImageMetadataFindFilesToDelete(metadataStorage)
err = synchronizeLocalImageMetadataFindFilesToDelete(imageDb)
if err != nil {
return err
}
return nil
}
func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*localFileStructure.FilesystemNode, metadataStorage datastore.ImageMetadataProvider, categories map[string]*piwigo.PiwigoCategory, checksumCalculator fileChecksumCalculator) error {
func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*localFileStructure.FilesystemNode, imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, checksumCalculator fileChecksumCalculator) error {
logrus.Debug("Entering synchronizeLocalImageMetadataScanNewFiles")
defer logrus.Debug("Leaving synchronizeLocalImageMetadataScanNewFiles")
@ -50,7 +50,7 @@ func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*local
continue
}
metadata, err := metadataStorage.ImageMetadata(file.Path)
metadata, err := imageDb.ImageMetadata(file.Path)
if err == datastore.ErrorRecordNotFound {
logrus.Debugf("Creating new metadata entry for %s.", file.Path)
metadata = datastore.ImageMetaData{}
@ -58,11 +58,11 @@ func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*local
metadata.FullImagePath = file.Path
metadata.CategoryPath = filepath.Dir(file.Key)
category, exist := categories[metadata.CategoryPath]
if exist {
metadata.CategoryId = category.Id
category, err := categoryDb.GetCategoryByKey(metadata.CategoryPath)
if err == nil {
metadata.CategoryId = category.PiwigoId
} else {
logrus.Warnf("No category found for image %s", file.Path)
logrus.Warnf("No category found for image %s - %s", file.Path, err)
}
} else if err != nil {
@ -84,7 +84,7 @@ func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*local
continue
}
err = metadataStorage.SaveImageMetadata(metadata)
err = imageDb.SaveImageMetadata(metadata)
if err != nil {
return err
}
@ -92,11 +92,11 @@ func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*local
return nil
}
func synchronizeLocalImageMetadataFindFilesToDelete(provider datastore.ImageMetadataProvider) error {
func synchronizeLocalImageMetadataFindFilesToDelete(imageDb datastore.ImageMetadataProvider) error {
logrus.Debug("Entering SynchronizeLocalImageMetadataFindFilesToDelete")
defer logrus.Debug("Leaving SynchronizeLocalImageMetadataFindFilesToDelete")
images, err := provider.ImageMetadataAll()
images, err := imageDb.ImageMetadataAll()
if err != nil {
return err
}
@ -105,7 +105,7 @@ func synchronizeLocalImageMetadataFindFilesToDelete(provider datastore.ImageMeta
if _, err := os.Stat(img.FullImagePath); os.IsNotExist(err) {
img.UploadRequired = false
img.DeleteRequired = true
err := provider.SaveImageMetadata(img)
err := imageDb.SaveImageMetadata(img)
if err != nil {
return err
}

62
internal/pkg/images/images_test.go
View File

@ -6,7 +6,7 @@
package images
//go:generate mockgen -destination=./piwigo_mock_test.go -package=images git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo PiwigoApi,PiwigoCategoryApi,PiwigoImageApi
//go:generate mockgen -destination=./datastore_mock_test.go -package=images git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore ImageMetadataProvider
//go:generate mockgen -destination=./datastore_mock_test.go -package=images git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore ImageMetadataProvider,CategoryProvider
import (
"errors"
@ -19,13 +19,17 @@ import (
)
func Test_synchronize_local_image_metadata_should_find_nothing_if_empty(t *testing.T) {
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"}
categoryMock := NewMockCategoryProvider(mockCtrl)
categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(0)
db := NewtestStore()
fileSystemNodes := map[string]*localFileStructure.FilesystemNode{}
err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator)
if err != nil {
t.Error(err)
}
@ -36,9 +40,12 @@ func Test_synchronize_local_image_metadata_should_find_nothing_if_empty(t *testi
}
func Test_synchronize_local_image_metadata_should_add_new_metadata(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"}
categoryMock := NewMockCategoryProvider(mockCtrl)
categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(1)
db := NewtestStore()
@ -53,7 +60,7 @@ func Test_synchronize_local_image_metadata_should_add_new_metadata(t *testing.T)
fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
// execute the sync metadata based on the file system results
err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator)
if err != nil {
t.Error(err)
}
@ -81,9 +88,12 @@ func Test_synchronize_local_image_metadata_should_add_new_metadata(t *testing.T)
}
func Test_synchronize_local_image_metadata_should_mark_unchanged_entries_without_piwigoid_as_uploads_and_reset_deleted(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"}
categoryMock := NewMockCategoryProvider(mockCtrl)
categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(0)
db := NewtestStore()
db.savedMetadata["2019/shooting1/abc.jpg"] = datastore.ImageMetaData{
@ -107,7 +117,7 @@ func Test_synchronize_local_image_metadata_should_mark_unchanged_entries_without
fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
// execute the sync metadata based on the file system results
err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator)
if err != nil {
t.Error(err)
}
@ -129,9 +139,12 @@ func Test_synchronize_local_image_metadata_should_mark_unchanged_entries_without
}
func Test_synchronize_local_image_metadata_should_mark_changed_entries_as_uploads_and_reset_deleted(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"}
categoryMock := NewMockCategoryProvider(mockCtrl)
categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(0)
db := NewtestStore()
db.savedMetadata["2019/shooting1/abc.jpg"] = datastore.ImageMetaData{
@ -154,7 +167,7 @@ func Test_synchronize_local_image_metadata_should_mark_changed_entries_as_upload
fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
// execute the sync metadata based on the file system results
err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator)
if err != nil {
t.Error(err)
}
@ -176,11 +189,14 @@ func Test_synchronize_local_image_metadata_should_mark_changed_entries_as_upload
}
func Test_synchronize_local_image_metadata_should_not_mark_unchanged_files_to_upload_and_reset_deleted(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"}
categoryMock := NewMockCategoryProvider(mockCtrl)
categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(0)
db := NewtestStore()
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
db.savedMetadata["2019/shooting1/abc.jpg"] = datastore.ImageMetaData{
Md5Sum: "2019/shooting1/abc.jpg",
FullImagePath: "2019/shooting1/abc.jpg",
@ -202,7 +218,7 @@ func Test_synchronize_local_image_metadata_should_not_mark_unchanged_files_to_up
fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
// execute the sync metadata based on the file system results
err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator)
if err != nil {
t.Error(err)
}
@ -221,8 +237,12 @@ func Test_synchronize_local_image_metadata_should_not_mark_unchanged_files_to_up
}
func Test_synchronize_local_image_metadata_should_not_process_directories(t *testing.T) {
categories := make(map[string]*piwigo.PiwigoCategory)
categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1}
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"}
categoryMock := NewMockCategoryProvider(mockCtrl)
categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(0)
db := NewtestStore()
@ -237,7 +257,7 @@ func Test_synchronize_local_image_metadata_should_not_process_directories(t *tes
fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode
// execute the sync metadata based on the file system results
err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator)
err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator)
if err != nil {
t.Error(err)
}