From 8ed570728e8e19355ea22d24ac40b770bd607b76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20H=C3=A4felfinger?= Date: Sat, 6 Apr 2019 22:14:17 +0200 Subject: [PATCH] refactored image handling to use new local data store --- internal/pkg/images/datastore_mock_test.go | 84 +++++++++++++++++++++- internal/pkg/images/images.go | 26 +++---- internal/pkg/images/images_test.go | 62 ++++++++++------ 3 files changed, 137 insertions(+), 35 deletions(-) diff --git a/internal/pkg/images/datastore_mock_test.go b/internal/pkg/images/datastore_mock_test.go index 4d3c5b2..5dfa324 100644 --- a/internal/pkg/images/datastore_mock_test.go +++ b/internal/pkg/images/datastore_mock_test.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore (interfaces: ImageMetadataProvider) +// Source: git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore (interfaces: ImageMetadataProvider,CategoryProvider) // Package images is a generated GoMock package. package images @@ -134,3 +134,85 @@ func (mr *MockImageMetadataProviderMockRecorder) SavePiwigoIdAndUpdateUploadFlag mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SavePiwigoIdAndUpdateUploadFlag", reflect.TypeOf((*MockImageMetadataProvider)(nil).SavePiwigoIdAndUpdateUploadFlag), arg0, arg1) } + +// MockCategoryProvider is a mock of CategoryProvider interface +type MockCategoryProvider struct { + ctrl *gomock.Controller + recorder *MockCategoryProviderMockRecorder +} + +// MockCategoryProviderMockRecorder is the mock recorder for MockCategoryProvider +type MockCategoryProviderMockRecorder struct { + mock *MockCategoryProvider +} + +// NewMockCategoryProvider creates a new mock instance +func NewMockCategoryProvider(ctrl *gomock.Controller) *MockCategoryProvider { + mock := &MockCategoryProvider{ctrl: ctrl} + mock.recorder = &MockCategoryProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockCategoryProvider) EXPECT() *MockCategoryProviderMockRecorder { + return m.recorder +} + +// GetCategoriesToCreate mocks base method +func (m *MockCategoryProvider) GetCategoriesToCreate() ([]datastore.CategoryData, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCategoriesToCreate") + ret0, _ := ret[0].([]datastore.CategoryData) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCategoriesToCreate indicates an expected call of GetCategoriesToCreate +func (mr *MockCategoryProviderMockRecorder) GetCategoriesToCreate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCategoriesToCreate", reflect.TypeOf((*MockCategoryProvider)(nil).GetCategoriesToCreate)) +} + +// GetCategoryByKey mocks base method +func (m *MockCategoryProvider) GetCategoryByKey(arg0 string) (datastore.CategoryData, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCategoryByKey", arg0) + ret0, _ := ret[0].(datastore.CategoryData) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCategoryByKey indicates an expected call of GetCategoryByKey +func (mr *MockCategoryProviderMockRecorder) GetCategoryByKey(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCategoryByKey", reflect.TypeOf((*MockCategoryProvider)(nil).GetCategoryByKey), arg0) +} + +// GetCategoryByPiwigoId mocks base method +func (m *MockCategoryProvider) GetCategoryByPiwigoId(arg0 int) (datastore.CategoryData, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCategoryByPiwigoId", arg0) + ret0, _ := ret[0].(datastore.CategoryData) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCategoryByPiwigoId indicates an expected call of GetCategoryByPiwigoId +func (mr *MockCategoryProviderMockRecorder) GetCategoryByPiwigoId(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCategoryByPiwigoId", reflect.TypeOf((*MockCategoryProvider)(nil).GetCategoryByPiwigoId), arg0) +} + +// SaveCategory mocks base method +func (m *MockCategoryProvider) SaveCategory(arg0 datastore.CategoryData) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveCategory", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveCategory indicates an expected call of SaveCategory +func (mr *MockCategoryProviderMockRecorder) SaveCategory(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveCategory", reflect.TypeOf((*MockCategoryProvider)(nil).SaveCategory), arg0) +} diff --git a/internal/pkg/images/images.go b/internal/pkg/images/images.go index b886008..7066eef 100644 --- a/internal/pkg/images/images.go +++ b/internal/pkg/images/images.go @@ -22,25 +22,25 @@ type fileChecksumCalculator func(filePath string) (string, error) // Update the local image metadata by walking through all found files and check if the modification date has changed // or if they are new to the local database. If the files is new or changed, the md5sum will be rebuilt as well. -func SynchronizeLocalImageMetadata(metadataStorage datastore.ImageMetadataProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, categories map[string]*piwigo.PiwigoCategory, checksumCalculator fileChecksumCalculator) error { +func SynchronizeLocalImageMetadata(imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, fileSystemNodes map[string]*localFileStructure.FilesystemNode, checksumCalculator fileChecksumCalculator) error { logrus.Debug("Starting SynchronizeLocalImageMetadata") defer logrus.Debug("Leaving SynchronizeLocalImageMetadata") logrus.Info("Synchronizing local image metadata database with local available images") - err := synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes, metadataStorage, categories, checksumCalculator) + err := synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes, imageDb, categoryDb, checksumCalculator) if err != nil { return err } - err = synchronizeLocalImageMetadataFindFilesToDelete(metadataStorage) + err = synchronizeLocalImageMetadataFindFilesToDelete(imageDb) if err != nil { return err } return nil } -func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*localFileStructure.FilesystemNode, metadataStorage datastore.ImageMetadataProvider, categories map[string]*piwigo.PiwigoCategory, checksumCalculator fileChecksumCalculator) error { +func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*localFileStructure.FilesystemNode, imageDb datastore.ImageMetadataProvider, categoryDb datastore.CategoryProvider, checksumCalculator fileChecksumCalculator) error { logrus.Debug("Entering synchronizeLocalImageMetadataScanNewFiles") defer logrus.Debug("Leaving synchronizeLocalImageMetadataScanNewFiles") @@ -50,7 +50,7 @@ func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*local continue } - metadata, err := metadataStorage.ImageMetadata(file.Path) + metadata, err := imageDb.ImageMetadata(file.Path) if err == datastore.ErrorRecordNotFound { logrus.Debugf("Creating new metadata entry for %s.", file.Path) metadata = datastore.ImageMetaData{} @@ -58,11 +58,11 @@ func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*local metadata.FullImagePath = file.Path metadata.CategoryPath = filepath.Dir(file.Key) - category, exist := categories[metadata.CategoryPath] - if exist { - metadata.CategoryId = category.Id + category, err := categoryDb.GetCategoryByKey(metadata.CategoryPath) + if err == nil { + metadata.CategoryId = category.PiwigoId } else { - logrus.Warnf("No category found for image %s", file.Path) + logrus.Warnf("No category found for image %s - %s", file.Path, err) } } else if err != nil { @@ -84,7 +84,7 @@ func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*local continue } - err = metadataStorage.SaveImageMetadata(metadata) + err = imageDb.SaveImageMetadata(metadata) if err != nil { return err } @@ -92,11 +92,11 @@ func synchronizeLocalImageMetadataScanNewFiles(fileSystemNodes map[string]*local return nil } -func synchronizeLocalImageMetadataFindFilesToDelete(provider datastore.ImageMetadataProvider) error { +func synchronizeLocalImageMetadataFindFilesToDelete(imageDb datastore.ImageMetadataProvider) error { logrus.Debug("Entering SynchronizeLocalImageMetadataFindFilesToDelete") defer logrus.Debug("Leaving SynchronizeLocalImageMetadataFindFilesToDelete") - images, err := provider.ImageMetadataAll() + images, err := imageDb.ImageMetadataAll() if err != nil { return err } @@ -105,7 +105,7 @@ func synchronizeLocalImageMetadataFindFilesToDelete(provider datastore.ImageMeta if _, err := os.Stat(img.FullImagePath); os.IsNotExist(err) { img.UploadRequired = false img.DeleteRequired = true - err := provider.SaveImageMetadata(img) + err := imageDb.SaveImageMetadata(img) if err != nil { return err } diff --git a/internal/pkg/images/images_test.go b/internal/pkg/images/images_test.go index 5ed0299..f4511eb 100644 --- a/internal/pkg/images/images_test.go +++ b/internal/pkg/images/images_test.go @@ -6,7 +6,7 @@ package images //go:generate mockgen -destination=./piwigo_mock_test.go -package=images git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo PiwigoApi,PiwigoCategoryApi,PiwigoImageApi -//go:generate mockgen -destination=./datastore_mock_test.go -package=images git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore ImageMetadataProvider +//go:generate mockgen -destination=./datastore_mock_test.go -package=images git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/datastore ImageMetadataProvider,CategoryProvider import ( "errors" @@ -19,13 +19,17 @@ import ( ) func Test_synchronize_local_image_metadata_should_find_nothing_if_empty(t *testing.T) { - categories := make(map[string]*piwigo.PiwigoCategory) - categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1} + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"} + categoryMock := NewMockCategoryProvider(mockCtrl) + categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(0) db := NewtestStore() fileSystemNodes := map[string]*localFileStructure.FilesystemNode{} - err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator) + err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator) if err != nil { t.Error(err) } @@ -36,9 +40,12 @@ func Test_synchronize_local_image_metadata_should_find_nothing_if_empty(t *testi } func Test_synchronize_local_image_metadata_should_add_new_metadata(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() - categories := make(map[string]*piwigo.PiwigoCategory) - categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1} + category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"} + categoryMock := NewMockCategoryProvider(mockCtrl) + categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(1) db := NewtestStore() @@ -53,7 +60,7 @@ func Test_synchronize_local_image_metadata_should_add_new_metadata(t *testing.T) fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode // execute the sync metadata based on the file system results - err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator) + err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator) if err != nil { t.Error(err) } @@ -81,9 +88,12 @@ func Test_synchronize_local_image_metadata_should_add_new_metadata(t *testing.T) } func Test_synchronize_local_image_metadata_should_mark_unchanged_entries_without_piwigoid_as_uploads_and_reset_deleted(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() - categories := make(map[string]*piwigo.PiwigoCategory) - categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1} + category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"} + categoryMock := NewMockCategoryProvider(mockCtrl) + categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(0) db := NewtestStore() db.savedMetadata["2019/shooting1/abc.jpg"] = datastore.ImageMetaData{ @@ -107,7 +117,7 @@ func Test_synchronize_local_image_metadata_should_mark_unchanged_entries_without fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode // execute the sync metadata based on the file system results - err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator) + err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator) if err != nil { t.Error(err) } @@ -129,9 +139,12 @@ func Test_synchronize_local_image_metadata_should_mark_unchanged_entries_without } func Test_synchronize_local_image_metadata_should_mark_changed_entries_as_uploads_and_reset_deleted(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() - categories := make(map[string]*piwigo.PiwigoCategory) - categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1} + category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"} + categoryMock := NewMockCategoryProvider(mockCtrl) + categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(0) db := NewtestStore() db.savedMetadata["2019/shooting1/abc.jpg"] = datastore.ImageMetaData{ @@ -154,7 +167,7 @@ func Test_synchronize_local_image_metadata_should_mark_changed_entries_as_upload fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode // execute the sync metadata based on the file system results - err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator) + err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator) if err != nil { t.Error(err) } @@ -176,11 +189,14 @@ func Test_synchronize_local_image_metadata_should_mark_changed_entries_as_upload } func Test_synchronize_local_image_metadata_should_not_mark_unchanged_files_to_upload_and_reset_deleted(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"} + categoryMock := NewMockCategoryProvider(mockCtrl) + categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(0) + db := NewtestStore() - - categories := make(map[string]*piwigo.PiwigoCategory) - categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1} - db.savedMetadata["2019/shooting1/abc.jpg"] = datastore.ImageMetaData{ Md5Sum: "2019/shooting1/abc.jpg", FullImagePath: "2019/shooting1/abc.jpg", @@ -202,7 +218,7 @@ func Test_synchronize_local_image_metadata_should_not_mark_unchanged_files_to_up fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode // execute the sync metadata based on the file system results - err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator) + err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator) if err != nil { t.Error(err) } @@ -221,8 +237,12 @@ func Test_synchronize_local_image_metadata_should_not_mark_unchanged_files_to_up } func Test_synchronize_local_image_metadata_should_not_process_directories(t *testing.T) { - categories := make(map[string]*piwigo.PiwigoCategory) - categories["2019/shooting1"] = &piwigo.PiwigoCategory{Id: 1} + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + category := datastore.CategoryData{CategoryId:1, Name:"shooting1", PiwigoId: 1, Key: "2019/shooting1"} + categoryMock := NewMockCategoryProvider(mockCtrl) + categoryMock.EXPECT().GetCategoryByKey(category.Key).Return(category, nil).Times(0) db := NewtestStore() @@ -237,7 +257,7 @@ func Test_synchronize_local_image_metadata_should_not_process_directories(t *tes fileSystemNodes[testFileSystemNode.Key] = testFileSystemNode // execute the sync metadata based on the file system results - err := SynchronizeLocalImageMetadata(db, fileSystemNodes, categories, testChecksumCalculator) + err := SynchronizeLocalImageMetadata(db, categoryMock, fileSystemNodes, testChecksumCalculator) if err != nil { t.Error(err) }