From 8068d829fe2a8157d3adea0c98aee033632db135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20H=C3=A4felfinger?= Date: Wed, 20 Mar 2019 00:14:10 +0100 Subject: [PATCH] Refactored piwigo context to a object oriented implementation Optimized http calls to reuse more code and DRY Interface for PiwigoContext WIP Naming WIP --- internal/app/app.go | 5 +- internal/app/category.go | 4 +- internal/app/datastore.go | 12 +- internal/app/datastore_test.go | 4 +- internal/app/images.go | 8 +- internal/app/images_test.go | 3 +- internal/pkg/piwigo/PiwigoContext.go | 217 +++++++++++++++++++++++--- internal/pkg/piwigo/authentication.go | 126 --------------- internal/pkg/piwigo/category.go | 100 ------------ internal/pkg/piwigo/picture.go | 159 ++----------------- internal/pkg/piwigo/responses.go | 127 +++++++++++++++ 11 files changed, 351 insertions(+), 414 deletions(-) delete mode 100644 internal/pkg/piwigo/authentication.go create mode 100644 internal/pkg/piwigo/responses.go diff --git a/internal/app/app.go b/internal/app/app.go index 3effcb2..a21525b 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -3,7 +3,6 @@ package app import ( "flag" "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/localFileStructure" - "git.haefelfinger.net/piwigo/PiwigoDirectoryUploader/internal/pkg/piwigo" "github.com/sirupsen/logrus" "os" ) @@ -23,7 +22,7 @@ func Run() { logErrorAndExit(err, 1) } - err = context.piwigo.LoginToPiwigoAndConfigureContext() + err = context.piwigo.Login() if err != nil { logErrorAndExit(err, 2) } @@ -58,7 +57,7 @@ func Run() { // logErrorAndExit(err, 8) //} - _ = piwigo.Logout(context.piwigo) + _ = context.piwigo.Logout() } func logErrorAndExit(err error, exitCode int) { diff --git a/internal/app/category.go b/internal/app/category.go index fba633f..b350094 100644 --- a/internal/app/category.go +++ b/internal/app/category.go @@ -12,7 +12,7 @@ import ( func getAllCategoriesFromServer(context *appContext) (map[string]*piwigo.PiwigoCategory, error) { logrus.Debugln("Starting GetAllCategories") - categories, err := piwigo.GetAllCategories(context.piwigo) + categories, err := context.piwigo.GetAllCategories() return categories, err } @@ -66,7 +66,7 @@ func createMissingCategories(context *appContext, missingCategories []string, ex } // create category on piwigo - id, err := piwigo.CreateCategory(context.piwigo, parentId, name) + id, err := context.piwigo.CreateCategory(parentId, name) if err != nil { return errors.New(fmt.Sprintf("Could not create category on piwigo: %s", err)) } diff --git a/internal/app/datastore.go b/internal/app/datastore.go index 2221adb..ca49d9a 100644 --- a/internal/app/datastore.go +++ b/internal/app/datastore.go @@ -29,7 +29,7 @@ func (img *ImageMetaData) String() string { type ImageMetadataProvider interface { ImageMetadata(relativePath string) (ImageMetaData, error) - ImageMetadataToUpload() ([]*ImageMetaData, error) + ImageMetadataToUpload() ([]ImageMetaData, error) SaveImageMetadata(m ImageMetaData) error SavePiwigoIdAndUpdateUploadFlag(md5Sum string, piwigoId int) error } @@ -90,7 +90,7 @@ func (d *localDataStore) ImageMetadata(relativePath string) (ImageMetaData, erro return img, err } -func (d *localDataStore) ImageMetadataToUpload() ([]*ImageMetaData, error) { +func (d *localDataStore) ImageMetadataToUpload() ([]ImageMetaData, error) { logrus.Tracef("Query all image metadata that represent files queued to upload") db, err := d.openDatabase() @@ -105,14 +105,14 @@ func (d *localDataStore) ImageMetadataToUpload() ([]*ImageMetaData, error) { } defer rows.Close() - images := []*ImageMetaData{} + images := []ImageMetaData{} for rows.Next() { img := &ImageMetaData{} err = ReadImageMetadataFromRow(rows, img) if err != nil { return nil, err } - images = append(images, img) + images = append(images, *img) } err = rows.Err() @@ -233,10 +233,10 @@ func (d *localDataStore) createTablesIfNeeded(db *sql.DB) error { } func (d *localDataStore) updateImageMetaData(tx *sql.Tx, data ImageMetaData) error { - stmt, err := tx.Prepare("UPDATE image SET piwigoId = ?, relativePath = ?, fileName = ?, md5sum = ?, lastChanged = ?, categoryPath = ?, categoryId = ? WHERE imageId = ?") + stmt, err := tx.Prepare("UPDATE image SET piwigoId = ?, relativePath = ?, fileName = ?, md5sum = ?, lastChanged = ?, categoryPath = ?, categoryId = ?, uploadRequired = ? WHERE imageId = ?") if err != nil { return err } - _, err = stmt.Exec(data.PiwigoId, data.RelativeImagePath, data.Filename, data.Md5Sum, data.LastChange, data.CategoryPath, data.CategoryId, data.ImageId) + _, err = stmt.Exec(data.PiwigoId, data.RelativeImagePath, data.Filename, data.Md5Sum, data.LastChange, data.CategoryPath, data.CategoryId, data.UploadRequired, data.ImageId) return err } diff --git a/internal/app/datastore_test.go b/internal/app/datastore_test.go index 0d3f4ab..471f2d7 100644 --- a/internal/app/datastore_test.go +++ b/internal/app/datastore_test.go @@ -57,12 +57,12 @@ func TestSaveAndQueryForUploadRecords(t *testing.T) { t.Fatalf("Could not query images to upload! %s", err) } - if len(images)<1 { + if len(images) < 1 { t.Fatal("Did not get any saved images to upload!") } imgLoad := images[0] - EnsureMetadataAreEqual("toupload", img, *imgLoad, t) + EnsureMetadataAreEqual("toupload", img, imgLoad, t) cleanupDatabase(t) } diff --git a/internal/app/images.go b/internal/app/images.go index a90bb8d..bbeb8d4 100644 --- a/internal/app/images.go +++ b/internal/app/images.go @@ -90,7 +90,7 @@ func checkPiwigoForChangedImages(provider ImageMetadataProvider, piwigoCtx *piwi if img.PiwigoId == 0 { continue } - state, err := piwigo.ImageCheckFile(piwigoCtx, img.PiwigoId, img.Md5Sum) + state, err := piwigoCtx.ImageCheckFile(img.PiwigoId, img.Md5Sum) if err != nil { logrus.Warnf("Error during file change check of file %s", img.RelativeImagePath) continue @@ -99,7 +99,7 @@ func checkPiwigoForChangedImages(provider ImageMetadataProvider, piwigoCtx *piwi if state == piwigo.ImageStateUptodate { logrus.Debugf("File %s - %d has not changed", img.RelativeImagePath, img.PiwigoId) img.UploadRequired = false - err = provider.SaveImageMetadata(*img) + err = provider.SaveImageMetadata(img) if err != nil { logrus.Warnf("Could not save image data of image %s", img.RelativeImagePath) } @@ -111,7 +111,7 @@ func checkPiwigoForChangedImages(provider ImageMetadataProvider, piwigoCtx *piwi // This function calls piwigo and checks if the given md5sum is already present. // Only files without a piwigo id are used to query the server. -func updatePiwigoIdIfAlreadyUploaded(provider ImageMetadataProvider, piwiCtx *piwigo.PiwigoContext) error { +func updatePiwigoIdIfAlreadyUploaded(provider ImageMetadataProvider, piwigoCtx *piwigo.PiwigoContext) error { logrus.Infof("checking for pending files that are already on piwigo and updating piwigoids...") images, err := provider.ImageMetadataToUpload() if err != nil { @@ -125,7 +125,7 @@ func updatePiwigoIdIfAlreadyUploaded(provider ImageMetadataProvider, piwiCtx *pi files = append(files, img.Md5Sum) } } - missingResults, err := piwigo.ImagesExistOnPiwigo(piwiCtx, files) + missingResults, err := piwigoCtx.ImagesExistOnPiwigo(files) if err != nil { return err } diff --git a/internal/app/images_test.go b/internal/app/images_test.go index f5a7e19..610691c 100644 --- a/internal/app/images_test.go +++ b/internal/app/images_test.go @@ -171,7 +171,6 @@ func TestSynchronizePiwigoMetadata(t *testing.T) { Filename: "abc.jpg", } - // execute the sync metadata based on the file system results //err := synchronizeLocalImageMetadata( db) //if err != nil { @@ -202,7 +201,7 @@ func (s *testStore) SaveImageMetadata(m ImageMetaData) error { return nil } -func (d *testStore) ImageMetadataToUpload() ([]*ImageMetaData, error) { +func (d *testStore) ImageMetadataToUpload() ([]ImageMetaData, error) { return nil, errors.New("N/A") } diff --git a/internal/pkg/piwigo/PiwigoContext.go b/internal/pkg/piwigo/PiwigoContext.go index da2763f..767be4f 100644 --- a/internal/pkg/piwigo/PiwigoContext.go +++ b/internal/pkg/piwigo/PiwigoContext.go @@ -1,19 +1,18 @@ package piwigo import ( + "encoding/json" "errors" "fmt" "github.com/sirupsen/logrus" "net/http" "net/http/cookiejar" "net/url" + "os" + "strconv" + "strings" ) -type PiwigoFormPoster interface { - getChunkSizeInKB() int - postForm(formData url.Values) (resp *http.Response, err error) -} - type PiwigoContext struct { url string username string @@ -47,29 +46,186 @@ func (context *PiwigoContext) Initialize(baseUrl string, username string, passwo return nil } -func (context *PiwigoContext) LoginToPiwigoAndConfigureContext() error { +func (context *PiwigoContext) Login() error { logrus.Infoln("Logging in to piwigo and getting chunk size configuration for uploads") - err := Login(context) + logrus.Debugf("Logging in to %s using user %s", context.url, context.username) + + if !strings.HasPrefix(context.url, "https") { + logrus.Warnf("The server url %s does not use https! Credentials are not encrypted!", context.url) + } + + formData := url.Values{} + formData.Set("method", "pwg.session.login") + formData.Set("username", context.username) + formData.Set("password", context.password) + + var loginResponse loginResponse + err := context.executePiwigoRequest(formData, &loginResponse) if err != nil { + errorMessage := fmt.Sprintf("Login failed: %d - %s", loginResponse.ErrorNumber, loginResponse.Message) + logrus.Errorln(errorMessage) + return errors.New(errorMessage) + } + + logrus.Infof("Login succeeded: %s", loginResponse.Status) + return context.initializeUploadChunkSize() +} + +func (context *PiwigoContext) Logout() error { + logrus.Debugf("Logging out from %s", context.url) + + formData := url.Values{} + formData.Set("method", "pwg.session.logout") + + var logoutResponse logoutResponse + err := context.executePiwigoRequest(formData, &logoutResponse) + if err != nil { + logrus.Errorf("Logout from %s failed", context.url) return err } - return initializeUploadChunkSize(context) + logrus.Infof("Successfully logged out from %s", context.url) + + return nil } -func (context *PiwigoContext) getChunkSizeInKB() int { - return context.chunkSizeInKB -} +func (context *PiwigoContext) GetStatus() (*getStatusResponse, error) { + logrus.Debugln("Getting current login state...") -func (context *PiwigoContext) postForm(formData url.Values) (resp *http.Response, err error) { - context.initializeCookieJarIfRequired() + formData := url.Values{} + formData.Set("method", "pwg.session.getStatus") - client := http.Client{Jar: context.cookies} - response, err := client.PostForm(context.url, formData) + var getStatusResponse getStatusResponse + err := context.executePiwigoRequest(formData, &getStatusResponse) + if err != nil { + errorMessage := fmt.Sprintln("Could not get session state from server") + logrus.Errorln(errorMessage) + return nil, errors.New(errorMessage) + } + + return &getStatusResponse, nil +} + +func (context *PiwigoContext) GetAllCategories() (map[string]*PiwigoCategory, error) { + formData := url.Values{} + formData.Set("method", "pwg.categories.getList") + formData.Set("recursive", "true") + + var getCategoryListResponse getCategoryListResponse + err := context.executePiwigoRequest(formData, &getCategoryListResponse) + if err != nil { + logrus.Errorf("Got error while loading categories: %s", err) + return nil, errors.New("Could not load categories") + } + + logrus.Infof("Successfully got all categories") + categories := buildCategoryMap(&getCategoryListResponse) + buildCategoryKeys(categories) + categoryLookups := buildLookupMap(categories) + + return categoryLookups, nil +} + +func (context *PiwigoContext) CreateCategory(parentId int, name string) (int, error) { + formData := url.Values{} + formData.Set("method", "pwg.categories.add") + formData.Set("name", name) + + // we only submit the parentid if there is one. + if parentId > 0 { + formData.Set("parent", fmt.Sprint(parentId)) + } + + var createCategoryResponse createCategoryResponse + err := context.executePiwigoRequest(formData, &createCategoryResponse) + if err != nil { + logrus.Errorln(err) + return 0, err + } + + logrus.Infof("Successfully created category %s with id %d", name, createCategoryResponse.Result.ID) + return createCategoryResponse.Result.ID, nil +} + +func (context *PiwigoContext) ImageCheckFile(piwigoId int, md5sum string) (int, error) { + formData := url.Values{} + formData.Set("method", "pwg.images.checkFiles") + formData.Set("image_id", strconv.Itoa(piwigoId)) + formData.Set("file_sum", md5sum) + + logrus.Tracef("Checking if file %s - %d needs to be uploaded", md5sum, piwigoId) + + var checkFilesResponse checkFilesResponse + err := context.executePiwigoRequest(formData, &checkFilesResponse) + if err != nil { + return ImageStateInvalid, err + } + + if checkFilesResponse.Result["file"] == "equals" { + return ImageStateUptodate, nil + } + return ImageStateDifferent, nil +} + +func (context *PiwigoContext) ImagesExistOnPiwigo(md5sums []string) (map[string]int, error) { + //TODO: make sure to split to multiple queries -> to honor max upload queries + md5sumList := strings.Join(md5sums, ",") + + formData := url.Values{} + formData.Set("method", "pwg.images.exist") + formData.Set("md5sum_list", md5sumList) + + logrus.Tracef("Looking up if files exist: %s", md5sumList) + + var imageExistResponse imageExistResponse + err := context.executePiwigoRequest(formData, &imageExistResponse) if err != nil { - logrus.Errorf("The HTTP request failed with error %s", err) return nil, err } - return response, nil + + existResults := make(map[string]int, len(imageExistResponse.Result)) + + for key, value := range imageExistResponse.Result { + if value == "" { + logrus.Tracef("Missing file with md5sum: %s", key) + existResults[key] = 0 + } else { + piwigoId, err := strconv.Atoi(value) + if err != nil { + logrus.Warnf("could not parse piwigoid of file %s", key) + continue + } + logrus.Tracef("Found piwigo id %d for md5sum %s", piwigoId, key) + existResults[key] = piwigoId + } + } + + return existResults, nil +} + +func (context *PiwigoContext) UploadImage(filePath string, md5sum string, category int) (int, error) { + if context.chunkSizeInKB <= 0 { + return 0, errors.New("Uploadchunk size is less or equal to zero. 512 is a recommendet value to begin with.") + } + + fileInfo, err := os.Stat(filePath) + if err != nil { + return 0, err + } + + fileSizeInKB := fileInfo.Size() / 1024 + logrus.Infof("Uploading %s using chunksize of %d KB and total size of %d KB", filePath, context.chunkSizeInKB, fileSizeInKB) + + err = uploadImageChunks(filePath, context, fileSizeInKB, md5sum) + if err != nil { + return 0, err + } + + imageId, err := uploadImageFinal(context, fileInfo.Name(), md5sum, category) + if err != nil { + return 0, err + } + + return imageId, nil } func (context *PiwigoContext) initializeCookieJarIfRequired() { @@ -82,8 +238,8 @@ func (context *PiwigoContext) initializeCookieJarIfRequired() { context.cookies = jar } -func initializeUploadChunkSize(context *PiwigoContext) error { - userStatus, err := GetStatus(context) +func (context *PiwigoContext) initializeUploadChunkSize() error { + userStatus, err := context.GetStatus() if err != nil { return err } @@ -91,3 +247,26 @@ func initializeUploadChunkSize(context *PiwigoContext) error { logrus.Debugf("Got chunksize of %d KB from server.", context.chunkSizeInKB) return nil } + +func (context *PiwigoContext) executePiwigoRequest(formData url.Values, decodedResponse responseStatuser) error { + context.initializeCookieJarIfRequired() + + client := http.Client{Jar: context.cookies} + response, err := client.PostForm(context.url, formData) + if err != nil { + return err + } + defer response.Body.Close() + + if err := json.NewDecoder(response.Body).Decode(decodedResponse); err != nil { + logrus.Errorln(err) + return err + } + + if decodedResponse.responseStatus() != "ok" { + errorMessage := fmt.Sprintf("Error on handling piwigo response: %s", decodedResponse) + logrus.Error(errorMessage) + return errors.New(errorMessage) + } + return nil +} diff --git a/internal/pkg/piwigo/authentication.go b/internal/pkg/piwigo/authentication.go deleted file mode 100644 index 710f557..0000000 --- a/internal/pkg/piwigo/authentication.go +++ /dev/null @@ -1,126 +0,0 @@ -package piwigo - -import ( - "encoding/json" - "errors" - "fmt" - "github.com/sirupsen/logrus" - "net/url" - "strings" -) - -type LoginResponse struct { - Status string `json:"stat"` - Result bool `json:"result"` - ErrorNumber int `json:"err"` - Message string `json:"message"` -} - -type GetStatusResponse struct { - Status string `json:"stat"` - Result struct { - Username string `json:"username"` - Status string `json:"status"` - Theme string `json:"theme"` - Language string `json:"language"` - PwgToken string `json:"pwg_token"` - Charset string `json:"charset"` - CurrentDatetime string `json:"current_datetime"` - Version string `json:"version"` - AvailableSizes []string `json:"available_sizes"` - UploadFileTypes string `json:"upload_file_types"` - UploadFormChunkSize int `json:"upload_form_chunk_size"` - } `json:"result"` -} - -type LogoutResponse struct { - Status string `json:"stat"` - Result bool `json:"result"` -} - -func Login(context *PiwigoContext) error { - logrus.Debugf("Logging in to %s using user %s", context.url, context.username) - - if !strings.HasPrefix(context.url, "https") { - logrus.Warnf("The server url %s does not use https! Credentials are not encrypted!", context.url) - } - - formData := url.Values{} - formData.Set("method", "pwg.session.login") - formData.Set("username", context.username) - formData.Set("password", context.password) - - response, err := context.postForm(formData) - if err != nil { - return err - } - defer response.Body.Close() - - var loginResponse LoginResponse - if err := json.NewDecoder(response.Body).Decode(&loginResponse); err != nil { - logrus.Errorln(err) - return err - } - - if loginResponse.Status != "ok" { - errorMessage := fmt.Sprintf("Login failed: %d - %s", loginResponse.ErrorNumber, loginResponse.Message) - logrus.Errorln(errorMessage) - return errors.New(errorMessage) - } - - logrus.Infof("Login succeeded: %s", loginResponse.Status) - return nil -} - -func Logout(context *PiwigoContext) error { - logrus.Debugf("Logging out from %s", context.url) - - formData := url.Values{} - formData.Set("method", "pwg.session.logout") - - response, err := context.postForm(formData) - if err != nil { - return err - } - defer response.Body.Close() - - var statusResponse LogoutResponse - if err := json.NewDecoder(response.Body).Decode(&statusResponse); err != nil { - logrus.Errorln(err) - } - - if statusResponse.Status != "ok" { - logrus.Errorf("Logout from %s failed", context.url) - } else { - logrus.Infof("Successfully logged out from %s", context.url) - } - - return nil -} - -func GetStatus(context PiwigoFormPoster) (*GetStatusResponse, error) { - logrus.Debugln("Getting current login state...") - - formData := url.Values{} - formData.Set("method", "pwg.session.getStatus") - - response, err := context.postForm(formData) - if err != nil { - return nil, err - } - defer response.Body.Close() - - var statusResponse GetStatusResponse - if err := json.NewDecoder(response.Body).Decode(&statusResponse); err != nil { - logrus.Errorln(err) - return nil, err - } - - if statusResponse.Status != "ok" { - errorMessage := fmt.Sprintln("Could not get session state from server") - logrus.Errorln(errorMessage) - return nil, errors.New(errorMessage) - } - - return &statusResponse, nil -} diff --git a/internal/pkg/piwigo/category.go b/internal/pkg/piwigo/category.go index 2070f03..8805ff3 100644 --- a/internal/pkg/piwigo/category.go +++ b/internal/pkg/piwigo/category.go @@ -1,11 +1,8 @@ package piwigo import ( - "encoding/json" - "errors" "fmt" "github.com/sirupsen/logrus" - "net/url" "os" ) @@ -16,71 +13,6 @@ type PiwigoCategory struct { Key string } -type getCategoryListResponse struct { - Status string `json:"stat"` - Result struct { - Categories []struct { - ID int `json:"id"` - Name string `json:"name"` - Comment string `json:"comment,omitempty"` - Permalink string `json:"permalink,omitempty"` - Status string `json:"status,omitempty"` - Uppercats string `json:"uppercats,omitempty"` - GlobalRank string `json:"global_rank,omitempty"` - IDUppercat int `json:"id_uppercat,string,omitempty"` - NbImages int `json:"nb_images,omitempty"` - TotalNbImages int `json:"total_nb_images,omitempty"` - RepresentativePictureID string `json:"representative_picture_id,omitempty"` - DateLast string `json:"date_last,omitempty"` - MaxDateLast string `json:"max_date_last,omitempty"` - NbCategories int `json:"nb_categories,omitempty"` - URL string `json:"url,omitempty"` - TnURL string `json:"tn_url,omitempty"` - } `json:"categories"` - } `json:"result"` -} - -type createCategoryResponse struct { - Status string `json:"stat"` - Err int `json:"err"` - Message string `json:"message"` - Result struct { - Info string `json:"info"` - ID int `json:"id"` - } `json:"result"` -} - -func GetAllCategories(context PiwigoFormPoster) (map[string]*PiwigoCategory, error) { - formData := url.Values{} - formData.Set("method", "pwg.categories.getList") - formData.Set("recursive", "true") - - response, err := context.postForm(formData) - if err != nil { - return nil, err - } - defer response.Body.Close() - - var statusResponse getCategoryListResponse - if err := json.NewDecoder(response.Body).Decode(&statusResponse); err != nil { - logrus.Errorln(err) - return nil, err - } - - if statusResponse.Status != "ok" { - logrus.Errorf("Got state %s while loading categories", statusResponse.Status) - return nil, errors.New("Could not load categories") - } - - logrus.Infof("Successfully got all categories") - - categories := buildCategoryMap(&statusResponse) - buildCategoryKeys(categories) - categoryLookups := buildLookupMap(categories) - - return categoryLookups, nil -} - func buildLookupMap(categories map[int]*PiwigoCategory) map[string]*PiwigoCategory { categoryLookups := map[string]*PiwigoCategory{} for _, category := range categories { @@ -118,35 +50,3 @@ func buildCategoryKeys(categories map[int]*PiwigoCategory) { category.Key = key } } - -func CreateCategory(context PiwigoFormPoster, parentId int, name string) (int, error) { - formData := url.Values{} - formData.Set("method", "pwg.categories.add") - formData.Set("name", name) - - // we only submit the parentid if there is one. - if parentId > 0 { - formData.Set("parent", fmt.Sprint(parentId)) - } - - response, err := context.postForm(formData) - if err != nil { - return 0, err - } - defer response.Body.Close() - - var createResponse createCategoryResponse - if err := json.NewDecoder(response.Body).Decode(&createResponse); err != nil { - logrus.Errorln(err) - return 0, err - } - - if createResponse.Status != "ok" { - logrus.Errorf("Got state %s while loading categories", createResponse.Status) - return 0, errors.New("Could not create category") - } - - logrus.Infof("Successfully got all categories from server...") - - return createResponse.Result.ID, nil -} diff --git a/internal/pkg/piwigo/picture.go b/internal/pkg/piwigo/picture.go index d9b3a86..1072ec1 100644 --- a/internal/pkg/piwigo/picture.go +++ b/internal/pkg/piwigo/picture.go @@ -3,7 +3,6 @@ package piwigo import ( "bufio" "encoding/base64" - "encoding/json" "errors" "fmt" "github.com/sirupsen/logrus" @@ -11,135 +10,15 @@ import ( "net/url" "os" "strconv" - "strings" ) -type uploadChunkResponse struct { - Status string `json:"stat"` - Result interface{} `json:"result"` -} - -type fileAddResponse struct { - Status string `json:"stat"` - Result struct { - ImageID int `json:"image_id"` - URL string `json:"url"` - } `json:"result"` -} - -type imageExistResponse struct { - Status string `json:"stat"` - Result map[string]string `json:"result"` -} - -type checkFilesResponse struct { - Status string `json:"stat"` - Result struct { - File string `json:"file"` - } `json:"result"` -} - const ( ImageStateInvalid = -1 ImageStateUptodate = 0 ImageStateDifferent = 1 ) -func ImageCheckFile(context PiwigoFormPoster, piwigoId int, md5sum string) (int, error) { - formData := url.Values{} - formData.Set("method", "pwg.images.exist") - formData.Set("image_id", strconv.Itoa(piwigoId)) - formData.Set("file_sum", md5sum) - - logrus.Tracef("Checking if file %s - %d needs to be uploaded", md5sum, piwigoId) - - response, err := context.postForm(formData) - if err != nil { - return ImageStateInvalid, err - } - defer response.Body.Close() - - var checkFilesResponse checkFilesResponse - if err := json.NewDecoder(response.Body).Decode(&checkFilesResponse); err != nil { - logrus.Errorln(err) - return ImageStateInvalid, err - } - - if checkFilesResponse.Result.File == "equals" { - return ImageStateUptodate, nil - } - return ImageStateDifferent, nil -} - -func ImagesExistOnPiwigo(context PiwigoFormPoster, md5sums []string) (map[string]int, error) { - //TODO: make sure to split to multiple queries -> to honor max upload queries - md5sumList := strings.Join(md5sums, ",") - - formData := url.Values{} - formData.Set("method", "pwg.images.exist") - formData.Set("md5sum_list", md5sumList) - - logrus.Tracef("Looking up if files exist: %s", md5sumList) - - response, err := context.postForm(formData) - if err != nil { - return nil, err - } - defer response.Body.Close() - - var imageExistResponse imageExistResponse - if err := json.NewDecoder(response.Body).Decode(&imageExistResponse); err != nil { - logrus.Errorln(err) - return nil, err - } - - existResults := make(map[string]int, len(imageExistResponse.Result)) - - for key, value := range imageExistResponse.Result { - if value == "" { - logrus.Tracef("Missing file with md5sum: %s", key) - existResults[key] = 0 - } else { - piwigoId, err := strconv.Atoi(value) - if err != nil { - logrus.Warnf("could not parse piwigoid of file %s", key) - continue - } - logrus.Tracef("Found piwigo id %d for md5sum %s", piwigoId, key) - existResults[key] = piwigoId - } - } - - return existResults, nil -} - -func UploadImage(context PiwigoFormPoster, filePath string, md5sum string, category int) (int, error) { - if context.getChunkSizeInKB() <= 0 { - return 0, errors.New("Uploadchunk size is less or equal to zero. 512 is a recommendet value to begin with.") - } - - fileInfo, err := os.Stat(filePath) - if err != nil { - return 0, err - } - - fileSizeInKB := fileInfo.Size() / 1024 - logrus.Infof("Uploading %s using chunksize of %d KB and total size of %d KB", filePath, context.getChunkSizeInKB(), fileSizeInKB) - - err = uploadImageChunks(filePath, context, fileSizeInKB, md5sum) - if err != nil { - return 0, err - } - - imageId, err := uploadImageFinal(context, fileInfo.Name(), md5sum, category) - if err != nil { - return 0, err - } - - return imageId, nil -} - -func uploadImageChunks(filePath string, context PiwigoFormPoster, fileSizeInKB int64, md5sum string) error { +func uploadImageChunks(filePath string, context *PiwigoContext, fileSizeInKB int64, md5sum string) error { file, err := os.Open(filePath) if err != nil { return err @@ -147,9 +26,9 @@ func uploadImageChunks(filePath string, context PiwigoFormPoster, fileSizeInKB i defer file.Close() reader := bufio.NewReader(file) - bufferSize := 1024 * context.getChunkSizeInKB() + bufferSize := 1024 * context.chunkSizeInKB buffer := make([]byte, bufferSize) - numberOfChunks := (fileSizeInKB / int64(context.getChunkSizeInKB())) + 1 + numberOfChunks := (fileSizeInKB / int64(context.chunkSizeInKB)) + 1 currentChunk := int64(0) for { @@ -176,7 +55,7 @@ func uploadImageChunks(filePath string, context PiwigoFormPoster, fileSizeInKB i return nil } -func uploadImageChunk(context PiwigoFormPoster, base64chunk string, md5sum string, position int64) error { +func uploadImageChunk(context *PiwigoContext, base64chunk string, md5sum string, position int64) error { formData := url.Values{} formData.Set("method", "pwg.images.addChunk") formData.Set("data", base64chunk) @@ -187,19 +66,9 @@ func uploadImageChunk(context PiwigoFormPoster, base64chunk string, md5sum strin logrus.Tracef("Uploading chunk %d of file with sum %s", position, md5sum) - response, err := context.postForm(formData) - if err != nil { - return err - } - defer response.Body.Close() - var uploadChunkResponse uploadChunkResponse - if err := json.NewDecoder(response.Body).Decode(&uploadChunkResponse); err != nil { - logrus.Errorln(err) - return err - } - - if uploadChunkResponse.Status != "ok" { + err := context.executePiwigoRequest(formData, &uploadChunkResponse) + if err != nil { logrus.Errorf("Got state %s while uploading chunk %d of %s", uploadChunkResponse.Status, position, md5sum) return errors.New(fmt.Sprintf("Got state %s while uploading chunk %d of %s", uploadChunkResponse.Status, position, md5sum)) } @@ -207,7 +76,7 @@ func uploadImageChunk(context PiwigoFormPoster, base64chunk string, md5sum strin return nil } -func uploadImageFinal(context PiwigoFormPoster, originalFilename string, md5sum string, categoryId int) (int, error) { +func uploadImageFinal(context *PiwigoContext, originalFilename string, md5sum string, categoryId int) (int, error) { formData := url.Values{} formData.Set("method", "pwg.images.add") formData.Set("original_sum", md5sum) @@ -217,19 +86,9 @@ func uploadImageFinal(context PiwigoFormPoster, originalFilename string, md5sum logrus.Debugf("Finalizing upload of file %s with sum %s to category %d", originalFilename, md5sum, categoryId) - response, err := context.postForm(formData) - if err != nil { - return 0, err - } - defer response.Body.Close() - var fileAddResponse fileAddResponse - if err := json.NewDecoder(response.Body).Decode(&fileAddResponse); err != nil { - logrus.Errorln(err) - return 0, err - } - - if fileAddResponse.Status != "ok" { + err := context.executePiwigoRequest(formData, &fileAddResponse) + if err != nil { logrus.Errorf("Got state %s while adding image %s", fileAddResponse.Status, originalFilename) return 0, errors.New(fmt.Sprintf("Got state %s while adding image %s", fileAddResponse.Status, originalFilename)) } diff --git a/internal/pkg/piwigo/responses.go b/internal/pkg/piwigo/responses.go new file mode 100644 index 0000000..533fcc5 --- /dev/null +++ b/internal/pkg/piwigo/responses.go @@ -0,0 +1,127 @@ +package piwigo + +type responseStatuser interface { + responseStatus() string +} + +type loginResponse struct { + Status string `json:"stat"` + Result bool `json:"result"` + ErrorNumber int `json:"err"` + Message string `json:"message"` +} + +func (r loginResponse) responseStatus() string { + return r.Status +} + +type getStatusResponse struct { + Status string `json:"stat"` + Result struct { + Username string `json:"username"` + Status string `json:"status"` + Theme string `json:"theme"` + Language string `json:"language"` + PwgToken string `json:"pwg_token"` + Charset string `json:"charset"` + CurrentDatetime string `json:"current_datetime"` + Version string `json:"version"` + AvailableSizes []string `json:"available_sizes"` + UploadFileTypes string `json:"upload_file_types"` + UploadFormChunkSize int `json:"upload_form_chunk_size"` + } `json:"result"` +} + +func (r getStatusResponse) responseStatus() string { + return r.Status +} + +type logoutResponse struct { + Status string `json:"stat"` + Result bool `json:"result"` +} + +func (r logoutResponse) responseStatus() string { + return r.Status +} + +type getCategoryListResponse struct { + Status string `json:"stat"` + Result struct { + Categories []struct { + ID int `json:"id"` + Name string `json:"name"` + Comment string `json:"comment,omitempty"` + Permalink string `json:"permalink,omitempty"` + Status string `json:"status,omitempty"` + Uppercats string `json:"uppercats,omitempty"` + GlobalRank string `json:"global_rank,omitempty"` + IDUppercat int `json:"id_uppercat,string,omitempty"` + NbImages int `json:"nb_images,omitempty"` + TotalNbImages int `json:"total_nb_images,omitempty"` + RepresentativePictureID string `json:"representative_picture_id,omitempty"` + DateLast string `json:"date_last,omitempty"` + MaxDateLast string `json:"max_date_last,omitempty"` + NbCategories int `json:"nb_categories,omitempty"` + URL string `json:"url,omitempty"` + TnURL string `json:"tn_url,omitempty"` + } `json:"categories"` + } `json:"result"` +} + +func (r getCategoryListResponse) responseStatus() string { + return r.Status +} + +type createCategoryResponse struct { + Status string `json:"stat"` + Err int `json:"err"` + Message string `json:"message"` + Result struct { + Info string `json:"info"` + ID int `json:"id"` + } `json:"result"` +} + +func (r createCategoryResponse) responseStatus() string { + return r.Status +} + +type uploadChunkResponse struct { + Status string `json:"stat"` + Result interface{} `json:"result"` +} + +func (r uploadChunkResponse) responseStatus() string { + return r.Status +} + +type fileAddResponse struct { + Status string `json:"stat"` + Result struct { + ImageID int `json:"image_id"` + URL string `json:"url"` + } `json:"result"` +} + +func (r fileAddResponse) responseStatus() string { + return r.Status +} + +type imageExistResponse struct { + Status string `json:"stat"` + Result map[string]string `json:"result"` +} + +func (r imageExistResponse) responseStatus() string { + return r.Status +} + +type checkFilesResponse struct { + Status string `json:"stat"` + Result map[string]string `json:"result"` +} + +func (r checkFilesResponse) responseStatus() string { + return r.Status +}