made checksum calculator exportable to use it in the app package

This commit is contained in:
Philipp Häfelfinger 2019-03-17 22:58:49 +01:00
parent 10ffa029f8
commit 02f59c47f9
3 changed files with 3 additions and 118 deletions

View File

@ -8,7 +8,7 @@ import (
"os" "os"
) )
func calculateFileCheckSums(filePath string) (string, error) { func CalculateFileCheckSums(filePath string) (string, error) {
file, err := os.Open(filePath) file, err := os.Open(filePath)
if err != nil { if err != nil {
logrus.Errorf("Could not open file %s", filePath) logrus.Errorf("Could not open file %s", filePath)

View File

@ -9,7 +9,7 @@ import (
func TestCalculateFileCheckSumsWithValidFile(t *testing.T) { func TestCalculateFileCheckSumsWithValidFile(t *testing.T) {
expectedSum := "2e7c66bd6657b1a8659ba05af26a0f7e" expectedSum := "2e7c66bd6657b1a8659ba05af26a0f7e"
sum, err := calculateFileCheckSums("../../../test/md5testfile.txt") sum, err := CalculateFileCheckSums("../../../test/md5testfile.txt")
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -23,7 +23,7 @@ func TestCalculateFileCheckSumsWithWrongPath(t *testing.T) {
hook := test.NewGlobal() hook := test.NewGlobal()
hook.Reset() hook.Reset()
sum, err := calculateFileCheckSums("unknownPath") sum, err := CalculateFileCheckSums("unknownPath")
if err == nil { if err == nil {
t.Error("there was no error using an invalid and unknown path.") t.Error("there was no error using an invalid and unknown path.")
} }

View File

@ -1,115 +0,0 @@
package localFileStructure
import (
"github.com/sirupsen/logrus"
"path/filepath"
"runtime"
"sync"
"time"
)
type ImageNode struct {
Path string
CategoryName string
ModTime time.Time
Md5Sum string
ImageId int
}
func GetImageList(fileSystem map[string]*FilesystemNode) ([]*ImageNode, error) {
logrus.Debugln("Starting GetImageList to prepare local image metadata.")
imageFiles := make([]*ImageNode, 0, len(fileSystem))
finished := make(chan bool, 1)
errChannel := make(chan error, 1)
queue := make(chan *FilesystemNode, 100)
results := make(chan *ImageNode, 200)
waitGroup := sync.WaitGroup{}
go resultCollector(results, &imageFiles)
waitGroup.Add(1)
go queueProducer(fileSystem, queue, &waitGroup)
numberOfCPUs := runtime.NumCPU()
for i := 0; i < numberOfCPUs; i++ {
logrus.Tracef("Starting getImageNodeWorker number %d", i)
waitGroup.Add(1)
go getImageNodeWorker(queue, results, errChannel, &waitGroup)
}
go func() {
waitGroup.Wait()
logrus.Debugln("All workers finished processing, closing channels.")
close(results)
close(finished)
}()
select {
case <-finished:
case err := <-errChannel:
if err != nil {
logrus.Errorf("Error during local image processing: %s", err)
return nil, err
}
}
logrus.Infof("Found %d local images to process", len(imageFiles))
return imageFiles, nil
}
func queueProducer(fileSystem map[string]*FilesystemNode, queue chan *FilesystemNode, waitGroup *sync.WaitGroup) {
logrus.Debugln("Starting queueProducer to fill the queue of the files to check and calculate the checksum")
for _, file := range fileSystem {
if file.IsDir {
continue
}
queue <- file
}
// after the last item is in the queue, we close it as there will be no more and we like
// the workers to exit.
close(queue)
logrus.Debugln("Finished queueProducer")
waitGroup.Done()
}
func resultCollector(results chan *ImageNode, imageFiles *[]*ImageNode) {
logrus.Debugln("Starting image node result collector")
for imageNode := range results {
logrus.Debugf("Local Image prepared - %s - %s - %s", imageNode.Md5Sum, imageNode.ModTime.Format(time.RFC3339), imageNode.Path)
*imageFiles = append(*imageFiles, imageNode)
}
logrus.Debugln("Finished resultCollector")
}
func getImageNodeWorker(queue chan *FilesystemNode, results chan *ImageNode, errChannel chan error, waitGroup *sync.WaitGroup) {
logrus.Debugln("Starting image file worker to gather local image informations")
for file := range queue {
md5sum, err := calculateFileCheckSums(file.Path)
if err != nil {
errChannel <- err
// we try the next image in the queue, as this might be just one error
continue
}
imageNode := &ImageNode{
Path: file.Path,
CategoryName: filepath.Dir(file.Key),
ModTime: file.ModTime,
Md5Sum: md5sum,
}
results <- imageNode
}
logrus.Debugln("Finished getImageNodeWorker")
waitGroup.Done()
}