made checksum calculator exportable to use it in the app package
This commit is contained in:
parent
10ffa029f8
commit
02f59c47f9
@ -8,7 +8,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
)
|
)
|
||||||
|
|
||||||
func calculateFileCheckSums(filePath string) (string, error) {
|
func CalculateFileCheckSums(filePath string) (string, error) {
|
||||||
file, err := os.Open(filePath)
|
file, err := os.Open(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Errorf("Could not open file %s", filePath)
|
logrus.Errorf("Could not open file %s", filePath)
|
||||||
|
@ -9,7 +9,7 @@ import (
|
|||||||
func TestCalculateFileCheckSumsWithValidFile(t *testing.T) {
|
func TestCalculateFileCheckSumsWithValidFile(t *testing.T) {
|
||||||
expectedSum := "2e7c66bd6657b1a8659ba05af26a0f7e"
|
expectedSum := "2e7c66bd6657b1a8659ba05af26a0f7e"
|
||||||
|
|
||||||
sum, err := calculateFileCheckSums("../../../test/md5testfile.txt")
|
sum, err := CalculateFileCheckSums("../../../test/md5testfile.txt")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
@ -23,7 +23,7 @@ func TestCalculateFileCheckSumsWithWrongPath(t *testing.T) {
|
|||||||
hook := test.NewGlobal()
|
hook := test.NewGlobal()
|
||||||
hook.Reset()
|
hook.Reset()
|
||||||
|
|
||||||
sum, err := calculateFileCheckSums("unknownPath")
|
sum, err := CalculateFileCheckSums("unknownPath")
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Error("there was no error using an invalid and unknown path.")
|
t.Error("there was no error using an invalid and unknown path.")
|
||||||
}
|
}
|
||||||
|
@ -1,115 +0,0 @@
|
|||||||
package localFileStructure
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type ImageNode struct {
|
|
||||||
Path string
|
|
||||||
CategoryName string
|
|
||||||
ModTime time.Time
|
|
||||||
Md5Sum string
|
|
||||||
ImageId int
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetImageList(fileSystem map[string]*FilesystemNode) ([]*ImageNode, error) {
|
|
||||||
logrus.Debugln("Starting GetImageList to prepare local image metadata.")
|
|
||||||
|
|
||||||
imageFiles := make([]*ImageNode, 0, len(fileSystem))
|
|
||||||
|
|
||||||
finished := make(chan bool, 1)
|
|
||||||
errChannel := make(chan error, 1)
|
|
||||||
queue := make(chan *FilesystemNode, 100)
|
|
||||||
results := make(chan *ImageNode, 200)
|
|
||||||
waitGroup := sync.WaitGroup{}
|
|
||||||
|
|
||||||
go resultCollector(results, &imageFiles)
|
|
||||||
|
|
||||||
waitGroup.Add(1)
|
|
||||||
go queueProducer(fileSystem, queue, &waitGroup)
|
|
||||||
|
|
||||||
numberOfCPUs := runtime.NumCPU()
|
|
||||||
for i := 0; i < numberOfCPUs; i++ {
|
|
||||||
logrus.Tracef("Starting getImageNodeWorker number %d", i)
|
|
||||||
waitGroup.Add(1)
|
|
||||||
go getImageNodeWorker(queue, results, errChannel, &waitGroup)
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
waitGroup.Wait()
|
|
||||||
|
|
||||||
logrus.Debugln("All workers finished processing, closing channels.")
|
|
||||||
|
|
||||||
close(results)
|
|
||||||
close(finished)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-finished:
|
|
||||||
case err := <-errChannel:
|
|
||||||
if err != nil {
|
|
||||||
logrus.Errorf("Error during local image processing: %s", err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
logrus.Infof("Found %d local images to process", len(imageFiles))
|
|
||||||
|
|
||||||
return imageFiles, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func queueProducer(fileSystem map[string]*FilesystemNode, queue chan *FilesystemNode, waitGroup *sync.WaitGroup) {
|
|
||||||
logrus.Debugln("Starting queueProducer to fill the queue of the files to check and calculate the checksum")
|
|
||||||
for _, file := range fileSystem {
|
|
||||||
if file.IsDir {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
queue <- file
|
|
||||||
}
|
|
||||||
|
|
||||||
// after the last item is in the queue, we close it as there will be no more and we like
|
|
||||||
// the workers to exit.
|
|
||||||
close(queue)
|
|
||||||
|
|
||||||
logrus.Debugln("Finished queueProducer")
|
|
||||||
|
|
||||||
waitGroup.Done()
|
|
||||||
}
|
|
||||||
|
|
||||||
func resultCollector(results chan *ImageNode, imageFiles *[]*ImageNode) {
|
|
||||||
logrus.Debugln("Starting image node result collector")
|
|
||||||
for imageNode := range results {
|
|
||||||
logrus.Debugf("Local Image prepared - %s - %s - %s", imageNode.Md5Sum, imageNode.ModTime.Format(time.RFC3339), imageNode.Path)
|
|
||||||
*imageFiles = append(*imageFiles, imageNode)
|
|
||||||
}
|
|
||||||
logrus.Debugln("Finished resultCollector")
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func getImageNodeWorker(queue chan *FilesystemNode, results chan *ImageNode, errChannel chan error, waitGroup *sync.WaitGroup) {
|
|
||||||
logrus.Debugln("Starting image file worker to gather local image informations")
|
|
||||||
for file := range queue {
|
|
||||||
md5sum, err := calculateFileCheckSums(file.Path)
|
|
||||||
if err != nil {
|
|
||||||
errChannel <- err
|
|
||||||
// we try the next image in the queue, as this might be just one error
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
imageNode := &ImageNode{
|
|
||||||
Path: file.Path,
|
|
||||||
CategoryName: filepath.Dir(file.Key),
|
|
||||||
ModTime: file.ModTime,
|
|
||||||
Md5Sum: md5sum,
|
|
||||||
}
|
|
||||||
|
|
||||||
results <- imageNode
|
|
||||||
}
|
|
||||||
|
|
||||||
logrus.Debugln("Finished getImageNodeWorker")
|
|
||||||
waitGroup.Done()
|
|
||||||
}
|
|
Loading…
Reference in New Issue
Block a user