public/photoview
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/photoview.git synced 2026-08-03 19:29:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring part 1: Fix some annoying linter warnings in API code (#1064)

Browse Source 
* names, constants of reusable strings, removed unneeded `if`, replaced deprecated imports

* more deprecated imports replaced

* What is this?))

* Fix a path and quote a var in Dockerfile

* Addressing review comments

---------

Co-authored-by: Konstantin Koval <kkb@ukr.net>
This commit is contained in:
Kostiantyn
2024-09-30 15:45:02 +03:00
committed by GitHub
parent 70a3944e21
commit d1193c5ca4
38 changed files with 334 additions and 309 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
api/scanner/exif/exif_parser_external.go
View File

@@ -63,7 +63,7 @@ func sanitizeEXIF(exif *models.MediaEXIF) {
}
}
func extractValidGpsData(fileInfo *exiftool.FileMetadata, media_path string) (*float64, *float64) {
func extractValidGpsData(fileInfo *exiftool.FileMetadata, mediaPath string) (*float64, *float64) {
var GPSLat, GPSLong *float64
// GPS coordinates - longitude
@@ -82,13 +82,13 @@ func extractValidGpsData(fileInfo *exiftool.FileMetadata, media_path string) (*f
if (GPSLat != nil && math.Abs(*GPSLat) > 90) || (GPSLong != nil && math.Abs(*GPSLong) > 90) {
log.Printf(
"Incorrect GPS data in the %s Exif data: %f, %f, while expected values between '-90' and '90'. Ignoring GPS data.",
media_path, *GPSLat, *GPSLong)
mediaPath, *GPSLat, *GPSLong)
return nil, nil
}
return GPSLat, GPSLong
}
func (p *externalExifParser) ParseExif(media_path string) (returnExif *models.MediaEXIF, returnErr error) {
func (p *externalExifParser) ParseExif(mediaPath string) (returnExif *models.MediaEXIF, returnErr error) {
// ExifTool - No print conversion mode
if p.et == nil {
et, err := exiftool.NewExiftool(exiftool.NoPrintConversion())
@@ -100,39 +100,39 @@ func (p *externalExifParser) ParseExif(media_path string) (returnExif *models.Me
}
}
fileInfo, err := p.dataLoader.Load(media_path)
fileInfo, err := p.dataLoader.Load(mediaPath)
if err != nil {
return nil, err
}
newExif := models.MediaEXIF{}
found_exif := false
foundExif := false
// Get description
description, err := fileInfo.GetString("ImageDescription")
if err == nil {
found_exif = true
foundExif = true
newExif.Description = &description
}
// Get camera model
model, err := fileInfo.GetString("Model")
if err == nil {
found_exif = true
foundExif = true
newExif.Camera = &model
}
// Get Camera make
make, err := fileInfo.GetString("Make")
if err == nil {
found_exif = true
foundExif = true
newExif.Maker = &make
}
// Get lens
lens, err := fileInfo.GetString("LensModel")
if err == nil {
found_exif = true
foundExif = true
newExif.Lens = &lens
}
@@ -144,13 +144,13 @@ func (p *externalExifParser) ParseExif(media_path string) (returnExif *models.Me
layout := "2006:01:02 15:04:05"
dateTime, err := time.Parse(layout, date)
if err == nil {
found_exif = true
foundExif = true
newExif.DateShot = &dateTime
} else {
layoutWithOffset := "2006:01:02 15:04:05-07:00"
dateTime, err = time.Parse(layoutWithOffset, date)
if err == nil {
found_exif = true
foundExif = true
newExif.DateShot = &dateTime
}
}
@@ -161,59 +161,59 @@ func (p *externalExifParser) ParseExif(media_path string) (returnExif *models.Me
// Get exposure time
exposureTime, err := fileInfo.GetFloat("ExposureTime")
if err == nil {
found_exif = true
foundExif = true
newExif.Exposure = &exposureTime
}
// Get aperture
aperture, err := fileInfo.GetFloat("Aperture")
if err == nil {
found_exif = true
foundExif = true
newExif.Aperture = &aperture
}
// Get ISO
iso, err := fileInfo.GetInt("ISO")
if err == nil {
found_exif = true
foundExif = true
newExif.Iso = &iso
}
// Get focal length
focalLen, err := fileInfo.GetFloat("FocalLength")
if err == nil {
found_exif = true
foundExif = true
newExif.FocalLength = &focalLen
}
// Get flash info
flash, err := fileInfo.GetInt("Flash")
if err == nil {
found_exif = true
foundExif = true
newExif.Flash = &flash
}
// Get orientation
orientation, err := fileInfo.GetInt("Orientation")
if err == nil {
found_exif = true
foundExif = true
newExif.Orientation = &orientation
}
// Get exposure program
expProgram, err := fileInfo.GetInt("ExposureProgram")
if err == nil {
found_exif = true
foundExif = true
newExif.ExposureProgram = &expProgram
}
// Get GPS data
newExif.GPSLatitude, newExif.GPSLongitude = extractValidGpsData(&fileInfo, media_path)
newExif.GPSLatitude, newExif.GPSLongitude = extractValidGpsData(&fileInfo, mediaPath)
if (newExif.GPSLatitude != nil) && (newExif.GPSLongitude != nil) {
found_exif = true
foundExif = true
}
if !found_exif {
if !foundExif {
return nil, nil
}

81
api/scanner/exif/exif_parser_internal.go
View File

@@ -17,12 +17,17 @@ import (
// internalExifParser is an exif parser that parses the media without the use of external tools
type internalExifParser struct{}
const couldNotReadXfromEXIFy = "could not read %s from EXIF: %s"
const warnEXIFtagXreturnedNully = "WARN: EXIF tag %s returned null: %s\n"
var ErrNullExifTag = errors.New("exif tag returned null")
func NewInternalExifParser() ExifParser {
return internalExifParser{}
}
func (p internalExifParser) ParseExif(media_path string) (returnExif *models.MediaEXIF, returnErr error) {
photoFile, err := os.Open(media_path)
func (p internalExifParser) ParseExif(mediaPath string) (returnExif *models.MediaEXIF, returnErr error) {
photoFile, err := os.Open(mediaPath)
if err != nil {
return nil, err
}
@@ -45,22 +50,22 @@ func (p internalExifParser) ParseExif(media_path string) (returnExif *models.Med
newExif := models.MediaEXIF{}
description, err := p.readStringTag(exifTags, exif.ImageDescription, media_path)
description, err := p.readStringTag(exifTags, exif.ImageDescription, mediaPath)
if err == nil {
newExif.Description = description
}
model, err := p.readStringTag(exifTags, exif.Model, media_path)
model, err := p.readStringTag(exifTags, exif.Model, mediaPath)
if err == nil {
newExif.Camera = model
}
maker, err := p.readStringTag(exifTags, exif.Make, media_path)
maker, err := p.readStringTag(exifTags, exif.Make, mediaPath)
if err == nil {
newExif.Maker = maker
}
lens, err := p.readStringTag(exifTags, exif.LensModel, media_path)
lens, err := p.readStringTag(exifTags, exif.LensModel, mediaPath)
if err == nil {
newExif.Lens = lens
}
@@ -68,8 +73,8 @@ func (p internalExifParser) ParseExif(media_path string) (returnExif *models.Med
date, err := exifTags.DateTime()
if err == nil {
_, tz := date.Zone()
date_utc := date.Add(time.Duration(tz) * time.Second).UTC()
newExif.DateShot = &date_utc
dateUTC := date.Add(time.Duration(tz) * time.Second).UTC()
newExif.DateShot = &dateUTC
}
exposureTag, err := exifTags.Get(exif.ExposureTime)
@@ -81,7 +86,7 @@ func (p internalExifParser) ParseExif(media_path string) (returnExif *models.Med
}
}
apertureRat, err := p.readRationalTag(exifTags, exif.FNumber, media_path)
apertureRat, err := p.readRationalTag(exifTags, exif.FNumber, mediaPath)
if err == nil {
aperture, _ := apertureRat.Float64()
newExif.Aperture = &aperture
@@ -89,11 +94,11 @@ func (p internalExifParser) ParseExif(media_path string) (returnExif *models.Med
isoTag, err := exifTags.Get(exif.ISOSpeedRatings)
if err != nil {
log.Printf("WARN: Could not read ISOSpeedRatings from EXIF: %v\n", media_path)
log.Printf("WARN: Could not read ISOSpeedRatings from EXIF: %v\n", mediaPath)
} else {
iso, err := isoTag.Int(0)
if err != nil {
log.Printf("WARN: Could not parse EXIF ISOSpeedRatings as integer: %v\n", media_path)
log.Printf("WARN: Could not parse EXIF ISOSpeedRatings as integer: %v\n", mediaPath)
} else {
iso64 := int64(iso)
newExif.Iso = &iso64
@@ -106,36 +111,32 @@ func (p internalExifParser) ParseExif(media_path string) (returnExif *models.Med
if err == nil {
focalLength, _ := focalLengthRat.Float64()
newExif.FocalLength = &focalLength
} else {
// For some photos, the focal length cannot be read as a rational value,
// but is instead the second value read as an integer
if err == nil {
focalLength, err := focalLengthTag.Int(1)
if err != nil {
log.Printf("WARN: Could not parse EXIF FocalLength as rational or integer: %v\n%s\n", media_path, err)
} else {
focalLenFloat := float64(focalLength)
newExif.FocalLength = &focalLenFloat
}
focalLength, err := focalLengthTag.Int(1)
if err != nil {
log.Printf("WARN: Could not parse EXIF FocalLength as rational or integer: %v\n%s\n", mediaPath, err)
} else {
focalLenFloat := float64(focalLength)
newExif.FocalLength = &focalLenFloat
}
}
}
flash, err := p.readIntegerTag(exifTags, exif.Flash, media_path)
flash, err := p.readIntegerTag(exifTags, exif.Flash, mediaPath)
if err == nil {
flash64 := int64(*flash)
newExif.Flash = &flash64
}
orientation, err := p.readIntegerTag(exifTags, exif.Orientation, media_path)
orientation, err := p.readIntegerTag(exifTags, exif.Orientation, mediaPath)
if err == nil {
orientation64 := int64(*orientation)
newExif.Orientation = &orientation64
}
exposureProgram, err := p.readIntegerTag(exifTags, exif.ExposureProgram, media_path)
exposureProgram, err := p.readIntegerTag(exifTags, exif.ExposureProgram, mediaPath)
if err == nil {
exposureProgram64 := int64(*exposureProgram)
newExif.ExposureProgram = &exposureProgram64
@@ -147,7 +148,7 @@ func (p internalExifParser) ParseExif(media_path string) (returnExif *models.Med
returnExif = &newExif
log.Printf(
"Incorrect GPS data in the %s Exif data: %f, %f, while expected values between '-90' and '90'. Ignoring GPS data.",
media_path, long, lat)
mediaPath, long, lat)
return
} else {
newExif.GPSLatitude = &lat
@@ -159,59 +160,59 @@ func (p internalExifParser) ParseExif(media_path string) (returnExif *models.Med
return
}
func (p *internalExifParser) readStringTag(tags *exif.Exif, name exif.FieldName, media_path string) (*string, error) {
func (p *internalExifParser) readStringTag(tags *exif.Exif, name exif.FieldName, mediaPath string) (*string, error) {
tag, err := tags.Get(name)
if err != nil {
return nil, errors.Wrapf(err, "could not read %s from EXIF: %s", name, media_path)
return nil, errors.Wrapf(err, couldNotReadXfromEXIFy, name, mediaPath)
}
if tag != nil {
value, err := tag.StringVal()
if err != nil {
return nil, errors.Wrapf(err, "could not parse %s from EXIF as string: %s", name, media_path)
return nil, errors.Wrapf(err, "could not parse %s from EXIF as string: %s", name, mediaPath)
}
return &value, nil
}
log.Printf("WARN: EXIF tag %s returned null: %s\n", name, media_path)
return nil, errors.New("exif tag returned null")
log.Printf(warnEXIFtagXreturnedNully, name, mediaPath)
return nil, ErrNullExifTag
}
func (p *internalExifParser) readRationalTag(tags *exif.Exif, name exif.FieldName, media_path string) (*big.Rat, error) {
func (p *internalExifParser) readRationalTag(tags *exif.Exif, name exif.FieldName, mediaPath string) (*big.Rat, error) {
tag, err := tags.Get(name)
if err != nil {
return nil, errors.Wrapf(err, "could not read %s from EXIF: %s", name, media_path)
return nil, errors.Wrapf(err, couldNotReadXfromEXIFy, name, mediaPath)
}
if tag != nil {
value, err := tag.Rat(0)
if err != nil {
return nil, errors.Wrapf(err, "could not parse %s from EXIF as rational: %s", name, media_path)
return nil, errors.Wrapf(err, "could not parse %s from EXIF as rational: %s", name, mediaPath)
}
return value, nil
}
log.Printf("WARN: EXIF tag %s returned null: %s\n", name, media_path)
return nil, errors.New("exif tag returned null")
log.Printf(warnEXIFtagXreturnedNully, name, mediaPath)
return nil, ErrNullExifTag
}
func (p *internalExifParser) readIntegerTag(tags *exif.Exif, name exif.FieldName, media_path string) (*int, error) {
func (p *internalExifParser) readIntegerTag(tags *exif.Exif, name exif.FieldName, mediaPath string) (*int, error) {
tag, err := tags.Get(name)
if err != nil {
return nil, errors.Wrapf(err, "could not read %s from EXIF: %s", name, media_path)
return nil, errors.Wrapf(err, couldNotReadXfromEXIFy, name, mediaPath)
}
if tag != nil {
value, err := tag.Int(0)
if err != nil {
return nil, errors.Wrapf(err, "Could not parse %s from EXIF as integer: %s", name, media_path)
return nil, errors.Wrapf(err, "Could not parse %s from EXIF as integer: %s", name, mediaPath)
}
return &value, nil
}
log.Printf("WARN: EXIF tag %s returned null: %s\n", name, media_path)
return nil, errors.New("exif tag returned null")
log.Printf(warnEXIFtagXreturnedNully, name, mediaPath)
return nil, ErrNullExifTag
}

4
api/scanner/media_encoding/executable_worker/executable_worker_test.go
View File

@@ -17,7 +17,9 @@ func TestMain(m *testing.M) {
func setPathWithCurrent(paths ...string) func() {
_, file, _, ok := runtime.Caller(0)
if !ok {
return func() {}
return func() {
// Return an empty function in case of error
}
}
base := filepath.Dir(file)

8
api/scanner/media_encoding/executable_worker/magick_cli_test.go
View File

@@ -7,6 +7,8 @@ import (
"github.com/photoview/photoview/api/scanner/media_encoding/executable_worker"
)
const testdataBinPath = "./testdata/bin"
func TestMagickCliNotExist(t *testing.T) {
done := setPathWithCurrent()
defer done()
@@ -18,7 +20,7 @@ func TestMagickCliNotExist(t *testing.T) {
}
func TestMagickCliIgnore(t *testing.T) {
donePath := setPathWithCurrent("./testdata/bin")
donePath := setPathWithCurrent(testdataBinPath)
defer donePath()
doneDisableRaw := setEnv("PHOTOVIEW_DISABLE_RAW_PROCESSING", "true")
@@ -31,7 +33,7 @@ func TestMagickCliIgnore(t *testing.T) {
}
func TestMagickCliFail(t *testing.T) {
donePath := setPathWithCurrent("./testdata/bin")
donePath := setPathWithCurrent(testdataBinPath)
defer donePath()
executable_worker.InitializeExecutableWorkers()
@@ -53,7 +55,7 @@ func TestMagickCliFail(t *testing.T) {
}
func TestMagickCliSucceed(t *testing.T) {
donePath := setPathWithCurrent("./testdata/bin")
donePath := setPathWithCurrent(testdataBinPath)
defer donePath()
executable_worker.InitializeExecutableWorkers()

6
api/scanner/periodic_scanner/periodic_scanner.go
View File

@@ -52,9 +52,9 @@ func InitializePeriodicScanner(db *gorm.DB) error {
}
func ChangePeriodicScanInterval(duration time.Duration) {
var new_ticker *time.Ticker = nil
var newTicker *time.Ticker = nil
if duration > 0 {
new_ticker = time.NewTicker(duration)
newTicker = time.NewTicker(duration)
log.Printf("Periodic scan interval changed: %s", duration.String())
} else {
log.Print("Periodic scan interval changed: disabled")
@@ -68,7 +68,7 @@ func ChangePeriodicScanInterval(duration time.Duration) {
mainPeriodicScanner.ticker.Stop()
}
mainPeriodicScanner.ticker = new_ticker
mainPeriodicScanner.ticker = newTicker
mainPeriodicScanner.ticker_changed <- true
}
}

9
api/scanner/scanner_album.go
View File

@@ -2,7 +2,6 @@ package scanner
import (
"fmt"
"io/ioutil"
"log"
"os"
"path"
@@ -118,7 +117,7 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
albumMedia := make([]*models.Media, 0)
dirContent, err := ioutil.ReadDir(ctx.GetAlbum().Path)
dirContent, err := os.ReadDir(ctx.GetAlbum().Path)
if err != nil {
return nil, err
}
@@ -133,7 +132,11 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
}
if !item.IsDir() && !isDirSymlink && ctx.GetCache().IsPathMedia(mediaPath) {
skip, err := scanner_tasks.Tasks.MediaFound(ctx, item, mediaPath)
itemInfo, err := item.Info()
if err != nil {
return nil, err
}
skip, err := scanner_tasks.Tasks.MediaFound(ctx, itemInfo, mediaPath)
if err != nil {
return nil, err
}

12
api/scanner/scanner_album_test.go
View File

@@ -9,6 +9,8 @@ import (
"github.com/stretchr/testify/assert"
)
const testDataPath = "./test_data"
func TestNewRootPath(t *testing.T) {
db := test_utils.DatabaseTest(t)
@@ -21,7 +23,7 @@ func TestNewRootPath(t *testing.T) {
}
t.Run("Insert valid root album", func(t *testing.T) {
album, err := scanner.NewRootAlbum(db, "./test_data", &user)
album, err := scanner.NewRootAlbum(db, testDataPath, &user)
if !assert.NoError(t, err) {
return
}
@@ -33,7 +35,7 @@ func TestNewRootPath(t *testing.T) {
t.Run("Insert duplicate root album", func(t *testing.T) {
_, err := scanner.NewRootAlbum(db, "./test_data", &user)
_, err := scanner.NewRootAlbum(db, testDataPath, &user)
assert.Error(t, err)
assert.Contains(t, err.Error(), "user already owns a path containing this path:")
@@ -57,7 +59,7 @@ func TestNewRootPath(t *testing.T) {
return
}
album, err := scanner.NewRootAlbum(db, "./test_data", &user2)
album, err := scanner.NewRootAlbum(db, testDataPath, &user2)
if !assert.NoError(t, err) {
return
}
@@ -65,8 +67,8 @@ func TestNewRootPath(t *testing.T) {
assert.NotNil(t, album)
assert.Contains(t, album.Path, "/api/scanner/test_data")
owner_count := db.Model(&album).Association("Owners").Count()
assert.EqualValues(t, 2, owner_count)
ownerCount := db.Model(&album).Association("Owners").Count()
assert.EqualValues(t, 2, ownerCount)
})
}

20
api/scanner/scanner_cache/cache.go
View File

@@ -27,25 +27,25 @@ func MakeAlbumCache() *AlbumScannerCache {
}
// Insert single album directory in cache
func (c *AlbumScannerCache) InsertAlbumPath(path string, contains_photo bool) {
func (c *AlbumScannerCache) InsertAlbumPath(path string, containsPhoto bool) {
c.mutex.Lock()
defer c.mutex.Unlock()
c.path_contains_photos[path] = contains_photo
c.path_contains_photos[path] = containsPhoto
}
// Insert album path and all parent directories up to the given root directory in cache
func (c *AlbumScannerCache) InsertAlbumPaths(end_path string, root string, contains_photo bool) {
curr_path := path.Clean(end_path)
root_path := path.Clean(root)
func (c *AlbumScannerCache) InsertAlbumPaths(endPath string, root string, containsPhoto bool) {
currPath := path.Clean(endPath)
rootPath := path.Clean(root)
c.mutex.Lock()
defer c.mutex.Unlock()
for curr_path != root_path || curr_path == "." {
for currPath != rootPath || currPath == "." {
c.path_contains_photos[curr_path] = contains_photo
c.path_contains_photos[currPath] = containsPhoto
curr_path = path.Dir(curr_path)
currPath = path.Dir(currPath)
}
}
@@ -103,11 +103,11 @@ func (c *AlbumScannerCache) GetAlbumIgnore(path string) *[]string {
return nil
}
func (c *AlbumScannerCache) InsertAlbumIgnore(path string, ignore_data []string) {
func (c *AlbumScannerCache) InsertAlbumIgnore(path string, ignoreData []string) {
c.mutex.Lock()
defer c.mutex.Unlock()
c.ignore_data[path] = ignore_data
c.ignore_data[path] = ignoreData
}
func (c *AlbumScannerCache) IsPathMedia(mediaPath string) bool {

8
api/scanner/scanner_media.go
View File

@@ -71,17 +71,17 @@ func ScanMedia(tx *gorm.DB, mediaPath string, albumId int, cache *scanner_cache.
// ProcessSingleMedia processes a single media, might be used to reprocess media with corrupted cache
// Function waits for processing to finish before returning.
func ProcessSingleMedia(db *gorm.DB, media *models.Media) error {
album_cache := scanner_cache.MakeAlbumCache()
albumCache := scanner_cache.MakeAlbumCache()
var album models.Album
if err := db.Model(media).Association("Album").Find(&album); err != nil {
return err
}
media_data := media_encoding.NewEncodeMediaData(media)
mediaData := media_encoding.NewEncodeMediaData(media)
task_context := scanner_task.NewTaskContext(context.Background(), db, &album, album_cache)
if err := scanMedia(task_context, media, &media_data, 0, 1); err != nil {
taskContext := scanner_task.NewTaskContext(context.Background(), db, &album, albumCache)
if err := scanMedia(taskContext, media, &mediaData, 0, 1); err != nil {
return errors.Wrap(err, "single media scan")
}

32
api/scanner/scanner_queue/queue.go
View File

@@ -18,6 +18,8 @@ import (
"gorm.io/gorm"
)
const globalScannerProgress = "global-scanner-progress"
// ScannerJob describes a job on the queue to be run by the scanner over a single album
type ScannerJob struct {
ctx scanner_task.TaskContext
@@ -104,11 +106,11 @@ func (queue *ScannerQueue) startBackgroundWorker() {
<-queue.idle_chan
queue.mutex.Lock()
should_stop := queue.close_chan != nil && len(queue.in_progress) == 0 && len(queue.up_next) == 0
shouldStop := queue.close_chan != nil && len(queue.in_progress) == 0 && len(queue.up_next) == 0
queue.running = false
queue.mutex.Unlock()
if should_stop {
if shouldStop {
*queue.close_chan <- true
break
}
@@ -121,14 +123,14 @@ func (queue *ScannerQueue) startBackgroundWorker() {
func (queue *ScannerQueue) CloseBackgroundWorker() {
queue.mutex.Lock()
close_chan := make(chan bool)
queue.close_chan = &close_chan
closeChan := make(chan bool)
queue.close_chan = &closeChan
queue.mutex.Unlock()
queue.notify()
log.Println("Waiting for scanner background worker to finish all jobs...")
<-close_chan
<-closeChan
}
func (queue *ScannerQueue) processQueue(notifyThrottle *utils.Throttle) {
@@ -162,14 +164,14 @@ func (queue *ScannerQueue) processQueue(notifyThrottle *utils.Throttle) {
}()
}
in_progress_length := len(global_scanner_queue.in_progress)
up_next_length := len(global_scanner_queue.up_next)
inProgressLength := len(global_scanner_queue.in_progress)
upNextLength := len(global_scanner_queue.up_next)
queue.mutex.Unlock()
if in_progress_length+up_next_length == 0 {
if inProgressLength+upNextLength == 0 {
notification.BroadcastNotification(&models.Notification{
Key: "global-scanner-progress",
Key: globalScannerProgress,
Type: models.NotificationTypeMessage,
Header: "Generating blurhashes",
Content: "Generating blurhashes for newly scanned media",
@@ -181,7 +183,7 @@ func (queue *ScannerQueue) processQueue(notifyThrottle *utils.Throttle) {
}
notification.BroadcastNotification(&models.Notification{
Key: "global-scanner-progress",
Key: globalScannerProgress,
Type: models.NotificationTypeMessage,
Header: "Scanner complete",
Content: "All jobs have been scanned",
@@ -190,10 +192,10 @@ func (queue *ScannerQueue) processQueue(notifyThrottle *utils.Throttle) {
} else {
notifyThrottle.Trigger(func() {
notification.BroadcastNotification(&models.Notification{
Key: "global-scanner-progress",
Key: globalScannerProgress,
Type: models.NotificationTypeMessage,
Header: "Scanning media",
Content: fmt.Sprintf("%d jobs in progress\n%d jobs waiting", in_progress_length, up_next_length),
Content: fmt.Sprintf("%d jobs in progress\n%d jobs waiting", inProgressLength, upNextLength),
})
})
}
@@ -229,8 +231,8 @@ func AddAllToQueue() error {
// AddUserToQueue finds all root albums owned by the given user and adds them to the scanner queue.
// Function does not block.
func AddUserToQueue(user *models.User) error {
album_cache := scanner_cache.MakeAlbumCache()
albums, album_errors := scanner.FindAlbumsForUser(global_scanner_queue.db, user, album_cache)
albumCache := scanner_cache.MakeAlbumCache()
albums, album_errors := scanner.FindAlbumsForUser(global_scanner_queue.db, user, albumCache)
for _, err := range album_errors {
return errors.Wrapf(err, "find albums for user (user_id: %d)", user.ID)
}
@@ -238,7 +240,7 @@ func AddUserToQueue(user *models.User) error {
global_scanner_queue.mutex.Lock()
for _, album := range albums {
global_scanner_queue.addJob(&ScannerJob{
ctx: scanner_task.NewTaskContext(context.Background(), global_scanner_queue.db, album, album_cache),
ctx: scanner_task.NewTaskContext(context.Background(), global_scanner_queue.db, album, albumCache),
})
}
global_scanner_queue.mutex.Unlock()

4
api/scanner/scanner_queue/queue_test.go
View File

@@ -24,7 +24,7 @@ func makeScannerJob(albumID int) ScannerJob {
return NewScannerJob(scanner_task.NewTaskContext(context.Background(), nil, makeAlbumWithID(albumID), scanner_cache.MakeAlbumCache()))
}
func TestScannerQueue_AddJob(t *testing.T) {
func TestScannerQueueAddJob(t *testing.T) {
scannerJobs := []ScannerJob{
makeScannerJob(100),
@@ -72,7 +72,7 @@ func TestScannerQueue_AddJob(t *testing.T) {
})
}
func TestScannerQueue_JobOnQueue(t *testing.T) {
func TestScannerQueueJobOnQueue(t *testing.T) {
scannerJobs := []ScannerJob{
makeScannerJob(100),

34
api/scanner/scanner_tasks/cleanup_tasks/cleanup_media_test.go
View File

@@ -30,23 +30,23 @@ func TestCleanupMedia(t *testing.T) {
return
}
test_dir := t.TempDir()
assert.NoError(t, copy.Copy("../../test_data", test_dir))
testDir := t.TempDir()
assert.NoError(t, copy.Copy("../../test_data", testDir))
countAllMedia := func() int {
var all_media []*models.Media
if !assert.NoError(t, db.Find(&all_media).Error) {
var allMedia []*models.Media
if !assert.NoError(t, db.Find(&allMedia).Error) {
return -1
}
return len(all_media)
return len(allMedia)
}
countAllMediaURLs := func() int {
var all_media_urls []*models.MediaURL
if !assert.NoError(t, db.Find(&all_media_urls).Error) {
var allMediaURLs []*models.MediaURL
if !assert.NoError(t, db.Find(&allMediaURLs).Error) {
return -1
}
return len(all_media_urls)
return len(allMediaURLs)
}
pass := "1234"
@@ -60,20 +60,20 @@ func TestCleanupMedia(t *testing.T) {
return
}
root_album := models.Album{
rootAlbum := models.Album{
Title: "root album",
Path: test_dir,
Path: testDir,
}
if !assert.NoError(t, db.Save(&root_album).Error) {
if !assert.NoError(t, db.Save(&rootAlbum).Error) {
return
}
err = db.Model(user1).Association("Albums").Append(&root_album)
err = db.Model(user1).Association("Albums").Append(&rootAlbum)
if !assert.NoError(t, err) {
return
}
err = db.Model(user2).Association("Albums").Append(&root_album)
err = db.Model(user2).Association("Albums").Append(&rootAlbum)
if !assert.NoError(t, err) {
return
}
@@ -84,25 +84,25 @@ func TestCleanupMedia(t *testing.T) {
assert.Equal(t, 18, countAllMediaURLs())
// move faces directory
assert.NoError(t, os.Rename(path.Join(test_dir, "faces"), path.Join(test_dir, "faces_moved")))
assert.NoError(t, os.Rename(path.Join(testDir, "faces"), path.Join(testDir, "faces_moved")))
test_utils.RunScannerAll(t, db)
assert.Equal(t, 9, countAllMedia())
assert.Equal(t, 18, countAllMediaURLs())
// remove faces_moved directory
assert.NoError(t, os.RemoveAll(path.Join(test_dir, "faces_moved")))
assert.NoError(t, os.RemoveAll(path.Join(testDir, "faces_moved")))
test_utils.RunScannerAll(t, db)
assert.Equal(t, 3, countAllMedia())
assert.Equal(t, 6, countAllMediaURLs())
})
t.Run("Modify images", func(t *testing.T) {
assert.NoError(t, os.Rename(path.Join(test_dir, "buttercup_close_summer_yellow.jpg"), path.Join(test_dir, "yellow-flower.jpg")))
assert.NoError(t, os.Rename(path.Join(testDir, "buttercup_close_summer_yellow.jpg"), path.Join(testDir, "yellow-flower.jpg")))
test_utils.RunScannerAll(t, db)
assert.Equal(t, 3, countAllMedia())
assert.Equal(t, 6, countAllMediaURLs())
assert.NoError(t, os.Remove(path.Join(test_dir, "lilac_lilac_bush_lilac.jpg")))
assert.NoError(t, os.Remove(path.Join(testDir, "lilac_lilac_bush_lilac.jpg")))
test_utils.RunScannerAll(t, db)
assert.Equal(t, 2, countAllMedia())
assert.Equal(t, 4, countAllMediaURLs())

4
api/scanner/scanner_tasks/cleanup_tasks/media_cleanup_task.go
View File

@@ -12,8 +12,8 @@ type MediaCleanupTask struct {
func (t MediaCleanupTask) AfterScanAlbum(ctx scanner_task.TaskContext, changedMedia []*models.Media, albumMedia []*models.Media) error {
cleanup_errors := CleanupMedia(ctx.GetDB(), ctx.GetAlbum().ID, albumMedia)
for _, err := range cleanup_errors {
cleanupErrors := CleanupMedia(ctx.GetDB(), ctx.GetAlbum().ID, albumMedia)
for _, err := range cleanupErrors {
scanner_utils.ScannerError("delete old media: %s", err)
}

10
api/scanner/scanner_tasks/processing_tasks/process_photo_task.go
View File

@@ -55,7 +55,6 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
return []*models.MediaURL{}, errors.Wrap(err, "error processing photo highres")
}
var photoDimensions *media_utils.PhotoDimensions
var baseImagePath string = photo.Path
// Generate high res jpeg
@@ -92,15 +91,14 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
}
}
var photoDimensions *media_utils.PhotoDimensions
// Save original photo to database
if origURL == nil {
// Make sure photo dimensions is set
if photoDimensions == nil {
photoDimensions, err = media_utils.GetPhotoDimensions(baseImagePath)
if err != nil {
return []*models.MediaURL{}, err
}
photoDimensions, err = media_utils.GetPhotoDimensions(baseImagePath)
if err != nil {
return []*models.MediaURL{}, err
}
original, err := saveOriginalPhotoToDB(ctx.GetDB(), photo, mediaData, photoDimensions)

24
api/scanner/scanner_tasks/processing_tasks/process_video_task.go
View File

@@ -87,12 +87,12 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
}
if videoWebURL == nil && !videoType.IsWebCompatible() {
web_video_name := fmt.Sprintf("web_video_%s_%s", path.Base(video.Path), utils.GenerateToken())
web_video_name = strings.ReplaceAll(web_video_name, ".", "_")
web_video_name = strings.ReplaceAll(web_video_name, " ", "_")
web_video_name = web_video_name + ".mp4"
webVideoName := fmt.Sprintf("web_video_%s_%s", path.Base(video.Path), utils.GenerateToken())
webVideoName = strings.ReplaceAll(webVideoName, ".", "_")
webVideoName = strings.ReplaceAll(webVideoName, " ", "_")
webVideoName = webVideoName + ".mp4"
webVideoPath := path.Join(mediaCachePath, web_video_name)
webVideoPath := path.Join(mediaCachePath, webVideoName)
err = executable_worker.FfmpegCli.EncodeMp4(video.Path, webVideoPath)
if err != nil {
@@ -111,7 +111,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
mediaURL := models.MediaURL{
MediaID: video.ID,
MediaName: web_video_name,
MediaName: webVideoName,
Width: webMetadata.Width,
Height: webMetadata.Height,
Purpose: models.VideoWeb,
@@ -132,12 +132,12 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
}
if videoThumbnailURL == nil {
video_thumb_name := fmt.Sprintf("video_thumb_%s_%s", path.Base(video.Path), utils.GenerateToken())
video_thumb_name = strings.ReplaceAll(video_thumb_name, ".", "_")
video_thumb_name = strings.ReplaceAll(video_thumb_name, " ", "_")
video_thumb_name = video_thumb_name + ".jpg"
videoThumbName := fmt.Sprintf("video_thumb_%s_%s", path.Base(video.Path), utils.GenerateToken())
videoThumbName = strings.ReplaceAll(videoThumbName, ".", "_")
videoThumbName = strings.ReplaceAll(videoThumbName, " ", "_")
videoThumbName = videoThumbName + ".jpg"
thumbImagePath := path.Join(mediaCachePath, video_thumb_name)
thumbImagePath := path.Join(mediaCachePath, videoThumbName)
err = executable_worker.FfmpegCli.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
if err != nil {
@@ -156,7 +156,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
thumbMediaURL := models.MediaURL{
MediaID: video.ID,
MediaName: video_thumb_name,
MediaName: videoThumbName,
Width: thumbDimensions.Width,
Height: thumbDimensions.Height,
Purpose: models.VideoThumbnail,

18
api/scanner/scanner_tasks/processing_tasks/processing_functions.go
View File

@@ -11,7 +11,7 @@ import (
"gorm.io/gorm"
)
func generateSaveHighResJPEG(tx *gorm.DB, media *models.Media, imageData *media_encoding.EncodeMediaData, highres_name string, imagePath string, mediaURL *models.MediaURL) (*models.MediaURL, error) {
func generateSaveHighResJPEG(tx *gorm.DB, media *models.Media, imageData *media_encoding.EncodeMediaData, highResName string, imagePath string, mediaURL *models.MediaURL) (*models.MediaURL, error) {
err := imageData.EncodeHighRes(imagePath)
if err != nil {
@@ -32,7 +32,7 @@ func generateSaveHighResJPEG(tx *gorm.DB, media *models.Media, imageData *media_
mediaURL = &models.MediaURL{
MediaID: media.ID,
MediaName: highres_name,
MediaName: highResName,
Width: photoDimensions.Width,
Height: photoDimensions.Height,
Purpose: models.PhotoHighRes,
@@ -41,7 +41,7 @@ func generateSaveHighResJPEG(tx *gorm.DB, media *models.Media, imageData *media_
}
if err := tx.Create(&mediaURL).Error; err != nil {
return nil, errors.Wrapf(err, "could not insert highres media url (%d, %s)", media.ID, highres_name)
return nil, errors.Wrapf(err, "could not insert highres media url (%d, %s)", media.ID, highResName)
}
} else {
mediaURL.Width = photoDimensions.Width
@@ -49,15 +49,15 @@ func generateSaveHighResJPEG(tx *gorm.DB, media *models.Media, imageData *media_
mediaURL.FileSize = fileStats.Size()
if err := tx.Save(&mediaURL).Error; err != nil {
return nil, errors.Wrapf(err, "could not update media url after side car changes (%d, %s)", media.ID, highres_name)
return nil, errors.Wrapf(err, "could not update media url after side car changes (%d, %s)", media.ID, highResName)
}
}
return mediaURL, nil
}
func generateSaveThumbnailJPEG(tx *gorm.DB, media *models.Media, thumbnail_name string, photoCachePath string, baseImagePath string, mediaURL *models.MediaURL) (*models.MediaURL, error) {
thumbOutputPath := path.Join(photoCachePath, thumbnail_name)
func generateSaveThumbnailJPEG(tx *gorm.DB, media *models.Media, thumbnailName string, photoCachePath string, baseImagePath string, mediaURL *models.MediaURL) (*models.MediaURL, error) {
thumbOutputPath := path.Join(photoCachePath, thumbnailName)
thumbSize, err := media_encoding.EncodeThumbnail(tx, baseImagePath, thumbOutputPath)
if err != nil {
@@ -73,7 +73,7 @@ func generateSaveThumbnailJPEG(tx *gorm.DB, media *models.Media, thumbnail_name
mediaURL = &models.MediaURL{
MediaID: media.ID,
MediaName: thumbnail_name,
MediaName: thumbnailName,
Width: thumbSize.Width,
Height: thumbSize.Height,
Purpose: models.PhotoThumbnail,
@@ -82,7 +82,7 @@ func generateSaveThumbnailJPEG(tx *gorm.DB, media *models.Media, thumbnail_name
}
if err := tx.Create(&mediaURL).Error; err != nil {
return nil, errors.Wrapf(err, "could not insert thumbnail media url (%d, %s)", media.ID, thumbnail_name)
return nil, errors.Wrapf(err, "could not insert thumbnail media url (%d, %s)", media.ID, thumbnailName)
}
} else {
mediaURL.Width = thumbSize.Width
@@ -90,7 +90,7 @@ func generateSaveThumbnailJPEG(tx *gorm.DB, media *models.Media, thumbnail_name
mediaURL.FileSize = fileStats.Size()
if err := tx.Save(&mediaURL).Error; err != nil {
return nil, errors.Wrapf(err, "could not update media url after side car changes (%d, %s)", media.ID, thumbnail_name)
return nil, errors.Wrapf(err, "could not update media url after side car changes (%d, %s)", media.ID, thumbnailName)
}
}

30
api/scanner/scanner_test.go
View File

@@ -25,16 +25,16 @@ func TestFullScan(t *testing.T) {
return
}
root_album := models.Album{
rootAlbum := models.Album{
Title: "root album",
Path: "./test_data",
}
if !assert.NoError(t, db.Save(&root_album).Error) {
if !assert.NoError(t, db.Save(&rootAlbum).Error) {
return
}
err = db.Model(user).Association("Albums").Append(&root_album)
err = db.Model(user).Association("Albums").Append(&rootAlbum)
if !assert.NoError(t, err) {
return
}
@@ -45,37 +45,37 @@ func TestFullScan(t *testing.T) {
test_utils.RunScannerOnUser(t, db, user)
var all_media []*models.Media
if !assert.NoError(t, db.Find(&all_media).Error) {
var allMedia []*models.Media
if !assert.NoError(t, db.Find(&allMedia).Error) {
return
}
assert.Equal(t, 9, len(all_media))
assert.Equal(t, 9, len(allMedia))
var all_media_url []*models.MediaURL
if !assert.NoError(t, db.Find(&all_media_url).Error) {
var allMediaURL []*models.MediaURL
if !assert.NoError(t, db.Find(&allMediaURL).Error) {
return
}
assert.Equal(t, 18, len(all_media_url))
assert.Equal(t, 18, len(allMediaURL))
// Verify that faces was recognized
assert.Eventually(t, func() bool {
var all_face_groups []*models.FaceGroup
if !assert.NoError(t, db.Find(&all_face_groups).Error) {
var allFaceGroups []*models.FaceGroup
if !assert.NoError(t, db.Find(&allFaceGroups).Error) {
return false
}
return len(all_face_groups) == 3
return len(allFaceGroups) == 3
}, time.Second*5, time.Millisecond*500)
assert.Eventually(t, func() bool {
var all_image_faces []*models.ImageFace
if !assert.NoError(t, db.Find(&all_image_faces).Error) {
var allImageFaces []*models.ImageFace
if !assert.NoError(t, db.Find(&allImageFaces).Error) {
return false
}
return len(all_image_faces) == 6
return len(allImageFaces) == 6
}, time.Second*5, time.Millisecond*500)
}

23
api/scanner/scanner_user.go
View File

@@ -3,7 +3,6 @@ package scanner
import (
"bufio"
"container/list"
"io/ioutil"
"log"
"os"
"path"
@@ -43,7 +42,7 @@ func getPhotoviewIgnore(ignorePath string) ([]string, error) {
return photoviewIgnore, scanner.Err()
}
func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cache.AlbumScannerCache) ([]*models.Album, []error) {
func FindAlbumsForUser(db *gorm.DB, user *models.User, albumCache *scanner_cache.AlbumScannerCache) ([]*models.Album, []error) {
if err := user.FillAlbums(db); err != nil {
return nil, []error{err}
@@ -101,7 +100,7 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
albumIgnore := albumInfo.ignore
// Read path
dirContent, err := ioutil.ReadDir(albumPath)
dirContent, err := os.ReadDir(albumPath)
if err != nil {
scanErrors = append(scanErrors, errors.Wrapf(err, "read directory (%s)", albumPath))
continue
@@ -156,7 +155,7 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
}
// Store album ignore
album_cache.InsertAlbumIgnore(albumPath, albumIgnore)
albumCache.InsertAlbumIgnore(albumPath, albumIgnore)
if err := tx.Create(&album).Error; err != nil {
return errors.Wrap(err, "insert album into database")
@@ -182,7 +181,7 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
}
// Update album ignore
album_cache.InsertAlbumIgnore(albumPath, albumIgnore)
albumCache.InsertAlbumIgnore(albumPath, albumIgnore)
}
userAlbums = append(userAlbums, album)
@@ -210,7 +209,7 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
continue
}
if (item.IsDir() || isDirSymlink) && directoryContainsPhotos(subalbumPath, album_cache, albumIgnore) {
if (item.IsDir() || isDirSymlink) && directoryContainsPhotos(subalbumPath, albumCache, albumIgnore) {
scanQueue.PushBack(scanInfo{
path: subalbumPath,
parent: album,
@@ -228,21 +227,21 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
func directoryContainsPhotos(rootPath string, cache *scanner_cache.AlbumScannerCache, albumIgnore []string) bool {
if contains_image := cache.AlbumContainsPhotos(rootPath); contains_image != nil {
return *contains_image
if containsImage := cache.AlbumContainsPhotos(rootPath); containsImage != nil {
return *containsImage
}
scanQueue := list.New()
scanQueue.PushBack(rootPath)
scanned_directories := make([]string, 0)
scannedDirectories := make([]string, 0)
for scanQueue.Front() != nil {
dirPath := scanQueue.Front().Value.(string)
scanQueue.Remove(scanQueue.Front())
scanned_directories = append(scanned_directories, dirPath)
scannedDirectories = append(scannedDirectories, dirPath)
// Update ignore dir list
photoviewIgnore, err := getPhotoviewIgnore(dirPath)
@@ -253,7 +252,7 @@ func directoryContainsPhotos(rootPath string, cache *scanner_cache.AlbumScannerC
}
ignoreEntries := ignore.CompileIgnoreLines(albumIgnore...)
dirContent, err := ioutil.ReadDir(dirPath)
dirContent, err := os.ReadDir(dirPath)
if err != nil {
scanner_utils.ScannerError("Could not read directory (%s): %s\n", dirPath, err.Error())
return false
@@ -285,7 +284,7 @@ func directoryContainsPhotos(rootPath string, cache *scanner_cache.AlbumScannerC
}
for _, scanned_path := range scanned_directories {
for _, scanned_path := range scannedDirectories {
log.Printf("Insert Album %s, contains photo is false", scanned_path)
cache.InsertAlbumPath(scanned_path, false)
}