public/photoview
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/photoview.git synced 2026-08-03 21:19:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Start on big scanner code refactoring

Browse Source
This commit is contained in:
viktorstrate
2022-02-14 23:57:45 +01:00
parent e9a4839f48
commit 6361df1793
12 changed files with 454 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
api/scanner/scanner_album.go
View File

@@ -6,16 +6,15 @@ import (
"log"
"os"
"path"
"time"
"github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/graphql/notification"
"github.com/photoview/photoview/api/scanner/face_detection"
"github.com/photoview/photoview/api/scanner/scanner_cache"
"github.com/photoview/photoview/api/scanner/media_encoding"
"github.com/photoview/photoview/api/scanner/scanner_task"
"github.com/photoview/photoview/api/scanner/scanner_tasks"
"github.com/photoview/photoview/api/scanner/scanner_utils"
"github.com/photoview/photoview/api/utils"
"github.com/pkg/errors"
ignore "github.com/sabhiram/go-gitignore"
"gorm.io/gorm"
)
@@ -56,7 +55,7 @@ func NewRootAlbum(db *gorm.DB, rootPath string, owner *models.User) (*models.Alb
}
if err := db.Model(&owner).Association("Albums").Append(&album); err != nil {
return nil, errors.Wrap(err, "failed to add owner to already existing album")
return nil, errors.Wrap(err, "add owner to already existing album")
}
return &album, nil
@@ -87,142 +86,145 @@ func ValidRootPath(rootPath string) bool {
return true
}
func scanAlbum(album *models.Album, cache *scanner_cache.AlbumScannerCache, db *gorm.DB) {
func ScanAlbum(ctx scanner_task.TaskContext) {
album_notify_key := utils.GenerateToken()
notifyThrottle := utils.NewThrottle(500 * time.Millisecond)
notifyThrottle.Trigger(nil)
newCtx, err := scanner_tasks.Tasks.BeforeScanAlbum(ctx)
if err != nil {
scanner_utils.ScannerError("before scan album (%s): %s", ctx.GetAlbum().Path, err)
return
}
ctx = newCtx
// Scan for photos
albumMedia, err := findMediaForAlbum(album, cache, db, func(photo *models.Media, newPhoto bool) {
if newPhoto {
notifyThrottle.Trigger(func() {
notification.BroadcastNotification(&models.Notification{
Key: album_notify_key,
Type: models.NotificationTypeMessage,
Header: fmt.Sprintf("Found new media in album '%s'", album.Title),
Content: fmt.Sprintf("Found %s", photo.Path),
})
})
}
})
albumMedia, err := findMediaForAlbum(ctx)
if err != nil {
scanner_utils.ScannerError("Failed to find media for album (%s): %s", album.Path, err)
scanner_utils.ScannerError("find media for album (%s): %s", ctx.GetAlbum().Path, err)
return
}
album_has_changes := false
albumHasChanges := false
for count, media := range albumMedia {
processing_was_needed := false
didProcess := false
transactionError := db.Transaction(func(tx *gorm.DB) error {
processing_was_needed, err = ProcessMedia(tx, media)
transactionError := ctx.GetDB().Transaction(func(tx *gorm.DB) error {
// processing_was_needed, err = ProcessMedia(tx, media)
didProcess, err = processMedia(ctx, media)
if err != nil {
return errors.Wrapf(err, "failed to process photo (%s)", media.Path)
return errors.Wrapf(err, "process media (%s)", media.Path)
}
if processing_was_needed {
album_has_changes = true
progress := float64(count) / float64(len(albumMedia)) * 100.0
notification.BroadcastNotification(&models.Notification{
Key: album_notify_key,
Type: models.NotificationTypeProgress,
Header: fmt.Sprintf("Processing media for album '%s'", album.Title),
Content: fmt.Sprintf("Processed media at %s", media.Path),
Progress: &progress,
})
if didProcess {
albumHasChanges = true
}
if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, media, didProcess, count, len(albumMedia)); err != nil {
return err
}
return nil
})
if transactionError != nil {
scanner_utils.ScannerError("Failed to begin database transaction: %s", transactionError)
scanner_utils.ScannerError("begin database transaction: %s", transactionError)
}
if processing_was_needed && media.Type == models.MediaTypePhoto {
if didProcess && media.Type == models.MediaTypePhoto {
go func(media *models.Media) {
if face_detection.GlobalFaceDetector == nil {
return
}
if err := face_detection.GlobalFaceDetector.DetectFaces(db, media); err != nil {
if err := face_detection.GlobalFaceDetector.DetectFaces(ctx.GetDB(), media); err != nil {
scanner_utils.ScannerError("Error detecting faces in image (%s): %s", media.Path, err)
}
}(media)
}
}
cleanup_errors := CleanupMedia(db, album.ID, albumMedia)
cleanup_errors := CleanupMedia(ctx.GetDB(), ctx.GetAlbum().ID, albumMedia)
for _, err := range cleanup_errors {
scanner_utils.ScannerError("Failed to delete old media: %s", err)
scanner_utils.ScannerError("delete old media: %s", err)
}
if album_has_changes {
timeoutDelay := 2000
notification.BroadcastNotification(&models.Notification{
Key: album_notify_key,
Type: models.NotificationTypeMessage,
Positive: true,
Header: fmt.Sprintf("Done processing media for album '%s'", album.Title),
Content: fmt.Sprintf("All media have been processed"),
Timeout: &timeoutDelay,
})
if err := scanner_tasks.Tasks.AfterScanAlbum(ctx, albumHasChanges); err != nil {
scanner_utils.ScannerError("after scan album: %s", err)
}
}
func findMediaForAlbum(album *models.Album, cache *scanner_cache.AlbumScannerCache, db *gorm.DB, onScanPhoto func(photo *models.Media, newPhoto bool)) ([]*models.Media, error) {
func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
albumPhotos := make([]*models.Media, 0)
albumMedia := make([]*models.Media, 0)
dirContent, err := ioutil.ReadDir(album.Path)
dirContent, err := ioutil.ReadDir(ctx.GetAlbum().Path)
if err != nil {
return nil, err
}
// Get ignore data
albumIgnore := ignore.CompileIgnoreLines(*cache.GetAlbumIgnore(album.Path)...)
for _, item := range dirContent {
photoPath := path.Join(album.Path, item.Name())
mediaPath := path.Join(ctx.GetAlbum().Path, item.Name())
isDirSymlink, err := utils.IsDirSymlink(photoPath)
isDirSymlink, err := utils.IsDirSymlink(mediaPath)
if err != nil {
log.Printf("Cannot detect whether %s is symlink to a directory. Pretending it is not", photoPath)
log.Printf("Cannot detect whether %s is symlink to a directory. Pretending it is not", mediaPath)
isDirSymlink = false
}
if !item.IsDir() && !isDirSymlink && cache.IsPathMedia(photoPath) {
// Match file against ignore data
if albumIgnore.MatchesPath(item.Name()) {
log.Printf("File %s ignored\n", item.Name())
if !item.IsDir() && !isDirSymlink && ctx.GetCache().IsPathMedia(mediaPath) {
skip, err := scanner_tasks.Tasks.MediaFound(ctx, item, mediaPath)
if err != nil {
return nil, err
}
if skip {
continue
}
// Skip the JPEGs that are compressed version of raw files
counterpartFile := scanForRawCounterpartFile(photoPath)
counterpartFile := scanForRawCounterpartFile(mediaPath)
if counterpartFile != nil {
continue
}
err := db.Transaction(func(tx *gorm.DB) error {
media, isNewMedia, err := ScanMedia(tx, photoPath, album.ID, cache)
err = ctx.GetDB().Transaction(func(tx *gorm.DB) error {
media, isNewMedia, err := ScanMedia(tx, mediaPath, ctx.GetAlbum().ID, ctx.GetCache())
if err != nil {
return errors.Wrapf(err, "Scanning media error (%s)", photoPath)
return errors.Wrapf(err, "scanning media error (%s)", mediaPath)
}
onScanPhoto(media, isNewMedia)
if err = scanner_tasks.Tasks.AfterMediaFound(ctx, media, isNewMedia); err != nil {
return err
}
albumPhotos = append(albumPhotos, media)
albumMedia = append(albumMedia, media)
return nil
})
if err != nil {
scanner_utils.ScannerError("Error scanning media for album (%d): %s\n", album.ID, err)
scanner_utils.ScannerError("Error scanning media for album (%d): %s\n", ctx.GetAlbum().ID, err)
continue
}
}
}
return albumPhotos, nil
return albumMedia, nil
}
func processMedia(ctx scanner_task.TaskContext, media *models.Media) (bool, error) {
mediaData := media_encoding.EncodeMediaData{
Media: media,
}
_, err := mediaData.ContentType()
if err != nil {
return false, errors.Wrapf(err, "get content-type of media (%s)", media.Path)
}
// Make sure media cache directory exists
mediaCachePath, err := makeMediaCacheDir(media)
if err != nil {
return false, errors.Wrap(err, "cache directory error")
}
return scanner_tasks.Tasks.ProcessMedia(ctx, &mediaData, *mediaCachePath)
}