From 3c23f823301634aeb4e6f1685ca51f8a23a34239 Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Mon, 23 Nov 2020 19:59:01 +0100 Subject: [PATCH] Replace database, mostly album related --- api/database/{mysql.go => database.go} | 8 ++- api/graphql/models/album.go | 35 ++---------- api/graphql/models/generated.go | 14 ++--- api/graphql/models/media_exif.go | 23 ++------ api/graphql/models/site_info.go | 49 ++++------------- api/scanner/cleanup_media.go | 2 +- api/scanner/periodic_scanner.go | 8 +-- api/scanner/process_photo.go | 5 +- api/scanner/queue.go | 2 +- api/scanner/scanner_album.go | 73 ++++++++++++-------------- api/scanner/scanner_media.go | 3 +- 11 files changed, 75 insertions(+), 147 deletions(-) rename api/database/{mysql.go => database.go} (92%) diff --git a/api/database/mysql.go b/api/database/database.go similarity index 92% rename from api/database/mysql.go rename to api/database/database.go index 27b305b4..103307b2 100644 --- a/api/database/mysql.go +++ b/api/database/database.go @@ -71,7 +71,13 @@ func SetupDatabase() (*gorm.DB, error) { func MigrateDatabase(db *gorm.DB) error { - db.AutoMigrate(&models.User{}, &models.AccessToken{}) + db.AutoMigrate( + &models.User{}, + &models.AccessToken{}, + &models.SiteInfo{}, + &models.Album{}, + &models.MediaEXIF{}, + ) return nil } diff --git a/api/graphql/models/album.go b/api/graphql/models/album.go index 1f29e13a..a42aa69c 100644 --- a/api/graphql/models/album.go +++ b/api/graphql/models/album.go @@ -1,48 +1,19 @@ package models import ( - "database/sql" + "gorm.io/gorm" ) type Album struct { - AlbumID int + gorm.Model Title string ParentAlbum *int OwnerID int + Owner User Path string PathHash string } -func (a *Album) ID() int { - return a.AlbumID -} - func (a *Album) FilePath() string { return a.Path } - -func NewAlbumFromRow(row *sql.Row) (*Album, error) { - album := Album{} - - if err := row.Scan(&album.AlbumID, &album.Title, &album.ParentAlbum, &album.OwnerID, &album.Path, &album.PathHash); err != nil { - return nil, err - } - - return &album, nil -} - -func NewAlbumsFromRows(rows *sql.Rows) ([]*Album, error) { - albums := make([]*Album, 0) - - for rows.Next() { - var album Album - if err := rows.Scan(&album.AlbumID, &album.Title, &album.ParentAlbum, &album.OwnerID, &album.Path, &album.PathHash); err != nil { - return nil, err - } - albums = append(albums, &album) - } - - rows.Close() - - return albums, nil -} diff --git a/api/graphql/models/generated.go b/api/graphql/models/generated.go index 3413a2e6..d727ede8 100644 --- a/api/graphql/models/generated.go +++ b/api/graphql/models/generated.go @@ -52,13 +52,13 @@ type SearchResult struct { } // General information about the site -type SiteInfo struct { - InitialSetup bool `json:"initialSetup"` - // How often automatic scans should be initiated in seconds - PeriodicScanInterval int `json:"periodicScanInterval"` - // How many max concurrent scanner jobs that should run at once - ConcurrentWorkers int `json:"concurrentWorkers"` -} +// type SiteInfo struct { +// InitialSetup bool `json:"initialSetup"` +// // How often automatic scans should be initiated in seconds +// PeriodicScanInterval int `json:"periodicScanInterval"` +// // How many max concurrent scanner jobs that should run at once +// ConcurrentWorkers int `json:"concurrentWorkers"` +// } type MediaType string diff --git a/api/graphql/models/media_exif.go b/api/graphql/models/media_exif.go index bd0d8951..6090cbb1 100644 --- a/api/graphql/models/media_exif.go +++ b/api/graphql/models/media_exif.go @@ -1,12 +1,13 @@ package models import ( - "database/sql" "time" + + "gorm.io/gorm" ) type MediaEXIF struct { - ExifID int + gorm.Model Camera *string Maker *string Lens *string @@ -21,21 +22,3 @@ type MediaEXIF struct { GPSLatitude *float64 GPSLonitude *float64 } - -func (exif *MediaEXIF) Media() *Media { - panic("not implemented") -} - -func (exif *MediaEXIF) ID() int { - return exif.ExifID -} - -func NewMediaExifFromRow(row *sql.Row) (*MediaEXIF, error) { - exif := MediaEXIF{} - - if err := row.Scan(&exif.ExifID, &exif.Camera, &exif.Maker, &exif.Lens, &exif.DateShot, &exif.Exposure, &exif.Aperture, &exif.Iso, &exif.FocalLength, &exif.Flash, &exif.Orientation, &exif.ExposureProgram, &exif.GPSLatitude, &exif.GPSLonitude); err != nil { - return nil, err - } - - return &exif, nil -} diff --git a/api/graphql/models/site_info.go b/api/graphql/models/site_info.go index 53041cd7..0dc6f19a 100644 --- a/api/graphql/models/site_info.go +++ b/api/graphql/models/site_info.go @@ -5,55 +5,26 @@ import ( "gorm.io/gorm" ) -// func initializeSiteInfoRow(db *gorm.DB) (*SiteInfo, error) { -// _, err := db.Exec("INSERT INTO site_info (initial_setup, periodic_scan_interval, concurrent_workers) VALUES (true, 0, 3)") -// if err != nil { -// return nil, errors.Wrap(err, "initialize site_info row") -// } - -// siteInfo := &SiteInfo{} - -// row := db.QueryRow("SELECT * FROM site_info") -// if err := row.Scan(&siteInfo.InitialSetup, &siteInfo.PeriodicScanInterval, &siteInfo.ConcurrentWorkers); err != nil { -// return nil, errors.Wrap(err, "get site_info row after initialization") -// } - -// return siteInfo, nil -// } +type SiteInfo struct { + gorm.Model + InitialSetup bool + PeriodicScanInterval int + ConcurrentWorkers int +} // GetSiteInfo gets the site info row from the database, and creates it if it does not exist func GetSiteInfo(db *gorm.DB) (*SiteInfo, error) { var siteInfo SiteInfo - result := db.FirstOrCreate(&siteInfo, SiteInfo{ + err := db.FirstOrCreate(&siteInfo, SiteInfo{ InitialSetup: true, PeriodicScanInterval: 0, ConcurrentWorkers: 3, - }) - if result.Error != nil { - return nil, errors.Wrap(result.Error, "get site info from database") + }).Error + if err != nil { + return nil, errors.Wrap(err, "get site info from database") } - // rows, err := db.Query("SELECT * FROM site_info") - // defer rows.Close() - // if err != nil { - // return nil, err - // } - - // siteInfo := &SiteInfo{} - - // if !rows.Next() { - // // Entry does not exist - // siteInfo, err = initializeSiteInfoRow(db) - // if err != nil { - // return nil, err - // } - // } else { - // if err := rows.Scan(&siteInfo.InitialSetup, &siteInfo.PeriodicScanInterval, &siteInfo.ConcurrentWorkers); err != nil { - // return nil, err - // } - // } - return &siteInfo, nil } diff --git a/api/scanner/cleanup_media.go b/api/scanner/cleanup_media.go index fa6c3eee..b59f7546 100644 --- a/api/scanner/cleanup_media.go +++ b/api/scanner/cleanup_media.go @@ -12,7 +12,7 @@ import ( "gorm.io/gorm" ) -func CleanupMedia(db *gorm.DB, albumId int, albumMedia []*models.Media) []error { +func CleanupMedia(db *gorm.DB, albumId uint, albumMedia []*models.Media) []error { albumMediaIds := make([]interface{}, len(albumMedia)) for i, photo := range albumMedia { albumMediaIds[i] = photo.MediaID diff --git a/api/scanner/periodic_scanner.go b/api/scanner/periodic_scanner.go index c8a4152b..c4984eb0 100644 --- a/api/scanner/periodic_scanner.go +++ b/api/scanner/periodic_scanner.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "github.com/viktorstrate/photoview/api/graphql/models" "gorm.io/gorm" ) @@ -18,14 +19,13 @@ type periodicScanner struct { var mainPeriodicScanner *periodicScanner = nil func getPeriodicScanInterval(db *gorm.DB) (time.Duration, error) { - row := db.QueryRow("SELECT periodic_scan_interval FROM site_info") - var intervalSeconds int - if err := row.Scan(&intervalSeconds); err != nil { + var siteInfo models.SiteInfo + if err := db.First(&siteInfo).Error; err != nil { return 0, err } - return time.Duration(intervalSeconds) * time.Second, nil + return time.Duration(siteInfo.PeriodicScanInterval) * time.Second, nil } func InitializePeriodicScanner(db *gorm.DB) error { diff --git a/api/scanner/process_photo.go b/api/scanner/process_photo.go index 4a5ba70a..c86b4d6d 100644 --- a/api/scanner/process_photo.go +++ b/api/scanner/process_photo.go @@ -11,6 +11,7 @@ import ( "github.com/pkg/errors" "github.com/viktorstrate/photoview/api/graphql/models" "github.com/viktorstrate/photoview/api/utils" + "gorm.io/gorm" // Image decoders _ "image/gif" @@ -42,7 +43,7 @@ func makePhotoURLChecker(tx *sql.Tx, mediaID int) (func(purpose models.MediaPurp }, nil } -func ProcessMedia(tx *sql.Tx, media *models.Media) (bool, error) { +func ProcessMedia(tx *gorm.DB, media *models.Media) (bool, error) { imageData := EncodeMediaData{ media: media, } @@ -65,7 +66,7 @@ func ProcessMedia(tx *sql.Tx, media *models.Media) (bool, error) { } } -func processPhoto(tx *sql.Tx, imageData *EncodeMediaData, photoCachePath *string) (bool, error) { +func processPhoto(tx *gorm.DB, imageData *EncodeMediaData, photoCachePath *string) (bool, error) { photo := imageData.media diff --git a/api/scanner/queue.go b/api/scanner/queue.go index 619661ba..1b7af570 100644 --- a/api/scanner/queue.go +++ b/api/scanner/queue.go @@ -196,7 +196,7 @@ func (queue *ScannerQueue) jobOnQueue(job *ScannerJob) (bool, error) { scannerJobs := append(queue.in_progress, queue.up_next...) for _, scannerJob := range scannerJobs { - if scannerJob.album.AlbumID == job.album.AlbumID { + if scannerJob.album.ID == job.album.ID { return true, nil } } diff --git a/api/scanner/scanner_album.go b/api/scanner/scanner_album.go index 4c0aa6fe..a150db00 100644 --- a/api/scanner/scanner_album.go +++ b/api/scanner/scanner_album.go @@ -6,6 +6,7 @@ import ( "path" "time" + "github.com/pkg/errors" "github.com/viktorstrate/photoview/api/graphql/models" "github.com/viktorstrate/photoview/api/graphql/notification" "github.com/viktorstrate/photoview/api/utils" @@ -37,37 +38,35 @@ func scanAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.DB) { album_has_changes := false for count, photo := range albumPhotos { - tx, err := db.Begin() - if err != nil { - ScannerError("Failed to begin database transaction: %s", err) - } + // tx, err := db.Begin() - processing_was_needed, err := ProcessMedia(tx, photo) - if err != nil { - tx.Rollback() - ScannerError("Failed to process photo (%s): %s", photo.Path, err) - continue - } + transactionResult := db.Transaction(func(tx *gorm.DB) error { + processing_was_needed, err := ProcessMedia(tx, photo) + if err != nil { + return errors.Wrapf(err, "failed to process photo (%s)", photo.Path) + } - if processing_was_needed { - album_has_changes = true - progress := float64(count) / float64(len(albumPhotos)) * 100.0 - notification.BroadcastNotification(&models.Notification{ - Key: album_notify_key, - Type: models.NotificationTypeProgress, - Header: fmt.Sprintf("Processing media for album '%s'", album.Title), - Content: fmt.Sprintf("Processed media at %s", photo.Path), - Progress: &progress, - }) - } + if processing_was_needed { + album_has_changes = true + progress := float64(count) / float64(len(albumPhotos)) * 100.0 + notification.BroadcastNotification(&models.Notification{ + Key: album_notify_key, + Type: models.NotificationTypeProgress, + Header: fmt.Sprintf("Processing media for album '%s'", album.Title), + Content: fmt.Sprintf("Processed media at %s", photo.Path), + Progress: &progress, + }) + } - err = tx.Commit() - if err != nil { - ScannerError("Failed to commit database transaction: %s", err) + return nil + }) + + if transactionResult.Error != nil { + ScannerError("Failed to begin database transaction: %s", transactionResult.Error) } } - cleanup_errors := CleanupMedia(db, album.AlbumID, albumPhotos) + cleanup_errors := CleanupMedia(db, album.ID, albumPhotos) for _, err := range cleanup_errors { ScannerError("Failed to delete old media: %s", err) } @@ -98,24 +97,20 @@ func findMediaForAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.D photoPath := path.Join(album.Path, item.Name()) if !item.IsDir() && isPathMedia(photoPath, cache) { - tx, err := db.Begin() - if err != nil { - ScannerError("Could not begin database transaction for image %s: %s\n", photoPath, err) - continue - } - photo, isNewPhoto, err := ScanMedia(tx, photoPath, album.AlbumID, cache) - if err != nil { - ScannerError("Scanning media error (%s): %s", photoPath, err) - tx.Rollback() - continue - } + db.Transaction(func(tx *gorm.DB) error { + photo, isNewPhoto, err := ScanMedia(tx, photoPath, album.ID, cache) + if err != nil { + return errors.Wrapf(err, "Scanning media error (%s)", photoPath) + } - onScanPhoto(photo, isNewPhoto) + onScanPhoto(photo, isNewPhoto) - albumPhotos = append(albumPhotos, photo) + albumPhotos = append(albumPhotos, photo) + + return nil + }) - tx.Commit() } } diff --git a/api/scanner/scanner_media.go b/api/scanner/scanner_media.go index 39f19b76..aba36bde 100644 --- a/api/scanner/scanner_media.go +++ b/api/scanner/scanner_media.go @@ -11,6 +11,7 @@ import ( "github.com/pkg/errors" "github.com/viktorstrate/photoview/api/graphql/models" + "gorm.io/gorm" ) func scanForSideCarFile(path string) *string { @@ -47,7 +48,7 @@ func hashSideCarFile(path *string) *string { return &hash } -func ScanMedia(tx *sql.Tx, mediaPath string, albumId int, cache *AlbumScannerCache) (*models.Media, bool, error) { +func ScanMedia(tx *gorm.DB, mediaPath string, albumId uint, cache *AlbumScannerCache) (*models.Media, bool, error) { mediaName := path.Base(mediaPath) // Check if image already exists