From 2e9aa29258a8166e34b4b7953ae1c82593ea11a3 Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Mon, 23 Nov 2020 20:43:00 +0100 Subject: [PATCH] Replace database, mostly media related --- api/database/database.go | 2 + api/graphql/models/media.go | 71 ++++------------------------ api/scanner/cleanup_media.go | 90 ++++++++++++------------------------ api/scanner/encode_photo.go | 2 +- api/scanner/exif.go | 12 +++-- api/scanner/scanner_media.go | 50 +++++++++----------- 6 files changed, 73 insertions(+), 154 deletions(-) diff --git a/api/database/database.go b/api/database/database.go index 103307b2..35edf992 100644 --- a/api/database/database.go +++ b/api/database/database.go @@ -75,6 +75,8 @@ func MigrateDatabase(db *gorm.DB) error { &models.User{}, &models.AccessToken{}, &models.SiteInfo{}, + &models.Media{}, + &models.MediaURL{}, &models.Album{}, &models.MediaEXIF{}, ) diff --git a/api/graphql/models/media.go b/api/graphql/models/media.go index 6e0f8452..4c649c21 100644 --- a/api/graphql/models/media.go +++ b/api/graphql/models/media.go @@ -1,21 +1,24 @@ package models import ( - "database/sql" "path" "strings" "time" "github.com/viktorstrate/photoview/api/utils" + "gorm.io/gorm" ) type Media struct { - MediaID int + gorm.Model Title string Path string PathHash string - AlbumId int - ExifId *int + AlbumId uint + Album Album + ExifId *uint + Exif MediaEXIF + MediaURL []MediaURL DateShot time.Time DateImported time.Time Favorite bool @@ -25,10 +28,6 @@ type Media struct { SideCarHash *string } -func (p *Media) ID() int { - return p.MediaID -} - type MediaPurpose string const ( @@ -40,8 +39,8 @@ const ( ) type MediaURL struct { - UrlID int - MediaId int + gorm.Model + MediaID int MediaName string Width int Height int @@ -50,32 +49,6 @@ type MediaURL struct { FileSize int } -func NewMediaFromRow(row *sql.Row) (*Media, error) { - media := Media{} - - if err := row.Scan(&media.MediaID, &media.Title, &media.Path, &media.PathHash, &media.AlbumId, &media.ExifId, &media.DateShot, &media.DateImported, &media.Favorite, &media.Type, &media.VideoMetadataId, &media.SideCarPath, &media.SideCarHash); err != nil { - return nil, err - } - - return &media, nil -} - -func NewMediaFromRows(rows *sql.Rows) ([]*Media, error) { - medias := make([]*Media, 0) - - for rows.Next() { - var media Media - if err := rows.Scan(&media.MediaID, &media.Title, &media.Path, &media.PathHash, &media.AlbumId, &media.ExifId, &media.DateShot, &media.DateImported, &media.Favorite, &media.Type, &media.VideoMetadataId, &media.SideCarPath, &media.SideCarHash); err != nil { - return nil, err - } - medias = append(medias, &media) - } - - rows.Close() - - return medias, nil -} - func (p *MediaURL) URL() string { imageUrl := utils.ApiEndpointUrl() @@ -96,29 +69,3 @@ func SanitizeMediaName(mediaName string) string { result = strings.ReplaceAll(result, ".", "_") return result } - -func NewMediaURLFromRow(row *sql.Row) (*MediaURL, error) { - url := MediaURL{} - - if err := row.Scan(&url.UrlID, &url.MediaId, &url.MediaName, &url.Width, &url.Height, &url.Purpose, &url.ContentType, &url.FileSize); err != nil { - return nil, err - } - - return &url, nil -} - -func NewMediaURLFromRows(rows *sql.Rows) ([]*MediaURL, error) { - urls := make([]*MediaURL, 0) - - for rows.Next() { - var url MediaURL - if err := rows.Scan(&url.UrlID, &url.MediaId, &url.MediaName, &url.Width, &url.Height, &url.Purpose, &url.ContentType, &url.FileSize); err != nil { - return nil, err - } - urls = append(urls, &url) - } - - rows.Close() - - return urls, nil -} diff --git a/api/scanner/cleanup_media.go b/api/scanner/cleanup_media.go index b59f7546..a3aadd76 100644 --- a/api/scanner/cleanup_media.go +++ b/api/scanner/cleanup_media.go @@ -1,7 +1,6 @@ package scanner import ( - "database/sql" "os" "path" "strconv" @@ -13,61 +12,40 @@ import ( ) func CleanupMedia(db *gorm.DB, albumId uint, albumMedia []*models.Media) []error { - albumMediaIds := make([]interface{}, len(albumMedia)) - for i, photo := range albumMedia { - albumMediaIds[i] = photo.MediaID + albumMediaIds := make([]uint, len(albumMedia)) + for i, media := range albumMedia { + albumMediaIds[i] = media.ID } - // Delete missing media - var rows *sql.Rows - var err error + // Will get from database + var mediaList []models.Media + + db.Where("album_id = ?", albumId) // Select media from database that was not found on hard disk if len(albumMedia) > 0 { - media_args := make([]interface{}, 0) - media_args = append(media_args, albumId) - media_args = append(media_args, albumMediaIds...) - - media_questions := strings.Repeat("?,", len(albumMediaIds))[:len(albumMediaIds)*2-1] - rows, err = db.Query( - "SELECT media_id FROM media WHERE album_id = ? AND media_id NOT IN ("+media_questions+")", - media_args..., - ) - } else { - rows, err = db.Query( - "SELECT media_id FROM media WHERE album_id = ?", - albumId, - ) + db.Not(albumMediaIds) } - if err != nil { + + if err := db.Find(&mediaList).Error; err != nil { return []error{errors.Wrap(err, "get media files to be deleted from database")} } - defer rows.Close() deleteErrors := make([]error, 0) - deleted_media_ids := make([]interface{}, 0) - for rows.Next() { - var media_id int - if err := rows.Scan(&media_id); err != nil { - deleteErrors = append(deleteErrors, errors.Wrapf(err, "parse media to be removed (media_id %d)", media_id)) - continue + for _, media := range mediaList { + + // deletedMediaIDs = append(deletedMediaIDs, media.ID) + cachePath := path.Join(PhotoCache(), strconv.Itoa(int(albumId)), strconv.Itoa(int(media.ID))) + err := os.RemoveAll(cachePath) + if err != nil { + deleteErrors = append(deleteErrors, errors.Wrapf(err, "delete unused cache folder (%s)", cachePath)) } - deleted_media_ids = append(deleted_media_ids, media_id) - cache_path := path.Join(PhotoCache(), strconv.Itoa(albumId), strconv.Itoa(media_id)) - err := os.RemoveAll(cache_path) - if err != nil { - deleteErrors = append(deleteErrors, errors.Wrapf(err, "delete unused cache folder (%s)", cache_path)) - } } - if len(deleted_media_ids) > 0 { - media_questions := strings.Repeat("?,", len(deleted_media_ids))[:len(deleted_media_ids)*2-1] - - if _, err := db.Exec("DELETE FROM media WHERE media_id IN ("+media_questions+")", deleted_media_ids...); err != nil { - deleteErrors = append(deleteErrors, errors.Wrap(err, "delete old media from database")) - } + if err := db.Delete(&mediaList).Error; err != nil { + deleteErrors = append(deleteErrors, errors.Wrap(err, "delete old media from database")) } return deleteErrors @@ -85,41 +63,31 @@ func deleteOldUserAlbums(db *gorm.DB, scannedAlbums []*models.Album, user *model // Delete old albums album_args := make([]interface{}, 0) - album_args = append(album_args, user.UserID) + album_args = append(album_args, user.ID) album_args = append(album_args, albumPaths...) + var albums []models.Album + albums_questions := strings.Repeat("MD5(?),", len(albumPaths))[:len(albumPaths)*7-1] - rows, err := db.Query("SELECT album_id FROM album WHERE album.owner_id = ? AND path_hash NOT IN ("+albums_questions+")", album_args...) - if err != nil { + if err := db.Where("album.owner_id = ? AND path_hash NOT IN ("+albums_questions+")", album_args...).Find(&albums).Error; err != nil { return []error{errors.Wrap(err, "get albums to be deleted from database")} } - defer rows.Close() deleteErrors := make([]error, 0) deleted_album_ids := make([]interface{}, 0) - for rows.Next() { - var album_id int - if err := rows.Scan(&album_id); err != nil { - deleteErrors = append(deleteErrors, errors.Wrapf(err, "parse album to be removed (album_id %d)", album_id)) - continue - } - - deleted_album_ids = append(deleted_album_ids, album_id) - cache_path := path.Join("./photo_cache", strconv.Itoa(album_id)) + for _, album := range albums { + deleted_album_ids = append(deleted_album_ids, album.ID) + cache_path := path.Join("./photo_cache", strconv.Itoa(int(album.ID))) err := os.RemoveAll(cache_path) if err != nil { deleteErrors = append(deleteErrors, errors.Wrapf(err, "delete unused cache folder (%s)", cache_path)) } } - if len(deleted_album_ids) > 0 { - albums_questions = strings.Repeat("?,", len(deleted_album_ids))[:len(deleted_album_ids)*2-1] - - if _, err := db.Exec("DELETE FROM album WHERE album_id IN ("+albums_questions+")", deleted_album_ids...); err != nil { - ScannerError("Could not delete old albums from database:\n%s\n", err) - deleteErrors = append(deleteErrors, errors.Wrap(err, "delete old albums from database")) - } + if err := db.Delete(&albums).Error; err != nil { + ScannerError("Could not delete old albums from database:\n%s\n", err) + deleteErrors = append(deleteErrors, errors.Wrap(err, "delete old albums from database")) } return deleteErrors diff --git a/api/scanner/encode_photo.go b/api/scanner/encode_photo.go index 92847b07..ddfde4ec 100644 --- a/api/scanner/encode_photo.go +++ b/api/scanner/encode_photo.go @@ -176,7 +176,7 @@ func (img *EncodeMediaData) photoImage(tx *sql.Tx) (image.Image, error) { } // Get orientation from exif data - row := tx.QueryRow("SELECT media_exif.orientation FROM media JOIN media_exif WHERE media.exif_id = media_exif.exif_id AND media.media_id = ?", img.media.MediaID) + row := tx.QueryRow("SELECT media_exif.orientation FROM media JOIN media_exif WHERE media.exif_id = media_exif.exif_id AND media.media_id = ?", img.media.ID) var orientation *int if err = row.Scan(&orientation); err != nil { // If not found use default orientation (not rotate) diff --git a/api/scanner/exif.go b/api/scanner/exif.go index a38a9b76..9642da59 100644 --- a/api/scanner/exif.go +++ b/api/scanner/exif.go @@ -8,21 +8,27 @@ import ( "os" "github.com/pkg/errors" + "gorm.io/gorm" "github.com/viktorstrate/photoview/api/graphql/models" "github.com/xor-gate/goexif2/exif" "github.com/xor-gate/goexif2/mknote" ) -func ScanEXIF(tx *sql.Tx, media *models.Media) (returnExif *models.MediaEXIF, returnErr error) { +func ScanEXIF(tx *gorm.DB, media *models.Media) (returnExif *models.MediaEXIF, returnErr error) { log.Printf("Scanning for EXIF") { // Check if EXIF data already exists if media.ExifId != nil { - row := tx.QueryRow("SELECT * FROM media_exif WHERE exif_id = ?", media.ExifId) - return models.NewMediaExifFromRow(row) + + var exif models.MediaEXIF + if err := tx.First(&exif, media.ExifId).Error; err != nil { + return nil, errors.Wrap(err, "get EXIF for media from database") + } + + return &exif, nil } row := tx.QueryRow("SELECT media_exif.* FROM media, media_exif WHERE media.exif_id = media_exif.exif_id AND media.media_id = ?", media.MediaID) diff --git a/api/scanner/scanner_media.go b/api/scanner/scanner_media.go index aba36bde..904958d2 100644 --- a/api/scanner/scanner_media.go +++ b/api/scanner/scanner_media.go @@ -2,7 +2,6 @@ package scanner import ( "crypto/md5" - "database/sql" "encoding/hex" "io" "log" @@ -51,14 +50,13 @@ func hashSideCarFile(path *string) *string { func ScanMedia(tx *gorm.DB, mediaPath string, albumId uint, cache *AlbumScannerCache) (*models.Media, bool, error) { mediaName := path.Base(mediaPath) - // Check if image already exists + // Check if media already exists { - row := tx.QueryRow("SELECT * FROM media WHERE path_hash = MD5(?)", mediaPath) - photo, err := models.NewMediaFromRow(row) - if err != sql.ErrNoRows { + var media models.Media + if err := tx.Where("path_hash = MD5(?)", mediaPath).First(&media).Error; errors.Is(err, gorm.ErrRecordNotFound) { if err == nil { log.Printf("Media already scanned: %s\n", mediaPath) - return photo, false, nil + return &media, false, nil } else { return nil, false, errors.Wrap(err, "scan media fetch from database") } @@ -72,16 +70,15 @@ func ScanMedia(tx *gorm.DB, mediaPath string, albumId uint, cache *AlbumScannerC return nil, false, errors.Wrap(err, "could determine if media was photo or video") } - var mediaTypeText string + var mediaTypeText models.MediaType + + var sideCarPath *string = nil + var sideCarHash *string = nil - var sideCarPath *string - sideCarPath = nil - var sideCarHash *string - sideCarHash = nil if mediaType.isVideo() { - mediaTypeText = "video" + mediaTypeText = models.MediaTypeVideo } else { - mediaTypeText = "photo" + mediaTypeText = models.MediaTypePhoto // search for sidecar files if mediaType.isRaw() { sideCarPath = scanForSideCarFile(mediaPath) @@ -96,22 +93,21 @@ func ScanMedia(tx *gorm.DB, mediaPath string, albumId uint, cache *AlbumScannerC return nil, false, err } - result, err := tx.Exec("INSERT INTO media (title, path, path_hash, side_car_path, side_car_hash, album_id, media_type, date_shot) VALUES (?, ?, MD5(path), ?, ?, ?, ?, ?)", mediaName, mediaPath, sideCarPath, sideCarHash, albumId, mediaTypeText, stat.ModTime()) - if err != nil { + media := models.Media{ + Title: mediaName, + Path: mediaPath, + SideCarPath: sideCarPath, + SideCarHash: sideCarHash, + AlbumId: albumId, + Type: mediaTypeText, + DateShot: stat.ModTime(), + } + + if err := tx.Create(&media).Error; err != nil { return nil, false, errors.Wrap(err, "could not insert media into database") } - media_id, err := result.LastInsertId() - if err != nil { - return nil, false, err - } - row := tx.QueryRow("SELECT * FROM media WHERE media_id = ?", media_id) - media, err := models.NewMediaFromRow(row) - if err != nil { - return nil, false, errors.Wrap(err, "failed to get media by id from database") - } - - _, err = ScanEXIF(tx, media) + _, err = ScanEXIF(tx, &media) if err != nil { log.Printf("WARN: ScanEXIF for %s failed: %s\n", mediaName, err) } @@ -122,5 +118,5 @@ func ScanMedia(tx *gorm.DB, mediaPath string, albumId uint, cache *AlbumScannerC } } - return media, true, nil + return &media, true, nil }