public/photoview
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/photoview.git synced 2026-08-03 21:59:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace database, mostly process photo

Browse Source
This commit is contained in:
viktorstrate
2020-11-24 11:46:49 +01:00
parent 2e9aa29258
commit cc5d323b2a
6 changed files with 117 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
api/graphql/models/media.go
View File

@@ -40,13 +40,14 @@ const (
type MediaURL struct {
gorm.Model
MediaID int
MediaID uint
Media Media
MediaName string
Width int
Height int
Purpose MediaPurpose
ContentType string
FileSize int
FileSize int64
}
func (p *MediaURL) URL() string {

5
api/scanner/encode_photo.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/viktorstrate/photoview/api/graphql/models"
"github.com/viktorstrate/photoview/api/utils"
"gopkg.in/vansante/go-ffprobe.v2"
"gorm.io/gorm"
)
type PhotoDimensions struct {
@@ -116,7 +117,7 @@ func (img *EncodeMediaData) ContentType() (*MediaType, error) {
return imgType, nil
}
func (img *EncodeMediaData) EncodeHighRes(tx *sql.Tx, outputPath string) error {
func (img *EncodeMediaData) EncodeHighRes(tx *gorm.DB, outputPath string) error {
contentType, err := img.ContentType()
if err != nil {
return err
@@ -165,7 +166,7 @@ func EncodeThumbnail(inputPath string, outputPath string) (*PhotoDimensions, err
}
// PhotoImage reads and decodes the image file and saves it in a cache so the photo in only decoded once
func (img *EncodeMediaData) photoImage(tx *sql.Tx) (image.Image, error) {
func (img *EncodeMediaData) photoImage(tx *gorm.DB) (image.Image, error) {
if img._photoImage != nil {
return img._photoImage, nil
}

113
api/scanner/exif.go
View File

@@ -1,7 +1,6 @@
package scanner
import (
"database/sql"
"fmt"
"log"
"math/big"
@@ -30,14 +29,6 @@ func ScanEXIF(tx *gorm.DB, media *models.Media) (returnExif *models.MediaEXIF, r
return &exif, nil
}
row := tx.QueryRow("SELECT media_exif.* FROM media, media_exif WHERE media.exif_id = media_exif.exif_id AND media.media_id = ?", media.MediaID)
exifData, err := models.NewMediaExifFromRow(row)
if err != nil && err != sql.ErrNoRows {
return nil, err
} else if exifData != nil {
return exifData, nil
}
}
photoFile, err := os.Open(media.Path)
@@ -60,49 +51,38 @@ func ScanEXIF(tx *gorm.DB, media *models.Media) (returnExif *models.MediaEXIF, r
return nil, errors.Wrap(err, "Could not decode EXIF")
}
valueNames := make([]string, 0)
exifValues := make([]interface{}, 0)
newExif := models.MediaEXIF{}
model, err := readStringTag(exifTags, exif.Model, media)
if err == nil {
valueNames = append(valueNames, "camera")
exifValues = append(exifValues, model)
newExif.Camera = model
}
maker, err := readStringTag(exifTags, exif.Make, media)
if err == nil {
valueNames = append(valueNames, "maker")
exifValues = append(exifValues, maker)
newExif.Maker = maker
}
lens, err := readStringTag(exifTags, exif.LensModel, media)
if err == nil {
valueNames = append(valueNames, "lens")
exifValues = append(exifValues, lens)
newExif.Lens = lens
}
date, err := exifTags.DateTime()
if err == nil {
valueNames = append(valueNames, "date_shot")
exifValues = append(exifValues, date)
_, err := tx.Exec("UPDATE media SET date_shot = ? WHERE media_id = ?", date, media.MediaID)
if err != nil {
log.Printf("WARN: Failed to update date_shot for media %s: %s", media.Title, err)
}
newExif.DateShot = &date
}
exposure, err := readRationalTag(exifTags, exif.ExposureTime, media)
if err == nil {
valueNames = append(valueNames, "exposure")
exifValues = append(exifValues, exposure.RatString())
exposureStr := exposure.RatString()
newExif.Exposure = &exposureStr
}
apertureRat, err := readRationalTag(exifTags, exif.FNumber, media)
if err == nil {
aperture, _ := apertureRat.Float32()
valueNames = append(valueNames, "aperture")
exifValues = append(exifValues, aperture)
aperture, _ := apertureRat.Float64()
newExif.Aperture = &aperture
}
isoTag, err := exifTags.Get(exif.ISOSpeedRatings)
@@ -113,8 +93,7 @@ func ScanEXIF(tx *gorm.DB, media *models.Media) (returnExif *models.MediaEXIF, r
if err != nil {
log.Printf("WARN: Could not parse EXIF ISOSpeedRatings as integer: %s\n", media.Title)
} else {
valueNames = append(valueNames, "iso")
exifValues = append(exifValues, iso)
newExif.Iso = &iso
}
}
@@ -122,9 +101,9 @@ func ScanEXIF(tx *gorm.DB, media *models.Media) (returnExif *models.MediaEXIF, r
if err == nil {
focalLengthRat, err := focalLengthTag.Rat(0)
if err == nil {
focalLength, _ := focalLengthRat.Float32()
valueNames = append(valueNames, "focal_length")
exifValues = append(exifValues, focalLength)
focalLength, _ := focalLengthRat.Float64()
newExif.FocalLength = &focalLength
} else {
// For some photos, the focal length cannot be read as a rational value,
// but is instead the second value read as an integer
@@ -134,8 +113,8 @@ func ScanEXIF(tx *gorm.DB, media *models.Media) (returnExif *models.MediaEXIF, r
if err != nil {
log.Printf("WARN: Could not parse EXIF FocalLength as rational or integer: %s\n%s\n", media.Title, err)
} else {
valueNames = append(valueNames, "focal_length")
exifValues = append(exifValues, focalLength)
focalLenFloat := float64(focalLength)
newExif.FocalLength = &focalLenFloat
}
}
}
@@ -143,76 +122,36 @@ func ScanEXIF(tx *gorm.DB, media *models.Media) (returnExif *models.MediaEXIF, r
flash, err := exifTags.Flash()
if err == nil {
valueNames = append(valueNames, "flash")
exifValues = append(exifValues, flash)
newExif.Flash = &flash
}
orientation, err := readIntegerTag(exifTags, exif.Orientation, media)
if err == nil {
valueNames = append(valueNames, "orientation")
exifValues = append(exifValues, *orientation)
newExif.Orientation = orientation
}
exposureProgram, err := readIntegerTag(exifTags, exif.ExposureProgram, media)
if err == nil {
valueNames = append(valueNames, "exposure_program")
exifValues = append(exifValues, *exposureProgram)
newExif.ExposureProgram = exposureProgram
}
lat, long, err := exifTags.LatLong()
if err == nil {
valueNames = append(valueNames, "gps_latitude")
exifValues = append(exifValues, lat)
valueNames = append(valueNames, "gps_longitude")
exifValues = append(exifValues, long)
newExif.GPSLatitude = &lat
newExif.GPSLonitude = &long
}
if len(valueNames) == 0 {
// If exif is empty
if newExif == (models.MediaEXIF{}) {
return nil, nil
}
prepareQuestions := ""
for range valueNames {
prepareQuestions += "?,"
}
prepareQuestions = prepareQuestions[0 : len(prepareQuestions)-1]
columns := ""
for _, name := range valueNames {
columns += name + ","
}
columns = columns[0 : len(columns)-1]
// Insert into database
result, err := tx.Exec("INSERT INTO media_exif ("+columns+") VALUES ("+prepareQuestions+")", exifValues...)
if err != nil {
return nil, err
// Add EXIF to database and link to media
if err := tx.Model(&media).Association("Exif").Replace(newExif); err != nil {
return nil, errors.Wrap(err, "save media exif to database")
}
exifID, err := result.LastInsertId()
if err != nil {
return nil, err
}
// Link exif to media in database
result, err = tx.Exec("UPDATE media SET exif_id = ? WHERE media_id = ?", exifID, media.MediaID)
if err != nil {
return nil, err
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return nil, errors.Wrap(err, "linking exif to media in database failed")
}
if rowsAffected == 0 {
return nil, errors.New("linking exif to media in database failed: 0 rows affected")
}
// Return newly created exif row
row := tx.QueryRow("SELECT * FROM media_exif WHERE exif_id = ?", exifID)
return models.NewMediaExifFromRow(row)
return &newExif, nil
}
func readStringTag(tags *exif.Exif, name exif.FieldName, media *models.Media) (*string, error) {

125
api/scanner/process_photo.go
View File

@@ -1,7 +1,6 @@
package scanner
import (
"database/sql"
"fmt"
"log"
"os"
@@ -23,23 +22,18 @@ import (
)
// Higher order function used to check if MediaURL for a given MediaPurpose exists
func makePhotoURLChecker(tx *sql.Tx, mediaID int) (func(purpose models.MediaPurpose) (*models.MediaURL, error), error) {
mediaURLExistsStmt, err := tx.Prepare("SELECT * FROM media_url WHERE media_id = ? AND purpose = ?")
if err != nil {
return nil, err
}
func makePhotoURLChecker(tx *gorm.DB, mediaID uint) (func(purpose models.MediaPurpose) (*models.MediaURL, error), error) {
return func(purpose models.MediaPurpose) (*models.MediaURL, error) {
row := mediaURLExistsStmt.QueryRow(mediaID, purpose)
mediaURL, err := models.NewMediaURLFromRow(row)
if err != nil {
if err == sql.ErrNoRows {
var mediaURL models.MediaURL
if err := tx.Where("purpose = ?", purpose).First(&mediaURL, mediaID).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return mediaURL, nil
return &mediaURL, nil
}, nil
}
@@ -74,7 +68,7 @@ func processPhoto(tx *gorm.DB, imageData *EncodeMediaData, photoCachePath *strin
didProcess := false
photoUrlFromDB, err := makePhotoURLChecker(tx, photo.MediaID)
photoUrlFromDB, err := makePhotoURLChecker(tx, photo.ID)
if err != nil {
return false, err
}
@@ -129,7 +123,7 @@ func processPhoto(tx *gorm.DB, imageData *EncodeMediaData, photoCachePath *strin
baseImagePath = path.Join(*photoCachePath, highres_name)
err = generateSaveHighResJPEG(tx, photo.MediaID, imageData, highres_name, baseImagePath, -1)
err = generateSaveHighResJPEG(tx, photo, imageData, highres_name, baseImagePath, nil)
if err != nil {
return false, err
}
@@ -179,7 +173,7 @@ func processPhoto(tx *gorm.DB, imageData *EncodeMediaData, photoCachePath *strin
// return err
// }
err = generateSaveThumbnailJPEG(tx, photo.MediaID, thumbnail_name, photoCachePath, baseImagePath, -1)
err = generateSaveThumbnailJPEG(tx, photo, thumbnail_name, photoCachePath, baseImagePath, nil)
if err != nil {
return false, err
}
@@ -211,7 +205,7 @@ func makeMediaCacheDir(photo *models.Media) (*string, error) {
}
// Make album cache dir if not exists
albumCachePath := path.Join(PhotoCache(), strconv.Itoa(photo.AlbumId))
albumCachePath := path.Join(PhotoCache(), strconv.Itoa(int(photo.ID)))
if _, err := os.Stat(albumCachePath); os.IsNotExist(err) {
if err := os.Mkdir(albumCachePath, os.ModePerm); err != nil {
return nil, errors.Wrap(err, "could not make album image cache directory")
@@ -219,7 +213,7 @@ func makeMediaCacheDir(photo *models.Media) (*string, error) {
}
// Make photo cache dir if not exists
photoCachePath := path.Join(albumCachePath, strconv.Itoa(photo.MediaID))
photoCachePath := path.Join(albumCachePath, strconv.Itoa(int(photo.ID)))
if _, err := os.Stat(photoCachePath); os.IsNotExist(err) {
if err := os.Mkdir(photoCachePath, os.ModePerm); err != nil {
return nil, errors.Wrap(err, "could not make photo image cache directory")
@@ -229,7 +223,7 @@ func makeMediaCacheDir(photo *models.Media) (*string, error) {
return &photoCachePath, nil
}
func saveOriginalPhotoToDB(tx *sql.Tx, photo *models.Media, imageData *EncodeMediaData, photoDimensions *PhotoDimensions) error {
func saveOriginalPhotoToDB(tx *gorm.DB, photo *models.Media, imageData *EncodeMediaData, photoDimensions *PhotoDimensions) error {
photoName := path.Base(photo.Path)
photoBaseName := photoName[0 : len(photoName)-len(path.Ext(photoName))]
photoBaseExt := path.Ext(photoName)
@@ -247,16 +241,24 @@ func saveOriginalPhotoToDB(tx *sql.Tx, photo *models.Media, imageData *EncodeMed
return errors.Wrap(err, "reading file stats of original photo")
}
_, err = tx.Exec("INSERT INTO media_url (media_id, media_name, width, height, purpose, content_type, file_size) VALUES (?, ?, ?, ?, ?, ?, ?)", photo.MediaID, original_image_name, photoDimensions.Width, photoDimensions.Height, models.MediaOriginal, contentType, fileStats.Size())
if err != nil {
log.Printf("Could not insert original photo url: %d, %s\n", photo.MediaID, photoName)
return err
mediaURL := models.MediaURL{
Media: *photo,
MediaName: original_image_name,
Width: photoDimensions.Width,
Height: photoDimensions.Height,
Purpose: models.MediaOriginal,
ContentType: string(*contentType),
FileSize: fileStats.Size(),
}
if err := tx.Create(&mediaURL).Error; err != nil {
return errors.Wrapf(err, "inserting original photo url: %d, %s", photo.ID, photoName)
}
return nil
}
func generateSaveHighResJPEG(tx *sql.Tx, mediaID int, imageData *EncodeMediaData, highres_name string, imagePath string, urlID int) error {
func generateSaveHighResJPEG(tx *gorm.DB, media *models.Media, imageData *EncodeMediaData, highres_name string, imagePath string, mediaURL *models.MediaURL) error {
err := imageData.EncodeHighRes(tx, imagePath)
if err != nil {
@@ -273,21 +275,35 @@ func generateSaveHighResJPEG(tx *sql.Tx, mediaID int, imageData *EncodeMediaData
return errors.Wrap(err, "reading file stats of highres photo")
}
if urlID < 0 {
_, err = tx.Exec("INSERT INTO media_url (media_id, media_name, width, height, purpose, content_type, file_size) VALUES (?, ?, ?, ?, ?, ?, ?)",
mediaID, highres_name, photoDimensions.Width, photoDimensions.Height, models.PhotoHighRes, "image/jpeg", fileStats.Size())
if mediaURL == nil {
mediaURL = &models.MediaURL{
MediaID: media.ID,
MediaName: highres_name,
Width: photoDimensions.Width,
Height: photoDimensions.Height,
Purpose: models.PhotoHighRes,
ContentType: "image/jpeg",
FileSize: fileStats.Size(),
}
if err := tx.Create(&mediaURL).Error; err != nil {
return errors.Wrapf(err, "could not insert highres media url (%d, %s)", media.ID, highres_name)
}
} else {
_, err = tx.Exec("UPDATE media_url SET width = ?, height = ?, file_size= ? WHERE url_id = ?",
photoDimensions.Width, photoDimensions.Height, fileStats.Size(), urlID)
}
if err != nil {
return errors.Wrapf(err, "could not insert highres media url (%d, %s)", mediaID, highres_name)
mediaURL.Width = photoDimensions.Width
mediaURL.Height = photoDimensions.Height
mediaURL.FileSize = fileStats.Size()
if err := tx.Save(&mediaURL).Error; err != nil {
return errors.Wrapf(err, "could not update media url after side car changes (%d, %s)", media.ID, highres_name)
}
}
return nil
}
func generateSaveThumbnailJPEG(tx *sql.Tx, mediaID int, thumbnail_name string, photoCachePath *string, baseImagePath string, urlID int) error {
func generateSaveThumbnailJPEG(tx *gorm.DB, media *models.Media, thumbnail_name string, photoCachePath *string, baseImagePath string, mediaURL *models.MediaURL) error {
thumbOutputPath := path.Join(*photoCachePath, thumbnail_name)
thumbSize, err := EncodeThumbnail(baseImagePath, thumbOutputPath)
@@ -300,20 +316,35 @@ func generateSaveThumbnailJPEG(tx *sql.Tx, mediaID int, thumbnail_name string, p
return errors.Wrap(err, "reading file stats of thumbnail photo")
}
if urlID < 0 {
_, err = tx.Exec("INSERT INTO media_url (media_id, media_name, width, height, purpose, content_type, file_size) VALUES (?, ?, ?, ?, ?, ?, ?)",
mediaID, thumbnail_name, thumbSize.Width, thumbSize.Height, models.PhotoThumbnail, "image/jpeg", fileStats.Size())
if mediaURL == nil {
mediaURL = &models.MediaURL{
MediaID: media.ID,
MediaName: thumbnail_name,
Width: thumbSize.Width,
Height: thumbSize.Height,
Purpose: models.PhotoThumbnail,
ContentType: "image/jpeg",
FileSize: fileStats.Size(),
}
if err := tx.Create(&mediaURL).Error; err != nil {
return errors.Wrapf(err, "could not insert thumbnail media url (%d, %s)", media.ID, thumbnail_name)
}
} else {
_, err = tx.Exec("UPDATE media_url SET width = ?, height = ?, file_size= ? WHERE url_id = ?",
thumbSize.Width, thumbSize.Height, fileStats.Size(), urlID)
}
if err != nil {
return err
mediaURL.Width = thumbSize.Width
mediaURL.Height = thumbSize.Height
mediaURL.FileSize = fileStats.Size()
if err := tx.Save(&mediaURL).Error; err != nil {
return errors.Wrapf(err, "could not update media url after side car changes (%d, %s)", media.ID, thumbnail_name)
}
}
return nil
}
func processRawSideCar(tx *sql.Tx, imageData *EncodeMediaData, highResURL *models.MediaURL, thumbURL *models.MediaURL, photoCachePath *string) error {
func processRawSideCar(tx *gorm.DB, imageData *EncodeMediaData, highResURL *models.MediaURL, thumbURL *models.MediaURL, photoCachePath *string) error {
photo := imageData.media
sideCarFileHasChanged := false
var currentFileHash *string
@@ -327,6 +358,7 @@ func processRawSideCar(tx *sql.Tx, imageData *EncodeMediaData, highResURL *model
} else if photo.SideCarPath != nil { // sidecar has been deleted since last scan
sideCarFileHasChanged = true
}
if sideCarFileHasChanged {
fmt.Printf("Detected changed sidecar file for %s recreating JPG's to reflect changes\n", photo.Path)
@@ -334,7 +366,7 @@ func processRawSideCar(tx *sql.Tx, imageData *EncodeMediaData, highResURL *model
baseImagePath := path.Join(*photoCachePath, highResURL.MediaName) // update base image path for thumbnail
tempHighResPath := baseImagePath + ".hold"
os.Rename(baseImagePath, tempHighResPath)
err := generateSaveHighResJPEG(tx, photo.MediaID, imageData, highResURL.MediaName, baseImagePath, highResURL.UrlID)
err := generateSaveHighResJPEG(tx, photo, imageData, highResURL.MediaName, baseImagePath, highResURL)
if err != nil {
os.Rename(tempHighResPath, baseImagePath)
return errors.Wrap(err, "recreating high-res cached image")
@@ -345,18 +377,21 @@ func processRawSideCar(tx *sql.Tx, imageData *EncodeMediaData, highResURL *model
thumbPath := path.Join(*photoCachePath, thumbURL.MediaName)
tempThumbPath := thumbPath + ".hold" // hold onto the original image incase for some reason we fail to recreate one with the new settings
os.Rename(thumbPath, tempThumbPath)
err = generateSaveThumbnailJPEG(tx, photo.MediaID, thumbURL.MediaName, photoCachePath, baseImagePath, thumbURL.UrlID)
err = generateSaveThumbnailJPEG(tx, photo, thumbURL.MediaName, photoCachePath, baseImagePath, thumbURL)
if err != nil {
os.Rename(tempThumbPath, thumbPath)
return errors.Wrap(err, "recreating thumbnail cached image")
}
os.Remove(tempThumbPath)
photo.SideCarHash = currentFileHash
photo.SideCarPath = currentSideCarPath
// save new side car hash
_, err = tx.Exec("UPDATE media SET side_car_hash = ?, side_car_path = ? WHERE media_id = ?", currentFileHash, currentSideCarPath, photo.MediaID)
if err != nil {
if err := tx.Save(&photo).Error; err != nil {
return errors.Wrapf(err, "could not update side car hash for media: %s", photo.Path)
}
}
return nil
}

6
api/scanner/process_video.go
View File

@@ -2,7 +2,6 @@ package scanner
import (
"context"
"database/sql"
"fmt"
"log"
"os"
@@ -14,15 +13,16 @@ import (
"github.com/viktorstrate/photoview/api/graphql/models"
"github.com/viktorstrate/photoview/api/utils"
"gopkg.in/vansante/go-ffprobe.v2"
"gorm.io/gorm"
)
func processVideo(tx *sql.Tx, mediaData *EncodeMediaData, videoCachePath *string) (bool, error) {
func processVideo(tx *gorm.DB, mediaData *EncodeMediaData, videoCachePath *string) (bool, error) {
video := mediaData.media
didProcess := false
log.Printf("Processing video: %s", video.Path)
mediaUrlFromDB, err := makePhotoURLChecker(tx, video.MediaID)
mediaUrlFromDB, err := makePhotoURLChecker(tx, video.ID)
if err != nil {
return false, err
}

4
api/scanner/video_metadata.go
View File

@@ -1,16 +1,16 @@
package scanner
import (
"database/sql"
"fmt"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/viktorstrate/photoview/api/graphql/models"
"gorm.io/gorm"
)
func ScanVideoMetadata(tx *sql.Tx, video *models.Media) error {
func ScanVideoMetadata(tx *gorm.DB, video *models.Media) error {
data, err := readVideoMetadata(video.Path)
if err != nil {