diff --git a/api/database/migrations/002_site_info_more_settings.down.sql b/api/database/migrations/002_site_info_more_settings.down.sql index 06053213..1e441e58 100644 --- a/api/database/migrations/002_site_info_more_settings.down.sql +++ b/api/database/migrations/002_site_info_more_settings.down.sql @@ -1,4 +1,4 @@ ALTER TABLE site_info DROP COLUMN IF EXISTS periodic_scan_interval, - DrOP COLUMN IF EXISTS concurrent_workers; + DROP COLUMN IF EXISTS concurrent_workers; diff --git a/api/database/migrations/003_sar_side_car_files.down.sql b/api/database/migrations/003_sar_side_car_files.down.sql new file mode 100644 index 00000000..12005bf2 --- /dev/null +++ b/api/database/migrations/003_sar_side_car_files.down.sql @@ -0,0 +1,4 @@ + +ALTER TABLE media + DROP COLUMN IF EXISTS side_car_path, + DROP COLUMN IF EXISTS side_car_hash; diff --git a/api/database/migrations/003_sar_side_car_files.up.sql b/api/database/migrations/003_sar_side_car_files.up.sql new file mode 100644 index 00000000..635f3513 --- /dev/null +++ b/api/database/migrations/003_sar_side_car_files.up.sql @@ -0,0 +1,4 @@ + +ALTER TABLE media + ADD COLUMN IF NOT EXISTS side_car_path varchar(1024) DEFAULT NULL, + ADD COLUMN IF NOT EXISTS side_car_hash varchar(32) DEFAULT NULL; diff --git a/api/graphql/models/media.go b/api/graphql/models/media.go index 24b397cb..6e0f8452 100644 --- a/api/graphql/models/media.go +++ b/api/graphql/models/media.go @@ -21,6 +21,8 @@ type Media struct { Favorite bool Type MediaType VideoMetadataId *int + SideCarPath *string + SideCarHash *string } func (p *Media) ID() int { @@ -51,7 +53,7 @@ type MediaURL struct { func NewMediaFromRow(row *sql.Row) (*Media, error) { media := Media{} - if err := row.Scan(&media.MediaID, &media.Title, &media.Path, &media.PathHash, &media.AlbumId, &media.ExifId, &media.DateShot, &media.DateImported, &media.Favorite, &media.Type, &media.VideoMetadataId); err != nil { + if err := row.Scan(&media.MediaID, &media.Title, &media.Path, &media.PathHash, &media.AlbumId, &media.ExifId, &media.DateShot, &media.DateImported, &media.Favorite, &media.Type, &media.VideoMetadataId, &media.SideCarPath, &media.SideCarHash); err != nil { return nil, err } @@ -63,7 +65,7 @@ func NewMediaFromRows(rows *sql.Rows) ([]*Media, error) { for rows.Next() { var media Media - if err := rows.Scan(&media.MediaID, &media.Title, &media.Path, &media.PathHash, &media.AlbumId, &media.ExifId, &media.DateShot, &media.DateImported, &media.Favorite, &media.Type, &media.VideoMetadataId); err != nil { + if err := rows.Scan(&media.MediaID, &media.Title, &media.Path, &media.PathHash, &media.AlbumId, &media.ExifId, &media.DateShot, &media.DateImported, &media.Favorite, &media.Type, &media.VideoMetadataId, &media.SideCarPath, &media.SideCarHash); err != nil { return nil, err } medias = append(medias, &media) diff --git a/api/scanner/media_type.go b/api/scanner/media_type.go index 08819611..5acfca59 100644 --- a/api/scanner/media_type.go +++ b/api/scanner/media_type.go @@ -1,6 +1,7 @@ package scanner import ( + "fmt" "io" "log" "os" @@ -231,10 +232,14 @@ func getMediaType(path string) (*MediaType, error) { ext := filepath.Ext(path) - fileExtType := fileExtensions[strings.ToLower(ext)] + fileExtType, found := fileExtensions[strings.ToLower(ext)] - if fileExtType.isSupported() { - return &fileExtType, nil + if found { + if fileExtType.isSupported() { + return &fileExtType, nil + } else { + return nil, errors.New(fmt.Sprintf("unsupported file type '%s' (%s)", ext, fileExtType)) + } } // If extension was not recognized try to read file header diff --git a/api/scanner/process_photo.go b/api/scanner/process_photo.go index ff18c920..4a5ba70a 100644 --- a/api/scanner/process_photo.go +++ b/api/scanner/process_photo.go @@ -96,10 +96,22 @@ func processPhoto(tx *sql.Tx, imageData *EncodeMediaData, photoCachePath *string return false, errors.Wrap(err, "error processing photo highres") } - // Generate high res jpeg var photoDimensions *PhotoDimensions var baseImagePath string = photo.Path + mediaType, err := getMediaType(photo.Path) + if err != nil { + return false, errors.Wrap(err, "could determine if media was photo or video") + } + + if mediaType.isRaw() { + err = processRawSideCar(tx, imageData, highResURL, thumbURL, photoCachePath) + if err != nil { + return false, err + } + } + + // Generate high res jpeg if highResURL == nil { contentType, err := imageData.ContentType() @@ -116,26 +128,10 @@ func processPhoto(tx *sql.Tx, imageData *EncodeMediaData, photoCachePath *string baseImagePath = path.Join(*photoCachePath, highres_name) - err = imageData.EncodeHighRes(tx, baseImagePath) - if err != nil { - return false, errors.Wrap(err, "creating high-res cached image") - } - - photoDimensions, err = GetPhotoDimensions(baseImagePath) + err = generateSaveHighResJPEG(tx, photo.MediaID, imageData, highres_name, baseImagePath, -1) if err != nil { return false, err } - - fileStats, err := os.Stat(baseImagePath) - if err != nil { - return false, errors.Wrap(err, "reading file stats of highres photo") - } - - _, err = tx.Exec("INSERT INTO media_url (media_id, media_name, width, height, purpose, content_type, file_size) VALUES (?, ?, ?, ?, ?, ?, ?)", - photo.MediaID, highres_name, photoDimensions.Width, photoDimensions.Height, models.PhotoHighRes, "image/jpeg", fileStats.Size()) - if err != nil { - return false, errors.Wrapf(err, "could not insert highres media url (%d, %s)", photo.MediaID, photo.Title) - } } } else { // Verify that highres photo still exists in cache @@ -182,19 +178,7 @@ func processPhoto(tx *sql.Tx, imageData *EncodeMediaData, photoCachePath *string // return err // } - thumbOutputPath := path.Join(*photoCachePath, thumbnail_name) - - thumbSize, err := EncodeThumbnail(baseImagePath, thumbOutputPath) - if err != nil { - return false, errors.Wrap(err, "could not create thumbnail cached image") - } - - fileStats, err := os.Stat(thumbOutputPath) - if err != nil { - return false, errors.Wrap(err, "reading file stats of thumbnail photo") - } - - _, err = tx.Exec("INSERT INTO media_url (media_id, media_name, width, height, purpose, content_type, file_size) VALUES (?, ?, ?, ?, ?, ?, ?)", photo.MediaID, thumbnail_name, thumbSize.Width, thumbSize.Height, models.PhotoThumbnail, "image/jpeg", fileStats.Size()) + err = generateSaveThumbnailJPEG(tx, photo.MediaID, thumbnail_name, photoCachePath, baseImagePath, -1) if err != nil { return false, err } @@ -270,3 +254,108 @@ func saveOriginalPhotoToDB(tx *sql.Tx, photo *models.Media, imageData *EncodeMed return nil } + +func generateSaveHighResJPEG(tx *sql.Tx, mediaID int, imageData *EncodeMediaData, highres_name string, imagePath string, urlID int) error { + + err := imageData.EncodeHighRes(tx, imagePath) + if err != nil { + return errors.Wrap(err, "creating high-res cached image") + } + + photoDimensions, err := GetPhotoDimensions(imagePath) + if err != nil { + return err + } + + fileStats, err := os.Stat(imagePath) + if err != nil { + return errors.Wrap(err, "reading file stats of highres photo") + } + + if urlID < 0 { + _, err = tx.Exec("INSERT INTO media_url (media_id, media_name, width, height, purpose, content_type, file_size) VALUES (?, ?, ?, ?, ?, ?, ?)", + mediaID, highres_name, photoDimensions.Width, photoDimensions.Height, models.PhotoHighRes, "image/jpeg", fileStats.Size()) + } else { + _, err = tx.Exec("UPDATE media_url SET width = ?, height = ?, file_size= ? WHERE url_id = ?", + photoDimensions.Width, photoDimensions.Height, fileStats.Size(), urlID) + } + if err != nil { + return errors.Wrapf(err, "could not insert highres media url (%d, %s)", mediaID, highres_name) + } + + return nil +} + +func generateSaveThumbnailJPEG(tx *sql.Tx, mediaID int, thumbnail_name string, photoCachePath *string, baseImagePath string, urlID int) error { + thumbOutputPath := path.Join(*photoCachePath, thumbnail_name) + + thumbSize, err := EncodeThumbnail(baseImagePath, thumbOutputPath) + if err != nil { + return errors.Wrap(err, "could not create thumbnail cached image") + } + + fileStats, err := os.Stat(thumbOutputPath) + if err != nil { + return errors.Wrap(err, "reading file stats of thumbnail photo") + } + + if urlID < 0 { + _, err = tx.Exec("INSERT INTO media_url (media_id, media_name, width, height, purpose, content_type, file_size) VALUES (?, ?, ?, ?, ?, ?, ?)", + mediaID, thumbnail_name, thumbSize.Width, thumbSize.Height, models.PhotoThumbnail, "image/jpeg", fileStats.Size()) + } else { + _, err = tx.Exec("UPDATE media_url SET width = ?, height = ?, file_size= ? WHERE url_id = ?", + thumbSize.Width, thumbSize.Height, fileStats.Size(), urlID) + } + if err != nil { + return err + } + return nil +} + +func processRawSideCar(tx *sql.Tx, imageData *EncodeMediaData, highResURL *models.MediaURL, thumbURL *models.MediaURL, photoCachePath *string) error { + photo := imageData.media + sideCarFileHasChanged := false + var currentFileHash *string + currentSideCarPath := scanForSideCarFile(photo.Path) + + if currentSideCarPath != nil { + currentFileHash = hashSideCarFile(currentSideCarPath) + if photo.SideCarHash == nil || *photo.SideCarHash != *currentFileHash { + sideCarFileHasChanged = true + } + } else if photo.SideCarPath != nil { // sidecar has been deleted since last scan + sideCarFileHasChanged = true + } + if sideCarFileHasChanged { + fmt.Printf("Detected changed sidecar file for %s recreating JPG's to reflect changes\n", photo.Path) + + // update high res image may be cropped so dimentions and file size can change + baseImagePath := path.Join(*photoCachePath, highResURL.MediaName) // update base image path for thumbnail + tempHighResPath := baseImagePath + ".hold" + os.Rename(baseImagePath, tempHighResPath) + err := generateSaveHighResJPEG(tx, photo.MediaID, imageData, highResURL.MediaName, baseImagePath, highResURL.UrlID) + if err != nil { + os.Rename(tempHighResPath, baseImagePath) + return errors.Wrap(err, "recreating high-res cached image") + } + os.Remove(tempHighResPath) + + // update thumbnail image may be cropped so dimentions and file size can change + thumbPath := path.Join(*photoCachePath, thumbURL.MediaName) + tempThumbPath := thumbPath + ".hold" // hold onto the original image incase for some reason we fail to recreate one with the new settings + os.Rename(thumbPath, tempThumbPath) + err = generateSaveThumbnailJPEG(tx, photo.MediaID, thumbURL.MediaName, photoCachePath, baseImagePath, thumbURL.UrlID) + if err != nil { + os.Rename(tempThumbPath, thumbPath) + return errors.Wrap(err, "recreating thumbnail cached image") + } + os.Remove(tempThumbPath) + + // save new side car hash + _, err = tx.Exec("UPDATE media SET side_car_hash = ?, side_car_path = ? WHERE media_id = ?", currentFileHash, currentSideCarPath, photo.MediaID) + if err != nil { + return errors.Wrapf(err, "could not update side car hash for media: %s", photo.Path) + } + } + return nil +} diff --git a/api/scanner/scanner_media.go b/api/scanner/scanner_media.go index 5585796c..39f19b76 100644 --- a/api/scanner/scanner_media.go +++ b/api/scanner/scanner_media.go @@ -1,7 +1,10 @@ package scanner import ( + "crypto/md5" "database/sql" + "encoding/hex" + "io" "log" "os" "path" @@ -10,6 +13,40 @@ import ( "github.com/viktorstrate/photoview/api/graphql/models" ) +func scanForSideCarFile(path string) *string { + testPath := path + ".xmp" + _, err := os.Stat(testPath) + + if os.IsNotExist(err) { + return nil + } else if err != nil { + // unexpected error logging + log.Printf("ERROR: %s", err) + return nil + } + return &testPath + +} + +func hashSideCarFile(path *string) *string { + if path == nil { + return nil + } + + f, err := os.Open(*path) + if err != nil { + log.Printf("ERROR: %s", err) + } + defer f.Close() + + h := md5.New() + if _, err := io.Copy(h, f); err != nil { + log.Printf("ERROR: %s", err) + } + hash := hex.EncodeToString(h.Sum(nil)) + return &hash +} + func ScanMedia(tx *sql.Tx, mediaPath string, albumId int, cache *AlbumScannerCache) (*models.Media, bool, error) { mediaName := path.Base(mediaPath) @@ -35,10 +72,22 @@ func ScanMedia(tx *sql.Tx, mediaPath string, albumId int, cache *AlbumScannerCac } var mediaTypeText string + + var sideCarPath *string + sideCarPath = nil + var sideCarHash *string + sideCarHash = nil if mediaType.isVideo() { mediaTypeText = "video" } else { mediaTypeText = "photo" + // search for sidecar files + if mediaType.isRaw() { + sideCarPath = scanForSideCarFile(mediaPath) + if sideCarPath != nil { + sideCarHash = hashSideCarFile(sideCarPath) + } + } } stat, err := os.Stat(mediaPath) @@ -46,7 +95,7 @@ func ScanMedia(tx *sql.Tx, mediaPath string, albumId int, cache *AlbumScannerCac return nil, false, err } - result, err := tx.Exec("INSERT INTO media (title, path, path_hash, album_id, media_type, date_shot) VALUES (?, ?, MD5(path), ?, ?, ?)", mediaName, mediaPath, albumId, mediaTypeText, stat.ModTime()) + result, err := tx.Exec("INSERT INTO media (title, path, path_hash, side_car_path, side_car_hash, album_id, media_type, date_shot) VALUES (?, ?, MD5(path), ?, ?, ?, ?, ?)", mediaName, mediaPath, sideCarPath, sideCarHash, albumId, mediaTypeText, stat.ModTime()) if err != nil { return nil, false, errors.Wrap(err, "could not insert media into database") }