Improve stability of scanner

api/routes/photos.go

@@ -22,23 +22,29 @@ func RegisterPhotoRoutes(db *sql.DB, router *mux.Router) {
 	router.HandleFunc("/{name}", func(w http.ResponseWriter, r *http.Request) {
 		image_name := mux.Vars(r)["name"]
 
-		row := db.QueryRow("SELECT photo_url.purpose, photo.path, photo.photo_id, photo.album_id, photo_url.content_type FROM photo_url, photo WHERE photo_url.photo_name = ? AND photo_url.photo_id = photo.photo_id", image_name)
+		row := db.QueryRow("SELECT photo_url.purpose, photo_url.content_type, photo_url.photo_id FROM photo_url, photo WHERE photo_url.photo_name = ? AND photo_url.photo_id = photo.photo_id", image_name)
 
 		var purpose models.PhotoPurpose
-		var photoPath string
 		var content_type string
-		var album_id int
 		var photo_id int
 
-		if err := row.Scan(&purpose, &photoPath, &photo_id, &album_id, &content_type); err != nil {
+		if err := row.Scan(&purpose, &content_type, &photo_id); err != nil {
 			w.WriteHeader(http.StatusNotFound)
 			w.Write([]byte("404"))
 			return
 		}
 
+		row = db.QueryRow("SELECT * FROM photo WHERE photo_id = ?", photo_id)
+		photo, err := models.NewPhotoFromRow(row)
+		if err != nil {
+			log.Printf("WARN: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte("internal server error"))
+		}
+
 		user := auth.UserFromContext(r.Context())
 		if user != nil {
-			row := db.QueryRow("SELECT owner_id FROM album WHERE album.album_id = ?", album_id)
+			row := db.QueryRow("SELECT owner_id FROM album WHERE album.album_id = ?", photo.AlbumId)
 			var owner_id int
 
 			if err := row.Scan(&owner_id); err != nil {
@@ -72,7 +78,7 @@ func RegisterPhotoRoutes(db *sql.DB, router *mux.Router) {
 				return
 			}
 
-			if shareToken.AlbumID != nil && album_id != *shareToken.AlbumID {
+			if shareToken.AlbumID != nil && photo.AlbumId != *shareToken.AlbumID {
 				// Check child albums
 				row := db.QueryRow(`
 					WITH recursive child_albums AS (
@@ -81,7 +87,7 @@ func RegisterPhotoRoutes(db *sql.DB, router *mux.Router) {
 						SELECT child.* FROM album child JOIN child_albums parent ON parent.album_id = child.parent_album
 					)
 					SELECT * FROM child_albums WHERE album_id = ?
-				`, *shareToken.AlbumID, album_id)
+				`, *shareToken.AlbumID, photo.AlbumId)
 
 				_, err := models.NewAlbumFromRow(row)
 				if err != nil {
@@ -105,23 +111,47 @@ func RegisterPhotoRoutes(db *sql.DB, router *mux.Router) {
 
 		}
 
-		var file *os.File
+		var cachedPath string
+		var file *os.File = nil
 
 		if purpose == models.PhotoThumbnail || purpose == models.PhotoHighRes {
-			var err error
-			file, err = os.Open(path.Join(scanner.PhotoCache(), strconv.Itoa(album_id), strconv.Itoa(photo_id), image_name))
-			if err != nil {
-				w.Write([]byte("Error: " + err.Error()))
-				return
-			}
+			cachedPath = path.Join(scanner.PhotoCache(), strconv.Itoa(photo.AlbumId), strconv.Itoa(photo_id), image_name)
 		}
 
 		if purpose == models.PhotoOriginal {
-			var err error
-			file, err = os.Open(photoPath)
-			if err != nil {
-				w.Write([]byte("Error: " + err.Error()))
-				return
+			cachedPath = photo.Path
+		}
+
+		file, err = os.Open(cachedPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				tx, err := db.Begin()
+				if err != nil {
+					log.Printf("ERROR: %s\n", err)
+					w.WriteHeader(http.StatusInternalServerError)
+					w.Write([]byte("internal server error"))
+					return
+				}
+
+				err = scanner.ProcessPhoto(tx, photo, &content_type)
+				if err != nil {
+					log.Printf("ERROR: processing image not found in cache: %s\n", err)
+					w.WriteHeader(http.StatusInternalServerError)
+					w.Write([]byte("internal server error"))
+					tx.Rollback()
+					return
+				}
+
+				file, err = os.Open(cachedPath)
+				if err != nil {
+					log.Printf("ERROR: after reprocessing image not found in cache: %s\n", err)
+					w.WriteHeader(http.StatusInternalServerError)
+					w.Write([]byte("internal server error"))
+					tx.Rollback()
+					return
+				}
+
+				tx.Commit()
 			}
 		}
 

api/scanner/album_scanner.go

@@ -166,7 +166,7 @@ func scan(database *sql.DB, user *models.User) {
 					continue
 				}
 
-				if err := ProcessImage(tx, photoPath, albumId, *content_type); err != nil {
+				if err := ScanPhoto(tx, photoPath, albumId, content_type); err != nil {
 					ScannerError("processing image %s: %s", photoPath, err)
 					tx.Rollback()
 					continue

api/scanner/photo_scanner.go

@@ -0,0 +1,49 @@
+package scanner
+
+import (
+	"database/sql"
+	"github.com/viktorstrate/photoview/api/graphql/models"
+	"log"
+	"path"
+)
+
+func ScanPhoto(tx *sql.Tx, photoPath string, albumId int, content_type *string) error {
+
+	log.Printf("Scanning image: %s\n", photoPath)
+
+	photoName := path.Base(photoPath)
+
+	// Check if image already exists
+	row := tx.QueryRow("SELECT (photo_id) FROM photo WHERE path = ?", photoPath)
+	var photo_id int64
+	if err := row.Scan(&photo_id); err != sql.ErrNoRows {
+		if err == nil {
+			log.Printf("Image already scanned: %s\n", photoPath)
+			return nil
+		} else {
+			return err
+		}
+	}
+
+	result, err := tx.Exec("INSERT INTO photo (title, path, album_id) VALUES (?, ?, ?)", photoName, photoPath, albumId)
+	if err != nil {
+		log.Printf("ERROR: Could not insert photo into database")
+		return err
+	}
+	photo_id, err = result.LastInsertId()
+	if err != nil {
+		return err
+	}
+
+	row = tx.QueryRow("SELECT * FROM photo WHERE photo_id = ?", photo_id)
+	photo, err := models.NewPhotoFromRow(row)
+	if err != nil {
+		return err
+	}
+
+	if err := ProcessPhoto(tx, photo, content_type); err != nil {
+		return err
+	}
+
+	return nil
+}

api/scanner/process_image.go

@@ -1,163 +0,0 @@
-package scanner
-
-import (
-	"database/sql"
-	"fmt"
-	"image"
-	"image/jpeg"
-	"log"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-
-	"github.com/nfnt/resize"
-	"github.com/viktorstrate/photoview/api/graphql/models"
-	"github.com/viktorstrate/photoview/api/utils"
-
-	// Image decoders
-	_ "golang.org/x/image/bmp"
-	// _ "golang.org/x/image/tiff"
-	_ "image/gif"
-	_ "image/png"
-
-	_ "github.com/nf/cr2"
-	_ "golang.org/x/image/webp"
-)
-
-func ProcessImage(tx *sql.Tx, photoPath string, albumId int, content_type string) error {
-
-	// log.Printf("Processing image: %s\n", photoPath)
-
-	photoName := path.Base(photoPath)
-
-	// Check if image already exists
-	row := tx.QueryRow("SELECT (photo_id) FROM photo WHERE path = ?", photoPath)
-	var photo_id int64
-	if err := row.Scan(&photo_id); err != sql.ErrNoRows {
-		if err == nil {
-			log.Printf("Image already processed: %s\n", photoPath)
-			return nil
-		} else {
-			return err
-		}
-	}
-
-	result, err := tx.Exec("INSERT INTO photo (title, path, album_id) VALUES (?, ?, ?)", photoName, photoPath, albumId)
-	if err != nil {
-		log.Printf("ERROR: Could not insert photo into database")
-		return err
-	}
-	photo_id, err = result.LastInsertId()
-	if err != nil {
-		return err
-	}
-
-	photo_file, err := os.Open(photoPath)
-	if err != nil {
-		return err
-	}
-	defer photo_file.Close()
-
-	image, _, err := image.Decode(photo_file)
-	if err != nil {
-		log.Println("ERROR: decoding image")
-		return err
-	}
-
-	photoBaseName := photoName[0 : len(photoName)-len(path.Ext(photoName))]
-	photoBaseExt := path.Ext(photoName)
-
-	// original photo url
-	original_image_name := fmt.Sprintf("%s_%s", photoBaseName, utils.GenerateToken())
-	original_image_name = strings.ReplaceAll(original_image_name, " ", "_") + photoBaseExt
-
-	_, err = tx.Exec("INSERT INTO photo_url (photo_id, photo_name, width, height, purpose, content_type) VALUES (?, ?, ?, ?, ?, ?)", photo_id, original_image_name, image.Bounds().Max.X, image.Bounds().Max.Y, models.PhotoOriginal, content_type)
-	if err != nil {
-		log.Printf("Could not insert original photo url: %d, %s\n", photo_id, photoName)
-		return err
-	}
-
-	// Thumbnail
-	thumbnailImage := resize.Thumbnail(1024, 1024, image, resize.Bilinear)
-
-	if _, err := os.Stat(PhotoCache()); os.IsNotExist(err) {
-		if err := os.Mkdir(PhotoCache(), os.ModePerm); err != nil {
-			log.Println("ERROR: Could not make image cache directory")
-			return err
-		}
-	}
-
-	// Make album cache dir
-	albumCachePath := path.Join(PhotoCache(), strconv.Itoa(albumId))
-	if _, err := os.Stat(albumCachePath); os.IsNotExist(err) {
-		if err := os.Mkdir(albumCachePath, os.ModePerm); err != nil {
-			log.Println("ERROR: Could not make album image cache directory")
-			return err
-		}
-	}
-
-	// Make photo cache dir
-	photoCachePath := path.Join(albumCachePath, strconv.Itoa(int(photo_id)))
-	if _, err := os.Stat(photoCachePath); os.IsNotExist(err) {
-		if err := os.Mkdir(photoCachePath, os.ModePerm); err != nil {
-			log.Println("ERROR: Could not make photo image cache directory")
-			return err
-		}
-	}
-
-	// Save thumbnail as jpg
-	thumbnail_name := fmt.Sprintf("thumbnail_%s_%s", photoName, utils.GenerateToken())
-	thumbnail_name = strings.ReplaceAll(thumbnail_name, ".", "_")
-	thumbnail_name = strings.ReplaceAll(thumbnail_name, " ", "_")
-	thumbnail_name = thumbnail_name + ".jpg"
-
-	photo_file, err = os.Create(path.Join(photoCachePath, thumbnail_name))
-	if err != nil {
-		log.Println("ERROR: Could not make thumbnail file")
-		return err
-	}
-	defer photo_file.Close()
-
-	jpeg.Encode(photo_file, thumbnailImage, &jpeg.Options{Quality: 70})
-
-	thumbSize := thumbnailImage.Bounds().Max
-	_, err = tx.Exec("INSERT INTO photo_url (photo_id, photo_name, width, height, purpose, content_type) VALUES (?, ?, ?, ?, ?, ?)", photo_id, thumbnail_name, thumbSize.X, thumbSize.Y, models.PhotoThumbnail, "image/jpeg")
-	if err != nil {
-		return err
-	}
-
-	// high res
-	original_web_safe := false
-	for _, web_mime := range WebMimetypes {
-		if content_type == web_mime {
-			original_web_safe = true
-			break
-		}
-	}
-
-	// Generate high res jpeg
-	if !original_web_safe {
-		highres_name := fmt.Sprintf("highres_%s_%s", photoName, utils.GenerateToken())
-		highres_name = strings.ReplaceAll(highres_name, ".", "_")
-		highres_name = strings.ReplaceAll(highres_name, " ", "_")
-		highres_name = highres_name + ".jpg"
-
-		photo_file, err = os.Create(path.Join(photoCachePath, highres_name))
-		if err != nil {
-			log.Println("ERROR: Could not make highres file")
-			return err
-		}
-		defer photo_file.Close()
-
-		jpeg.Encode(photo_file, image, &jpeg.Options{Quality: 70})
-
-		_, err = tx.Exec("INSERT INTO photo_url (photo_id, photo_name, width, height, purpose, content_type) VALUES (?, ?, ?, ?, ?, ?)", photo_id, highres_name, image.Bounds().Max.X, image.Bounds().Max.Y, models.PhotoHighRes, "image/jpeg")
-		if err != nil {
-			log.Printf("Could not insert highres photo url: %d, %s\n", photo_id, photoName)
-			return err
-		}
-	}
-
-	return nil
-}

api/scanner/process_photo.go

@@ -0,0 +1,289 @@
+package scanner
+
+import (
+	"database/sql"
+	"fmt"
+	"image"
+	"image/jpeg"
+	"log"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/nfnt/resize"
+	"github.com/viktorstrate/photoview/api/graphql/models"
+	"github.com/viktorstrate/photoview/api/utils"
+
+	// Image decoders
+	_ "golang.org/x/image/bmp"
+	// _ "golang.org/x/image/tiff"
+	_ "image/gif"
+	_ "image/png"
+
+	_ "github.com/nf/cr2"
+	_ "golang.org/x/image/webp"
+)
+
+func makePhotoURLChecker(tx *sql.Tx, photoID int) (func(purpose models.PhotoPurpose) (*models.PhotoURL, error), error) {
+	photoURLExistsStmt, err := tx.Prepare("SELECT * FROM photo_url WHERE photo_id = ? AND purpose = ?")
+	if err != nil {
+		return nil, err
+	}
+
+	return func(purpose models.PhotoPurpose) (*models.PhotoURL, error) {
+		row := photoURLExistsStmt.QueryRow(photoID, purpose)
+		photoURL, err := models.NewPhotoURLFromRow(row)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, nil
+			}
+			return nil, err
+		}
+
+		return photoURL, nil
+	}, nil
+}
+
+func ProcessPhoto(tx *sql.Tx, photo *models.Photo, content_type *string) error {
+
+	log.Printf("Processing photo: %s\n", photo.Path)
+
+	imageData := processImageData{
+		photoPath: photo.Path,
+	}
+
+	photoName := path.Base(photo.Path)
+
+	photoBaseName := photoName[0 : len(photoName)-len(path.Ext(photoName))]
+	photoBaseExt := path.Ext(photoName)
+
+	photoChecker, err := makePhotoURLChecker(tx, photo.PhotoID)
+	if err != nil {
+		return err
+	}
+
+	// original photo url
+	origURL, err := photoChecker(models.PhotoOriginal)
+	if err != nil {
+		return err
+	}
+
+	if origURL == nil {
+		original_image_name := fmt.Sprintf("%s_%s", photoBaseName, utils.GenerateToken())
+		original_image_name = strings.ReplaceAll(original_image_name, " ", "_") + photoBaseExt
+
+		photoImage, err := imageData.PhotoImage()
+		if err != nil {
+			return err
+		}
+
+		_, err = tx.Exec("INSERT INTO photo_url (photo_id, photo_name, width, height, purpose, content_type) VALUES (?, ?, ?, ?, ?, ?)", photo.PhotoID, original_image_name, photoImage.Bounds().Max.X, photoImage.Bounds().Max.Y, models.PhotoOriginal, content_type)
+		if err != nil {
+			log.Printf("Could not insert original photo url: %d, %s\n", photo.PhotoID, photoName)
+			return err
+		}
+	}
+
+	// Thumbnail
+	thumbURL, err := photoChecker(models.PhotoThumbnail)
+	if err != nil {
+		return err
+	}
+
+	// Highres
+	highResURL, err := photoChecker(models.PhotoHighRes)
+	if err != nil {
+		return err
+	}
+
+	// Make sure photo cache directory exists
+	photoCachePath, err := makePhotoCacheDir(photo)
+	if err != nil {
+		return err
+	}
+
+	// Save thumbnail to cache
+	if thumbURL == nil {
+		thumbnail_name := fmt.Sprintf("thumbnail_%s_%s", photoName, utils.GenerateToken())
+		thumbnail_name = strings.ReplaceAll(thumbnail_name, ".", "_")
+		thumbnail_name = strings.ReplaceAll(thumbnail_name, " ", "_")
+		thumbnail_name = thumbnail_name + ".jpg"
+
+		thumbnailImage, err := imageData.ThumbnailImage()
+		if err != nil {
+			return err
+		}
+
+		err = encodeImageJPEG(path.Join(*photoCachePath, thumbnail_name), thumbnailImage, &jpeg.Options{Quality: 70})
+		if err != nil {
+			log.Println("ERROR: creating high-res cached image")
+			return err
+		}
+
+		thumbSize := thumbnailImage.Bounds().Max
+		_, err = tx.Exec("INSERT INTO photo_url (photo_id, photo_name, width, height, purpose, content_type) VALUES (?, ?, ?, ?, ?, ?)", photo.PhotoID, thumbnail_name, thumbSize.X, thumbSize.Y, models.PhotoThumbnail, "image/jpeg")
+		if err != nil {
+			return err
+		}
+	} else if thumbURL != nil {
+		thumbPath := path.Join(*photoCachePath, thumbURL.PhotoName)
+
+		if _, err := os.Stat(thumbPath); os.IsNotExist(err) {
+			fmt.Printf("Thumbnail photo found in database but not in cache, re-encoding photo to cache: %s\n", thumbURL.PhotoName)
+
+			thumbnailImage, err := imageData.ThumbnailImage()
+			if err != nil {
+				return err
+			}
+
+			err = encodeImageJPEG(thumbPath, thumbnailImage, &jpeg.Options{Quality: 70})
+			if err != nil {
+				log.Println("ERROR: creating thumbnail cached image")
+				return err
+			}
+		}
+	}
+
+	// high res
+	original_web_safe := false
+	for _, web_mime := range WebMimetypes {
+		if *content_type == web_mime {
+			original_web_safe = true
+			break
+		}
+	}
+
+	// Generate high res jpeg
+	if highResURL == nil {
+		if !original_web_safe {
+			highres_name := fmt.Sprintf("highres_%s_%s", photoName, utils.GenerateToken())
+			highres_name = strings.ReplaceAll(highres_name, ".", "_")
+			highres_name = strings.ReplaceAll(highres_name, " ", "_")
+			highres_name = highres_name + ".jpg"
+
+			photoImage, err := imageData.PhotoImage()
+			if err != nil {
+				return err
+			}
+
+			err = encodeImageJPEG(path.Join(*photoCachePath, highres_name), photoImage, &jpeg.Options{Quality: 70})
+			if err != nil {
+				log.Println("ERROR: creating high-res cached image")
+				return err
+			}
+
+			_, err = tx.Exec("INSERT INTO photo_url (photo_id, photo_name, width, height, purpose, content_type) VALUES (?, ?, ?, ?, ?, ?)",
+				photo.PhotoID, highres_name, photoImage.Bounds().Max.X, photoImage.Bounds().Max.Y, models.PhotoHighRes, "image/jpeg")
+			if err != nil {
+				log.Printf("Could not insert highres photo url: %d, %s\n", photo.PhotoID, photoName)
+				return err
+			}
+		}
+	} else if highResURL != nil {
+		highResPath := path.Join(*photoCachePath, highResURL.PhotoName)
+
+		if _, err := os.Stat(highResPath); os.IsNotExist(err) {
+			fmt.Printf("High-res photo found in database but not in cache, re-encoding photo to cache: %s\n", highResURL.PhotoName)
+
+			photoImage, err := imageData.PhotoImage()
+			if err != nil {
+				return err
+			}
+
+			err = encodeImageJPEG(highResPath, photoImage, &jpeg.Options{Quality: 70})
+			if err != nil {
+				log.Println("ERROR: creating high-res cached image")
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func makePhotoCacheDir(photo *models.Photo) (*string, error) {
+
+	// Make root cache dir if not exists
+	if _, err := os.Stat(PhotoCache()); os.IsNotExist(err) {
+		if err := os.Mkdir(PhotoCache(), os.ModePerm); err != nil {
+			log.Println("ERROR: Could not make root image cache directory")
+			return nil, err
+		}
+	}
+
+	// Make album cache dir if not exists
+	albumCachePath := path.Join(PhotoCache(), strconv.Itoa(photo.AlbumId))
+	if _, err := os.Stat(albumCachePath); os.IsNotExist(err) {
+		if err := os.Mkdir(albumCachePath, os.ModePerm); err != nil {
+			log.Println("ERROR: Could not make album image cache directory")
+			return nil, err
+		}
+	}
+
+	// Make photo cache dir if not exists
+	photoCachePath := path.Join(albumCachePath, strconv.Itoa(photo.PhotoID))
+	if _, err := os.Stat(photoCachePath); os.IsNotExist(err) {
+		if err := os.Mkdir(photoCachePath, os.ModePerm); err != nil {
+			log.Println("ERROR: Could not make photo image cache directory")
+			return nil, err
+		}
+	}
+
+	return &photoCachePath, nil
+}
+
+func encodeImageJPEG(photoPath string, photoImage image.Image, jpegOptions *jpeg.Options) error {
+	photo_file, err := os.Create(photoPath)
+	if err != nil {
+		log.Printf("ERROR: Could not create file: %s\n", photoPath)
+		return err
+	}
+	defer photo_file.Close()
+
+	err = jpeg.Encode(photo_file, photoImage, jpegOptions)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type processImageData struct {
+	photoPath       string
+	_photoImage     image.Image
+	_thumbnailImage image.Image
+}
+
+func (img *processImageData) PhotoImage() (image.Image, error) {
+	if img._photoImage != nil {
+		return img._photoImage, nil
+	}
+
+	photoFile, err := os.Open(img.photoPath)
+	if err != nil {
+		return nil, err
+	}
+	defer photoFile.Close()
+
+	photoImg, _, err := image.Decode(photoFile)
+	if err != nil {
+		log.Println("ERROR: decoding image")
+		return nil, err
+	}
+
+	img._photoImage = photoImg
+	return img._photoImage, nil
+}
+
+func (img *processImageData) ThumbnailImage() (image.Image, error) {
+	photoImage, err := img.PhotoImage()
+	if err != nil {
+		return nil, err
+	}
+
+	thumbImage := resize.Thumbnail(1024, 1024, photoImage, resize.Bilinear)
+	img._thumbnailImage = thumbImage
+
+	return img._thumbnailImage, nil
+}

ui/src/components/messages/Messages.js

@@ -124,7 +124,9 @@ const Messages = () => {
           </animated.div>
         )
       })}
-      <SubscriptionsHook messages={messages} setMessages={setMessages} />
+      {localStorage.getItem('token') && (
+        <SubscriptionsHook messages={messages} setMessages={setMessages} />
+      )}
     </Container>
   )
 }