From e4df1fb70618ce5cb59cbe972b1ec98632608fb1 Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Tue, 16 Feb 2021 11:27:28 +0100 Subject: [PATCH] Save face rectangles to database --- api/graphql/models/face_detection.go | 74 +++++++++++++++++++ api/scanner/encode_photo.go | 56 +------------- api/scanner/face_detection/face_detector.go | 10 ++- api/scanner/image_helpers/photo_dimensions.go | 55 ++++++++++++++ api/scanner/process_photo.go | 9 ++- api/scanner/process_video.go | 5 +- 6 files changed, 149 insertions(+), 60 deletions(-) create mode 100644 api/scanner/image_helpers/photo_dimensions.go diff --git a/api/graphql/models/face_detection.go b/api/graphql/models/face_detection.go index 97aac1e0..9628f2b4 100644 --- a/api/graphql/models/face_detection.go +++ b/api/graphql/models/face_detection.go @@ -4,8 +4,13 @@ import ( "bytes" "database/sql/driver" "encoding/binary" + "fmt" + "image" + "strconv" + "strings" "github.com/Kagami/go-face" + "github.com/photoview/photoview/api/scanner/image_helpers" ) type FaceGroup struct { @@ -20,6 +25,7 @@ type ImageFace struct { MediaID int `gorm:"not null;index"` Media Media `gorm:"constraint:OnDelete:CASCADE;"` Descriptor FaceDescriptor `gorm:"not null"` + Rectangle FaceRectangle `gorm:"not null"` } type FaceDescriptor face.Descriptor @@ -46,3 +52,71 @@ func (fd FaceDescriptor) Value() (driver.Value, error) { return buf.Bytes(), nil } + +type FaceRectangle struct { + minX, maxX float32 + minY, maxY float32 +} + +// ToDBFaceRectangle converts a pixel absolute rectangle to a relative FaceRectangle to be saved in the database +func ToDBFaceRectangle(imgRec image.Rectangle, imagePath string) (*FaceRectangle, error) { + size, err := image_helpers.GetPhotoDimensions(imagePath) + if err != nil { + return nil, err + } + + return &FaceRectangle{ + minX: float32(imgRec.Min.X) / float32(size.Width), + maxX: float32(imgRec.Max.X) / float32(size.Width), + minY: float32(imgRec.Min.Y) / float32(size.Height), + maxY: float32(imgRec.Max.Y) / float32(size.Height), + }, nil +} + +// GormDataType datatype used in database +func (fr FaceRectangle) GormDataType() string { + return "VARCHAR(64)" +} + +// Scan tells GORM how to convert database data to Go format +func (fr *FaceRectangle) Scan(value interface{}) error { + byteArray := value.([]uint8) + slices := strings.Split(string(byteArray), ":") + + if len(slices) != 4 { + return fmt.Errorf("Invalid face rectangle format, expected 4 values, got %d", len(slices)) + } + + minX, err := strconv.ParseFloat(slices[0], 32) + if err != nil { + return err + } + + maxX, err := strconv.ParseFloat(slices[0], 32) + if err != nil { + return err + } + + minY, err := strconv.ParseFloat(slices[0], 32) + if err != nil { + return err + } + + maxY, err := strconv.ParseFloat(slices[0], 32) + if err != nil { + return err + } + + fr.minX = float32(minX) + fr.minX = float32(maxX) + fr.minX = float32(minY) + fr.minX = float32(maxY) + + return nil +} + +// Value tells GORM how to save into the database +func (fr FaceRectangle) Value() (driver.Value, error) { + result := fmt.Sprintf("%f:%f:%f:%f", fr.minX, fr.maxX, fr.minY, fr.maxY) + return result, nil +} diff --git a/api/scanner/encode_photo.go b/api/scanner/encode_photo.go index 377529ef..faf9e0bc 100644 --- a/api/scanner/encode_photo.go +++ b/api/scanner/encode_photo.go @@ -6,18 +6,14 @@ import ( "os" "github.com/disintegration/imaging" - "github.com/pkg/errors" "github.com/photoview/photoview/api/graphql/models" + "github.com/photoview/photoview/api/scanner/image_helpers" "github.com/photoview/photoview/api/utils" + "github.com/pkg/errors" "gopkg.in/vansante/go-ffprobe.v2" "gorm.io/gorm" ) -type PhotoDimensions struct { - Width int - Height int -} - func DecodeImage(imagePath string) (image.Image, error) { file, err := os.Open(imagePath) if err != nil { @@ -33,32 +29,6 @@ func DecodeImage(imagePath string) (image.Image, error) { return image, nil } -func PhotoDimensionsFromRect(rect image.Rectangle) PhotoDimensions { - return PhotoDimensions{ - Width: rect.Bounds().Max.X, - Height: rect.Bounds().Max.Y, - } -} - -func (dimensions *PhotoDimensions) ThumbnailScale() PhotoDimensions { - aspect := float64(dimensions.Width) / float64(dimensions.Height) - - var width, height int - - if aspect > 1 { - width = 1024 - height = int(1024 / aspect) - } else { - width = int(1024 * aspect) - height = 1024 - } - - return PhotoDimensions{ - Width: width, - Height: height, - } -} - // EncodeMediaData is used to easily decode media data, with a cache so expensive operations are not repeated type EncodeMediaData struct { media *models.Media @@ -83,24 +53,6 @@ func EncodeImageJPEG(image image.Image, outputPath string, jpegQuality int) erro return nil } -func GetPhotoDimensions(imagePath string) (*PhotoDimensions, error) { - photoFile, err := os.Open(imagePath) - if err != nil { - return nil, err - } - defer photoFile.Close() - - config, _, err := image.DecodeConfig(photoFile) - if err != nil { - return nil, err - } - - return &PhotoDimensions{ - Width: config.Width, - Height: config.Height, - }, nil -} - // ContentType reads the image to determine its content type func (img *EncodeMediaData) ContentType() (*MediaType, error) { if img._contentType != nil { @@ -148,13 +100,13 @@ func (img *EncodeMediaData) EncodeHighRes(tx *gorm.DB, outputPath string) error return nil } -func EncodeThumbnail(inputPath string, outputPath string) (*PhotoDimensions, error) { +func EncodeThumbnail(inputPath string, outputPath string) (*image_helpers.PhotoDimensions, error) { inputImage, err := DecodeImage(inputPath) if err != nil { return nil, err } - dimensions := PhotoDimensionsFromRect(inputImage.Bounds()) + dimensions := image_helpers.PhotoDimensionsFromRect(inputImage.Bounds()) dimensions = dimensions.ThumbnailScale() thumbImage := imaging.Resize(inputImage, dimensions.Width, dimensions.Height, imaging.NearestNeighbor) diff --git a/api/scanner/face_detection/face_detector.go b/api/scanner/face_detection/face_detector.go index 9f41864d..5bbca944 100644 --- a/api/scanner/face_detection/face_detector.go +++ b/api/scanner/face_detection/face_detector.go @@ -97,21 +97,27 @@ func (fd *FaceDetector) DetectFaces(media *models.Media) error { } for _, face := range faces { - fd.classifyFace(&face, media) + fd.classifyFace(&face, media, thumbnailPath) } return nil } -func (fd *FaceDetector) classifyFace(face *face.Face, media *models.Media) error { +func (fd *FaceDetector) classifyFace(face *face.Face, media *models.Media, imagePath string) error { fd.mutex.Lock() defer fd.mutex.Unlock() match := fd.rec.ClassifyThreshold(face.Descriptor, 0.2) + faceRect, err := models.ToDBFaceRectangle(face.Rectangle, imagePath) + if err != nil { + return err + } + imageFace := models.ImageFace{ MediaID: media.ID, Descriptor: models.FaceDescriptor(face.Descriptor), + Rectangle: *faceRect, } var faceGroup models.FaceGroup diff --git a/api/scanner/image_helpers/photo_dimensions.go b/api/scanner/image_helpers/photo_dimensions.go new file mode 100644 index 00000000..84dc146a --- /dev/null +++ b/api/scanner/image_helpers/photo_dimensions.go @@ -0,0 +1,55 @@ +package image_helpers + +import ( + "image" + "os" +) + +type PhotoDimensions struct { + Width int + Height int +} + +func GetPhotoDimensions(imagePath string) (*PhotoDimensions, error) { + photoFile, err := os.Open(imagePath) + if err != nil { + return nil, err + } + defer photoFile.Close() + + config, _, err := image.DecodeConfig(photoFile) + if err != nil { + return nil, err + } + + return &PhotoDimensions{ + Width: config.Width, + Height: config.Height, + }, nil +} + +func PhotoDimensionsFromRect(rect image.Rectangle) PhotoDimensions { + return PhotoDimensions{ + Width: rect.Bounds().Max.X, + Height: rect.Bounds().Max.Y, + } +} + +func (dimensions *PhotoDimensions) ThumbnailScale() PhotoDimensions { + aspect := float64(dimensions.Width) / float64(dimensions.Height) + + var width, height int + + if aspect > 1 { + width = 1024 + height = int(1024 / aspect) + } else { + width = int(1024 * aspect) + height = 1024 + } + + return PhotoDimensions{ + Width: width, + Height: height, + } +} diff --git a/api/scanner/process_photo.go b/api/scanner/process_photo.go index a89d918f..389d25cf 100644 --- a/api/scanner/process_photo.go +++ b/api/scanner/process_photo.go @@ -8,6 +8,7 @@ import ( "strconv" "github.com/photoview/photoview/api/graphql/models" + "github.com/photoview/photoview/api/scanner/image_helpers" "github.com/photoview/photoview/api/utils" "github.com/pkg/errors" "gorm.io/gorm" @@ -110,7 +111,7 @@ func processPhoto(tx *gorm.DB, imageData *EncodeMediaData, photoCachePath *strin return false, errors.Wrap(err, "error processing photo highres") } - var photoDimensions *PhotoDimensions + var photoDimensions *image_helpers.PhotoDimensions var baseImagePath string = photo.Path mediaType, err := getMediaType(photo.Path) @@ -161,7 +162,7 @@ func processPhoto(tx *gorm.DB, imageData *EncodeMediaData, photoCachePath *strin // Make sure photo dimensions is set if photoDimensions == nil { - photoDimensions, err = GetPhotoDimensions(baseImagePath) + photoDimensions, err = image_helpers.GetPhotoDimensions(baseImagePath) if err != nil { return false, err } @@ -242,7 +243,7 @@ func makeMediaCacheDir(media *models.Media) (*string, error) { return &photoCachePath, nil } -func saveOriginalPhotoToDB(tx *gorm.DB, photo *models.Media, imageData *EncodeMediaData, photoDimensions *PhotoDimensions) error { +func saveOriginalPhotoToDB(tx *gorm.DB, photo *models.Media, imageData *EncodeMediaData, photoDimensions *image_helpers.PhotoDimensions) error { originalImageName := generateUniqueMediaName(photo.Path) contentType, err := imageData.ContentType() @@ -279,7 +280,7 @@ func generateSaveHighResJPEG(tx *gorm.DB, media *models.Media, imageData *Encode return nil, errors.Wrap(err, "creating high-res cached image") } - photoDimensions, err := GetPhotoDimensions(imagePath) + photoDimensions, err := image_helpers.GetPhotoDimensions(imagePath) if err != nil { return nil, err } diff --git a/api/scanner/process_video.go b/api/scanner/process_video.go index e2e579ed..342ad781 100644 --- a/api/scanner/process_video.go +++ b/api/scanner/process_video.go @@ -10,6 +10,7 @@ import ( "time" "github.com/photoview/photoview/api/graphql/models" + "github.com/photoview/photoview/api/scanner/image_helpers" "github.com/photoview/photoview/api/utils" "github.com/pkg/errors" "gopkg.in/vansante/go-ffprobe.v2" @@ -131,7 +132,7 @@ func processVideo(tx *gorm.DB, mediaData *EncodeMediaData, videoCachePath *strin return false, errors.Wrapf(err, "failed to generate thumbnail for video (%s)", video.Title) } - thumbDimensions, err := GetPhotoDimensions(thumbImagePath) + thumbDimensions, err := image_helpers.GetPhotoDimensions(thumbImagePath) if err != nil { return false, errors.Wrap(err, "get dimensions of video thumbnail image") } @@ -167,7 +168,7 @@ func processVideo(tx *gorm.DB, mediaData *EncodeMediaData, videoCachePath *strin return false, errors.Wrapf(err, "failed to generate thumbnail for video (%s)", video.Title) } - thumbDimensions, err := GetPhotoDimensions(thumbImagePath) + thumbDimensions, err := image_helpers.GetPhotoDimensions(thumbImagePath) if err != nil { return false, errors.Wrap(err, "get dimensions of video thumbnail image") }