From 672d3351606cbc73c4cef9c64c88c75866d6f234 Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Tue, 23 Jul 2019 17:49:45 +0200 Subject: [PATCH] Properly delete images from cache --- api/src/index.js | 28 ++++++++++++++++---- api/src/scanner/Scanner.js | 46 ++++++++++++++++----------------- api/src/scanner/processImage.js | 39 ++++++++++++++++++++++++---- api/src/scanner/scanAlbum.js | 41 ++++++++++++++++++++++++----- api/src/scanner/utils.js | 5 ++++ 5 files changed, 119 insertions(+), 40 deletions(-) diff --git a/api/src/index.js b/api/src/index.js index b54e3b11..eb3c7922 100644 --- a/api/src/index.js +++ b/api/src/index.js @@ -38,6 +38,7 @@ const typeDefs = fs import usersResolver from './resolvers/users' import scannerResolver from './resolvers/scanner' import photosResolver from './resolvers/photos' +import { isRawImage } from './scanner/utils' const resolvers = [usersResolver, scannerResolver, photosResolver] @@ -157,15 +158,32 @@ app.use('/images/:id/:image', async function(req, res) { let imagePath = path.resolve(config.cachePath, 'images', id, image) - if (image != 'extracted.jpg' && image != 'thumbnail.jpg') { + if (!(await fs.exists(imagePath))) { + if (image == 'thumbnail.jpg') { + console.log('Thumbnail not found, generating', photo.path) + await scanner.processImage(id) + + if (!(await fs.exists(imagePath))) { + throw new Error('Thumbnail not found after image processing') + } + + return res.sendFile(imagePath) + } + imagePath = photo.path } - const imageFound = await fs.exists(imagePath) - - if (!imageFound) { - console.log('Image not found', imagePath) + if (await isRawImage(imagePath)) { + console.log('RAW preview image not found, generating', imagePath) await scanner.processImage(id) + + imagePath = path.resolve(config.cachePath, 'images', id, image) + + if (!(await fs.exists(imagePath))) { + throw new Error('RAW preview not found after image processing') + } + + return res.sendFile(imagePath) } res.sendFile(imagePath) diff --git a/api/src/scanner/Scanner.js b/api/src/scanner/Scanner.js index 20460011..9d45e911 100644 --- a/api/src/scanner/Scanner.js +++ b/api/src/scanner/Scanner.js @@ -1,13 +1,5 @@ -import fs from 'fs-extra' -import path from 'path' -import { resolve as pathResolve, basename as pathBasename } from 'path' import { PubSub } from 'apollo-server' -import uuid from 'uuid' -import { exiftool } from 'exiftool-vendored' -import sharp from 'sharp' -import { isImage, isRawImage } from './utils' -import { promisify } from 'util' -import config from '../config' +import _ from 'lodash' import _scanUser from './scanUser' import _scanAlbum from './scanAlbum' import _processImage from './processImage' @@ -29,13 +21,27 @@ class PhotoScanner { this.imagesToProgress = 0 this.finishedImages = 0 - this.addImageToProgress = () => { + this.markImageToProgress = () => { this.imagesToProgress++ } - this.addFinishedImage = () => { + this.markFinishedImage = () => { this.finishedImages++ } + + this.broadcastProgress = _.debounce(() => { + console.log( + `Progress: ${(this.finishedImages / this.imagesToProgress) * 100}` + ) + this.pubsub.publish(EVENT_SCANNER_PROGRESS, { + scannerStatusUpdate: { + progress: (this.finishedImages / this.imagesToProgress) * 100, + finished: false, + error: false, + errorMessage: '', + }, + }) + }, 250) } async scanUser(user) { @@ -46,31 +52,24 @@ class PhotoScanner { await _scanAlbum( { driver: this.driver, - addImageToProgress: this.addImageToProgress, - addFinishedImage: this.addFinishedImage, + markImageToProgress: this.markImageToProgress, + markFinishedImage: this.markFinishedImage, processImage: this.processImage, }, album ) - - this.pubsub.publish(EVENT_SCANNER_PROGRESS, { - scannerStatusUpdate: { - progress: this.finishedImages / this.imagesToProgress, - finished: false, - error: false, - errorMessage: '', - }, - }) } async processImage(id) { await _processImage( { driver: this.driver, - addFinishedImage: this.addFinishedImage, + addFinishedImage: this.markFinishedImage, }, id ) + + this.broadcastProgress() } async scanAll() { @@ -100,6 +99,7 @@ class PhotoScanner { console.log( `Done scanning ${this.finishedImages} of ${this.imagesToProgress}` ) + this.pubsub.publish(EVENT_SCANNER_PROGRESS, { scannerStatusUpdate: { progress: 100, diff --git a/api/src/scanner/processImage.js b/api/src/scanner/processImage.js index da400e07..ab80f34e 100644 --- a/api/src/scanner/processImage.js +++ b/api/src/scanner/processImage.js @@ -2,8 +2,7 @@ import fs from 'fs-extra' import path from 'path' import { exiftool } from 'exiftool-vendored' import sharp from 'sharp' -import { isRawImage, imageSize } from './utils' -import config from '../config' +import { isRawImage, imageSize, getImageCachePath } from './utils' export default async function processImage({ driver, addFinishedImage }, id) { const session = driver.session() @@ -19,9 +18,9 @@ export default async function processImage({ driver, addFinishedImage }, id) { const photo = result.records[0].get('p').properties - console.log('Processing photo', photo.path) + // console.log('Processing photo', photo.path) - const imagePath = path.resolve(config.cachePath, 'images', id) + const imagePath = getImageCachePath(id) await fs.remove(imagePath) await fs.mkdirp(imagePath) @@ -34,7 +33,37 @@ export default async function processImage({ driver, addFinishedImage }, id) { const extractedPath = path.resolve(imagePath, 'extracted.jpg') await exiftool.extractPreview(photo.path, extractedPath) - originalPath = extractedPath + const rawTags = await exiftool.read(photo.path) + // ISO, FNumber, Model, ExposureTime, FocalLength, LensType + // console.log(rawTags) + + let rotateAngle = null + switch (rawTags.Orientation) { + case 8: + rotateAngle = -90 + break + case 3: + rotateAngle = 180 + break + case 6: + rotateAngle = 90 + } + + // Replace extension with .jpg + let processedBase = path.basename(photo.path).match(/(.*)(\..*)/) + processedBase = + processedBase == null ? path.basename(photo.path) : processedBase[1] + processedBase += '.jpg' + + const processedPath = path.resolve(imagePath, processedBase) + await sharp(extractedPath) + .jpeg({ quality: 80 }) + .rotate(rotateAngle) + .toFile(processedPath) + + fs.remove(extractedPath) + + originalPath = processedPath } // Resize image diff --git a/api/src/scanner/scanAlbum.js b/api/src/scanner/scanAlbum.js index 6121e106..b956532d 100644 --- a/api/src/scanner/scanAlbum.js +++ b/api/src/scanner/scanAlbum.js @@ -1,26 +1,28 @@ import fs from 'fs-extra' import path from 'path' import uuid from 'uuid' -import { isImage } from './utils' -import config from '../config' +import { isImage, getImageCachePath } from './utils' export default async function scanAlbum( - { driver, addImageToProgress, addFinishedImage, processImage }, + { driver, markImageToProgress, markFinishedImage, processImage }, album ) { const { title, path: albumPath, id } = album console.log('Scanning album', title) + let processedImages = [] + const list = await fs.readdir(albumPath) let processingImagePromises = [] for (const item of list) { const itemPath = path.resolve(albumPath, item) + processedImages.push(itemPath) if (await isImage(itemPath)) { const session = driver.session() - addImageToProgress() + markImageToProgress() const photoResult = await session.run( `MATCH (p:Photo {path: {imgPath} })<--(a:Album {id: {albumId}}) RETURN p`, @@ -36,15 +38,14 @@ export default async function scanAlbum( const id = photoResult.records[0].get('p').properties.id const thumbnailPath = path.resolve( - config.cachePath, - id, + getImageCachePath(id), 'thumbnail.jpg' ) if (!(await fs.exists(thumbnailPath))) { processingImagePromises.push(processImage(id)) } else { - addFinishedImage() + markFinishedImage() } } else { console.log(`Found new image at ${itemPath}`) @@ -66,6 +67,32 @@ export default async function scanAlbum( } } + const session = driver.session() + + const deletedImagesResult = await session.run( + `MATCH (a:Album { id: {albumId} })-[:CONTAINS]->(p:Photo)-->(trail) + WHERE NOT p.path IN {images} + WITH p, p.id AS imageId, trail + DETACH DELETE p, trail + RETURN DISTINCT imageId`, + { + albumId: id, + images: processedImages, + } + ) + + const deletedImages = deletedImagesResult.records.map(record => + record.get('imageId') + ) + + for (const imageId of deletedImages) { + await fs.remove(getImageCachePath(imageId)) + } + + console.log(`Deleted ${deletedImages.length} images from album ${title}`) + + session.close() + await Promise.all(processingImagePromises) console.log('Done processing album', album.title) } diff --git a/api/src/scanner/utils.js b/api/src/scanner/utils.js index 9bada1db..101b9474 100644 --- a/api/src/scanner/utils.js +++ b/api/src/scanner/utils.js @@ -2,6 +2,8 @@ import fs from 'fs-extra' import readChunk from 'read-chunk' import imageType from 'image-type' import { promisify } from 'util' +import path from 'path' +import config from '../config' export const isImage = async path => { if ((await fs.stat(path)).isDirectory()) { @@ -31,3 +33,6 @@ export const isRawImage = async path => { } export const imageSize = promisify(require('image-size')) + +export const getImageCachePath = id => + path.resolve(config.cachePath, 'images', id)