From e5688bdee857e42bb7e80767308f92fe1bbaf525 Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Wed, 21 Aug 2019 18:20:43 +0200 Subject: [PATCH] Improve scanner --- api/src/resolvers/scanner.js | 10 +- api/src/scanner/Scanner.js | 160 ++++++++++++--------- api/src/scanner/processImage.js | 7 +- api/src/scanner/scanAlbum.js | 24 ++-- api/src/scanner/scanAll.js | 9 +- api/src/scanner/scanUser.js | 11 +- api/src/schema.graphql | 2 +- ui/src/Messages.js | 8 +- ui/src/Pages/SettingsPage/SettingsPage.js | 2 +- ui/src/components/albumGallery/AlbumBox.js | 3 +- 10 files changed, 135 insertions(+), 101 deletions(-) diff --git a/api/src/resolvers/scanner.js b/api/src/resolvers/scanner.js index fe3d300c..9b19d644 100644 --- a/api/src/resolvers/scanner.js +++ b/api/src/resolvers/scanner.js @@ -2,21 +2,13 @@ import { EVENT_SCANNER_PROGRESS } from '../scanner/Scanner' const Mutation = { async scanAll(root, args, ctx, info) { - if (ctx.scanner.isRunning) { - return { - finished: false, - success: false, - errorMessage: 'Scanner already running', - } - } - ctx.scanner.scanAll() return { finished: false, success: true, progress: 0, - errorMessage: null, + message: 'Starting scanner', } }, } diff --git a/api/src/scanner/Scanner.js b/api/src/scanner/Scanner.js index 8befd348..0913336f 100644 --- a/api/src/scanner/Scanner.js +++ b/api/src/scanner/Scanner.js @@ -7,6 +7,66 @@ import _scanAll from './scanAll' export const EVENT_SCANNER_PROGRESS = 'SCANNER_PROGRESS' +async function _execScan(scanner, scanFunction) { + try { + if (scanner.isRunning) throw new Error('Scanner already running') + scanner.isRunning = true + scanner.imageProgress = {} + + const session = scanner.driver.session() + const photoResult = await session.run( + 'MATCH (photo:Photo) RETURN photo.id as photoId' + ) + session.close() + + photoResult.records + .map(x => x.get('photoId')) + .forEach(id => { + scanner.markImageToProgress(id) + }) + + scanner.pubsub.publish(EVENT_SCANNER_PROGRESS, { + scannerStatusUpdate: { + progress: 0, + finished: false, + success: true, + message: 'Scan started', + }, + }) + + console.log('Calling scan function') + await scanFunction() + console.log('Scan function ended') + + console.log( + `Done scanning ${Object.keys(scanner.imageProgress).length} photos` + ) + + scanner.pubsub.publish(EVENT_SCANNER_PROGRESS, { + scannerStatusUpdate: { + progress: 100, + finished: true, + success: true, + message: `Done scanning ${ + Object.keys(scanner.imageProgress).length + } photos`, + }, + }) + } catch (e) { + console.error(`SCANNER ERROR: ${e.message}\n${e.stack}`) + scanner.pubsub.publish(EVENT_SCANNER_PROGRESS, { + scannerStatusUpdate: { + progress: 0, + finished: true, + success: false, + message: `Scanner error: ${e.message}`, + }, + }) + } finally { + scanner.isRunning = false + } +} + class PhotoScanner { constructor(driver) { this.driver = driver @@ -18,97 +78,67 @@ class PhotoScanner { this.scanUser = this.scanUser.bind(this) this.scanAll = this.scanAll.bind(this) - this.imagesToProgress = 0 - this.finishedImages = 0 + this.imageProgress = {} - this.markImageToProgress = () => { - this.imagesToProgress++ + this.markImageToProgress = imageId => { + if (!this.imageProgress[imageId]) this.imageProgress[imageId] = false } - this.markFinishedImage = () => { - this.finishedImages++ + this.markFinishedImage = imageId => { + this.imageProgress[imageId] = true + this.broadcastProgress() } + this.finishedImages = () => + Object.values(this.imageProgress).reduce((prev, x) => { + x ? prev++ : prev + return prev + }, 0) + this.broadcastProgress = _.debounce(() => { - if (this.imagesToProgress == 0) return + if (!this.isRunning) return + if (Object.keys(this.imageProgress).length == 0) return - console.log( - `Progress: ${(this.finishedImages / this.imagesToProgress) * 100}` - ) + let progress = + (this.finishedImages() / Object.keys(this.imageProgress).length) * 100 + + console.log(`Progress: ${progress}`) this.pubsub.publish(EVENT_SCANNER_PROGRESS, { scannerStatusUpdate: { - progress: (this.finishedImages / this.imagesToProgress) * 100, + progress, finished: false, success: true, - errorMessage: '', + message: `${this.finishedImages()} photos scanned`, }, }) }, 250) + + this.markImageToProgress = this.markImageToProgress.bind(this) + this.markFinishedImage = this.markFinishedImage.bind(this) + this.finishedImages = this.finishedImages.bind(this) } async scanUser(user) { - await _scanUser({ driver: this.driver, scanAlbum: this.scanAlbum }, user) + await _execScan(this, async () => { + await _scanUser({ driver: this.driver, scanAlbum: this.scanAlbum }, user) + }) } async scanAlbum(album) { - await _scanAlbum( - { - driver: this.driver, - markImageToProgress: this.markImageToProgress, - markFinishedImage: this.markFinishedImage, - processImage: this.processImage, - }, - album - ) + await _execScan(this, async () => { + await _scanAlbum(this, album) + }) } async processImage(id) { - await _processImage( - { - driver: this.driver, - markFinishedImage: this.markFinishedImage, - }, - id - ) - - this.broadcastProgress() + await _execScan(this, async () => { + await _processImage(this, id) + }) } async scanAll() { - this.pubsub.publish(EVENT_SCANNER_PROGRESS, { - scannerStatusUpdate: { - progress: 0, - finished: false, - success: true, - errorMessage: '', - }, - }) - - try { - await _scanAll({ driver: this.driver, scanUser: this.scanUser }) - } catch (error) { - this.pubsub.publish(EVENT_SCANNER_PROGRESS, { - scannerStatusUpdate: { - progress: 0, - finished: false, - success: false, - errorMessage: error.message, - }, - }) - throw error - } - - console.log( - `Done scanning ${this.finishedImages} of ${this.imagesToProgress}` - ) - - this.pubsub.publish(EVENT_SCANNER_PROGRESS, { - scannerStatusUpdate: { - progress: 100, - finished: true, - success: true, - errorMessage: '', - }, + await _execScan(this, async () => { + await _scanAll(this) }) } } diff --git a/api/src/scanner/processImage.js b/api/src/scanner/processImage.js index 78357f6d..651d3ec8 100644 --- a/api/src/scanner/processImage.js +++ b/api/src/scanner/processImage.js @@ -54,7 +54,8 @@ async function addExifTags({ session, photo }) { console.log('Added exif tags to photo', photo.path) } -export default async function processImage({ driver, markFinishedImage }, id) { +export default async function processImage(scanner, id) { + const { driver, markFinishedImage } = scanner const session = driver.session() const result = await session.run( @@ -77,7 +78,7 @@ export default async function processImage({ driver, markFinishedImage }, id) { ) if (urlResult.records.length == 2) { - markFinishedImage() + markFinishedImage(id) session.close() @@ -196,5 +197,5 @@ export default async function processImage({ driver, markFinishedImage }, id) { session.close() - markFinishedImage() + markFinishedImage(id) } diff --git a/api/src/scanner/scanAlbum.js b/api/src/scanner/scanAlbum.js index 0ec9c327..64a5abf0 100644 --- a/api/src/scanner/scanAlbum.js +++ b/api/src/scanner/scanAlbum.js @@ -2,12 +2,12 @@ import fs from 'fs-extra' import path from 'path' import generateID from '../id-generator' import { isImage, getImageCachePath } from './utils' +import _processImage from './processImage' -export default async function scanAlbum( - { driver, markImageToProgress, markFinishedImage, processImage }, - album -) { +export default async function scanAlbum(scanner, album) { + const { driver, markImageToProgress } = scanner const { title, path: albumPath, id } = album + console.log('Scanning album', title) let processedImages = [] @@ -22,8 +22,6 @@ export default async function scanAlbum( if (await isImage(itemPath)) { const session = driver.session() - markImageToProgress() - const photoResult = await session.run( `MATCH (p:Photo {path: {imgPath} })<--(a:Album {id: {albumId}}) RETURN p`, { @@ -36,29 +34,33 @@ export default async function scanAlbum( // console.log(`Photo already exists ${item}`) const photoId = photoResult.records[0].get('p').properties.id + markImageToProgress(photoId) const thumbnailPath = path.resolve( getImageCachePath(photoId, id), 'thumbnail.jpg' ) - processingImagePromises.push(processImage(photoId)) + processingImagePromises.push(_processImage(scanner, photoId)) } else { console.log(`Found new image at ${itemPath}`) - const imageId = generateID() + + const photoId = generateID() + markImageToProgress(photoId) + await session.run( `MATCH (a:Album { id: {albumId} }) CREATE (p:Photo {id: {id}, path: {path}, title: {title} }) CREATE (a)-[:CONTAINS]->(p)`, { - id: imageId, + id: photoId, path: itemPath, title: item, albumId: id, } ) - processingImagePromises.push(processImage(imageId)) + processingImagePromises.push(_processImage(scanner, photoId)) } } } @@ -91,4 +93,6 @@ export default async function scanAlbum( await Promise.all(processingImagePromises) console.log('Done processing album', album.title) + + scanner.broadcastProgress() } diff --git a/api/src/scanner/scanAll.js b/api/src/scanner/scanAll.js index 8e77a8bf..c8970b89 100644 --- a/api/src/scanner/scanAll.js +++ b/api/src/scanner/scanAll.js @@ -1,4 +1,7 @@ -export default function scanAll({ driver, scanUser }) { +import _scanUser from './scanUser' + +export default function scanAll(scanner) { + const { driver } = scanner return new Promise((resolve, reject) => { let session = driver.session() @@ -20,7 +23,7 @@ export default function scanAll({ driver, scanUser }) { for (let user of usersToScan) { try { - await scanUser(user) + await _scanUser(scanner, user) } catch (reason) { console.log( `User scan exception for user ${user.username} ${reason}` @@ -28,6 +31,8 @@ export default function scanAll({ driver, scanUser }) { reject(reason) } } + + resolve() }, onError: error => { session.close() diff --git a/api/src/scanner/scanUser.js b/api/src/scanner/scanUser.js index 7d031a69..f354bf37 100644 --- a/api/src/scanner/scanUser.js +++ b/api/src/scanner/scanUser.js @@ -2,9 +2,12 @@ import fs from 'fs-extra' import { resolve as pathResolve } from 'path' import generateID from '../id-generator' import { isImage, getAlbumCachePath } from './utils' +import _scanAlbum from './scanAlbum' -export default async function scanUser({ driver, scanAlbum }, user) { - console.log('Scanning user', user.username, 'at', user.path) +export default async function scanUser(scanner, user) { + const { driver } = scanner + + console.log('Scanning user', user.username, 'at', user.rootPath) let foundAlbumIds = [] @@ -50,7 +53,7 @@ export default async function scanUser({ driver, scanAlbum }, user) { foundImageOrAlbum = true nextParentAlbum = album.id foundAlbumIds.push(album.id) - await scanAlbum(album) + await _scanAlbum(scanner, album) continue } @@ -106,7 +109,7 @@ export default async function scanUser({ driver, scanAlbum }, user) { } foundAlbumIds.push(album.id) - await scanAlbum(album) + await _scanAlbum(scanner, album) session.close() } diff --git a/api/src/schema.graphql b/api/src/schema.graphql index 27bd9327..aec27837 100644 --- a/api/src/schema.graphql +++ b/api/src/schema.graphql @@ -95,8 +95,8 @@ type AuthorizeResult { type ScannerResult { finished: Boolean! success: Boolean! - errorMessage: String progress: Float + message: String } type Result { diff --git a/ui/src/Messages.js b/ui/src/Messages.js index 0c2b6bbc..81c4fe0a 100644 --- a/ui/src/Messages.js +++ b/ui/src/Messages.js @@ -10,7 +10,7 @@ const syncSubscription = gql` scannerStatusUpdate { finished success - errorMessage + message progress } } @@ -49,7 +49,7 @@ const MessageProgress = ({ header, content, percent = 0, ...props }) => { MessageProgress.propTypes = { header: PropTypes.string, - content: PropTypes.element, + content: PropTypes.any, percent: PropTypes.number, } @@ -90,9 +90,7 @@ class Messages extends Component { this.setState({ showSyncMessage: false }) }} header={update.finished ? 'Synced' : 'Syncing'} - content={ - update.finished ? 'Finished syncing' : 'Syncing in progress' - } + content={update.message} percent={update.progress} /> ) diff --git a/ui/src/Pages/SettingsPage/SettingsPage.js b/ui/src/Pages/SettingsPage/SettingsPage.js index 455e714a..340d3a4b 100644 --- a/ui/src/Pages/SettingsPage/SettingsPage.js +++ b/ui/src/Pages/SettingsPage/SettingsPage.js @@ -10,7 +10,7 @@ const scanMutation = gql` mutation scanAllMutation { scanAll { success - errorMessage + message } } ` diff --git a/ui/src/components/albumGallery/AlbumBox.js b/ui/src/components/albumGallery/AlbumBox.js index a4dd449e..275a85c1 100644 --- a/ui/src/components/albumGallery/AlbumBox.js +++ b/ui/src/components/albumGallery/AlbumBox.js @@ -57,7 +57,8 @@ export const AlbumBox = ({ album, customLink, ...props }) => { thumbnail = thumbnail || - (album.subAlbums[0] && + (album.subAlbums && + album.subAlbums[0] && album.subAlbums[0].photos[0] && album.subAlbums[0].photos[0].thumbnail.url)