Add simple scanner integration test

api/database/database.go

@@ -190,6 +190,10 @@ func MigrateDatabase(db *gorm.DB) error {
 
 func ClearDatabase(db *gorm.DB) error {
 	err := db.Transaction(func(tx *gorm.DB) error {
+		if err := tx.Exec("SET FOREIGN_KEY_CHECKS = 0;").Error; err != nil {
+			return err
+		}
+
 		sess := tx.Session(&gorm.Session{AllowGlobalUpdate: true})
 		for _, model := range database_models {
 			if err := sess.Delete(model).Error; err != nil {
@@ -197,6 +201,10 @@ func ClearDatabase(db *gorm.DB) error {
 			}
 		}
 
+		if err := tx.Exec("SET FOREIGN_KEY_CHECKS = 1;").Error; err != nil {
+			return err
+		}
+
 		return nil
 	})
 

api/scanner/queue.go

@@ -33,6 +33,8 @@ type ScannerQueue struct {
 	up_next     []ScannerJob
 	db          *gorm.DB
 	settings    ScannerQueueSettings
+	close_chan  *chan bool
+	running     bool
 }
 
 var global_scanner_queue ScannerQueue
@@ -56,6 +58,8 @@ func InitializeScannerQueue(db *gorm.DB) error {
 		up_next:     make([]ScannerJob, 0),
 		db:          db,
 		settings:    ScannerQueueSettings{max_concurrent_tasks: concurrentWorkers},
+		close_chan:  nil,
+		running:     true,
 	}
 
 	go global_scanner_queue.startBackgroundWorker()
@@ -63,6 +67,10 @@ func InitializeScannerQueue(db *gorm.DB) error {
 	return nil
 }
 
+func CloseScannerQueue() {
+	global_scanner_queue.CloseBackgroundWorker()
+}
+
 func ChangeScannerConcurrentWorkers(newMaxWorkers int) {
 	global_scanner_queue.mutex.Lock()
 	defer global_scanner_queue.mutex.Unlock()
@@ -78,60 +86,88 @@ func (queue *ScannerQueue) startBackgroundWorker() {
 	for {
 		log.Println("Queue waiting")
 		<-queue.idle_chan
-		log.Println("Queue waiting for lock")
+
 		queue.mutex.Lock()
-		log.Printf("Queue running: in_progress: %d, max_tasks: %d, queue_len: %d\n", len(queue.in_progress), queue.settings.max_concurrent_tasks, len(queue.up_next))
-
-		for len(queue.in_progress) < queue.settings.max_concurrent_tasks && len(queue.up_next) > 0 {
-			log.Println("Queue starting job")
-			nextJob := queue.up_next[0]
-			queue.up_next = queue.up_next[1:]
-			queue.in_progress = append(queue.in_progress, nextJob)
-
-			go func() {
-				log.Println("Starting job")
-				nextJob.Run(queue.db)
-				log.Println("Job finished")
-
-				// Delete finished job from queue
-				queue.mutex.Lock()
-				for i, x := range queue.in_progress {
-					if x == nextJob {
-						queue.in_progress[i] = queue.in_progress[len(queue.in_progress)-1]
-						queue.in_progress = queue.in_progress[0 : len(queue.in_progress)-1]
-						break
-					}
-				}
-				queue.mutex.Unlock()
-
-				queue.notify()
-			}()
-		}
-
-		in_progress_length := len(global_scanner_queue.in_progress)
-		up_next_length := len(global_scanner_queue.up_next)
-
+		should_stop := queue.close_chan != nil && len(queue.in_progress) == 0 && len(queue.up_next) == 0
+		queue.running = false
 		queue.mutex.Unlock()
 
-		if in_progress_length+up_next_length == 0 {
-			notification.BroadcastNotification(&models.Notification{
-				Key:      "global-scanner-progress",
-				Type:     models.NotificationTypeMessage,
-				Header:   fmt.Sprintf("Scanner complete"),
-				Content:  fmt.Sprintf("All jobs have been scanned"),
-				Positive: true,
-			})
-		} else {
-			notifyThrottle.Trigger(func() {
-				notification.BroadcastNotification(&models.Notification{
-					Key:     "global-scanner-progress",
-					Type:    models.NotificationTypeMessage,
-					Header:  fmt.Sprintf("Scanning media"),
-					Content: fmt.Sprintf("%d jobs in progress\n%d jobs waiting", in_progress_length, up_next_length),
-				})
-			})
+		if should_stop {
+			*queue.close_chan <- true
+			break
 		}
 
+		queue.processQueue(&notifyThrottle)
+	}
+
+	log.Println("Scanner background worker stopped")
+}
+
+func (queue *ScannerQueue) CloseBackgroundWorker() {
+	queue.mutex.Lock()
+	close_chan := make(chan bool)
+	queue.close_chan = &close_chan
+	queue.mutex.Unlock()
+
+	queue.notify()
+
+	log.Println("Waiting for scanner background worker to finish all jobs...")
+	<-close_chan
+}
+
+func (queue *ScannerQueue) processQueue(notifyThrottle *utils.Throttle) {
+	log.Println("Queue waiting for lock")
+	queue.mutex.Lock()
+	log.Printf("Queue running: in_progress: %d, max_tasks: %d, queue_len: %d\n", len(queue.in_progress), queue.settings.max_concurrent_tasks, len(queue.up_next))
+
+	for len(queue.in_progress) < queue.settings.max_concurrent_tasks && len(queue.up_next) > 0 {
+		log.Println("Queue starting job")
+		nextJob := queue.up_next[0]
+		queue.up_next = queue.up_next[1:]
+		queue.in_progress = append(queue.in_progress, nextJob)
+
+		go func() {
+			log.Println("Starting job")
+			nextJob.Run(queue.db)
+			log.Println("Job finished")
+
+			// Delete finished job from queue
+			queue.mutex.Lock()
+			for i, x := range queue.in_progress {
+				if x == nextJob {
+					queue.in_progress[i] = queue.in_progress[len(queue.in_progress)-1]
+					queue.in_progress = queue.in_progress[0 : len(queue.in_progress)-1]
+					break
+				}
+			}
+			queue.mutex.Unlock()
+
+			queue.notify()
+		}()
+	}
+
+	in_progress_length := len(global_scanner_queue.in_progress)
+	up_next_length := len(global_scanner_queue.up_next)
+
+	queue.mutex.Unlock()
+
+	if in_progress_length+up_next_length == 0 {
+		notification.BroadcastNotification(&models.Notification{
+			Key:      "global-scanner-progress",
+			Type:     models.NotificationTypeMessage,
+			Header:   fmt.Sprintf("Scanner complete"),
+			Content:  fmt.Sprintf("All jobs have been scanned"),
+			Positive: true,
+		})
+	} else {
+		notifyThrottle.Trigger(func() {
+			notification.BroadcastNotification(&models.Notification{
+				Key:     "global-scanner-progress",
+				Type:    models.NotificationTypeMessage,
+				Header:  fmt.Sprintf("Scanning media"),
+				Content: fmt.Sprintf("%d jobs in progress\n%d jobs waiting", in_progress_length, up_next_length),
+			})
+		})
 	}
 }
 

api/scanner/scanner_test.go

@@ -4,9 +4,56 @@ import (
 	"os"
 	"testing"
 
+	"github.com/photoview/photoview/api/graphql/models"
+	"github.com/photoview/photoview/api/scanner"
 	"github.com/photoview/photoview/api/test_utils"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestMain(m *testing.M) {
-	os.Exit(test_utils.UnitTestRun(m))
+	os.Exit(test_utils.IntegrationTestRun(m))
+}
+
+func TestFullScan(t *testing.T) {
+	test_utils.FilesystemTest(t)
+	db := test_utils.DatabaseTest(t)
+
+	pass := "1234"
+	user, err := models.RegisterUser(db, "test_user", &pass, true)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	root_album := models.Album{
+		Title: "root album",
+		Path:  "./test_data",
+	}
+
+	if !assert.NoError(t, db.Save(&root_album).Error) {
+		return
+	}
+
+	err = db.Model(user).Association("Albums").Append(&root_album)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	if !assert.NoError(t, scanner.InitializeScannerQueue(db)) {
+		return
+	}
+
+	if !assert.NoError(t, scanner.AddUserToQueue(user)) {
+		return
+	}
+
+	// wait for all jobs to finish
+	scanner.CloseScannerQueue()
+
+	var all_media []*models.Media
+	if !assert.NoError(t, db.Find(&all_media).Error) {
+		return
+	}
+
+	assert.Equal(t, 10, len(all_media))
+
 }

api/test_utils/integration_setup.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/joho/godotenv"
+	"github.com/photoview/photoview/api/utils"
 	"gorm.io/gorm"
 )
 
@@ -55,6 +56,7 @@ func FilesystemTest(t *testing.T) {
 	if !*integration_flags.Filesystem {
 		t.Skip("Filesystem integration tests disabled")
 	}
+	utils.ConfigureTestCache(t.TempDir())
 }
 
 func DatabaseTest(t *testing.T) *gorm.DB {

api/test_utils/test_db_manager.go

@@ -47,6 +47,8 @@ func (dbm *TestDBManager) setup() error {
 
 	dbm.DB = db
 
+	dbm.reset()
+
 	return nil
 }
 

api/utils/utils.go

@@ -42,8 +42,18 @@ func HandleError(message string, err error) PhotoviewError {
 	}
 }
 
+var test_cache_path string = ""
+
+func ConfigureTestCache(tmp_dir string) {
+	test_cache_path = tmp_dir
+}
+
 // MediaCachePath returns the path for where the media cache is located on the file system
 func MediaCachePath() string {
+	if test_cache_path != "" {
+		return test_cache_path
+	}
+
 	photoCache := EnvMediaCachePath.GetValue()
 	if photoCache == "" {
 		photoCache = "./media_cache"