Process jpgs if raw processing is disabled. (#1096)

api/scanner/media_encoding/executable_worker/executable_worker_test.go

@@ -2,9 +2,6 @@ package executable_worker_test
 
 import (
 	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
 	"testing"
 
 	"github.com/photoview/photoview/api/scanner/media_encoding/executable_worker"
@@ -15,35 +12,7 @@ func TestMain(m *testing.M) {
 	os.Exit(test_utils.IntegrationTestRun(m))
 }
 
-func setPathWithCurrent(paths ...string) func() {
+const testdataBinPath = "./testdata/bin"
-	_, file, _, ok := runtime.Caller(0)
-	if !ok {
-		return func() {
-			// Return an empty function in case of error
-		}
-	}
-
-	base := filepath.Dir(file)
-
-	for i, path := range paths {
-		paths[i] = filepath.Join(base, path)
-	}
-
-	originalPath := os.Getenv("PATH")
-	os.Setenv("PATH", strings.Join(paths, ":"))
-
-	return func() {
-		os.Setenv("PATH", originalPath)
-	}
-}
-
-func setEnv(key, value string) func() {
-	org := os.Getenv(key)
-	os.Setenv(key, value)
-	return func() {
-		os.Setenv(key, org)
-	}
-}
 
 func TestInitFfprobePath(t *testing.T) {
 	t.Run("PathFail", func(t *testing.T) {
@@ -54,10 +23,10 @@ func TestInitFfprobePath(t *testing.T) {
 	})
 
 	t.Run("VersionFail", func(t *testing.T) {
-		donePath := setPathWithCurrent("./testdata/bin")
+		donePath := test_utils.SetPathWithCurrent(testdataBinPath)
 		defer donePath()
 
-		doneEnv := setEnv("FAIL_WITH", "expect failure")
+		doneEnv := test_utils.SetEnv("FAIL_WITH", "expect failure")
 		defer doneEnv()
 
 		err := executable_worker.SetFfprobePath()
@@ -67,7 +36,7 @@ func TestInitFfprobePath(t *testing.T) {
 	})
 
 	t.Run("Succeed", func(t *testing.T) {
-		donePath := setPathWithCurrent("./testdata/bin")
+		donePath := test_utils.SetPathWithCurrent(testdataBinPath)
 		defer donePath()
 
 		err := executable_worker.SetFfprobePath()

api/scanner/media_encoding/executable_worker/ffmpeg_cli_test.go

@@ -5,12 +5,13 @@ import (
 	"testing"
 
 	"github.com/photoview/photoview/api/scanner/media_encoding/executable_worker"
+	"github.com/photoview/photoview/api/test_utils"
 	"github.com/photoview/photoview/api/utils"
 	"gopkg.in/vansante/go-ffprobe.v2"
 )
 
 func TestFfmpegNotExist(t *testing.T) {
-	done := setPathWithCurrent()
+	done := test_utils.SetPathWithCurrent()
 	defer done()
 
 	executable_worker.InitializeExecutableWorkers()
@@ -21,10 +22,10 @@ func TestFfmpegNotExist(t *testing.T) {
 }
 
 func TestFfmpegIgnore(t *testing.T) {
-	donePath := setPathWithCurrent("./testdata/bin")
+	donePath := test_utils.SetPathWithCurrent(testdataBinPath)
 	defer donePath()
 
-	doneEnv := setEnv("PHOTOVIEW_DISABLE_VIDEO_ENCODING", "true")
+	doneEnv := test_utils.SetEnv("PHOTOVIEW_DISABLE_VIDEO_ENCODING", "true")
 	defer doneEnv()
 
 	executable_worker.InitializeExecutableWorkers()
@@ -35,17 +36,17 @@ func TestFfmpegIgnore(t *testing.T) {
 }
 
 func TestFfmpeg(t *testing.T) {
-	done := setPathWithCurrent("./testdata/bin")
+	done := test_utils.SetPathWithCurrent(testdataBinPath)
 	defer done()
 
 	executable_worker.InitializeExecutableWorkers()
 
 	if !executable_worker.Ffmpeg.IsInstalled() {
-		t.Error("Ffmpeg should be installed")
+		t.Fatal("Ffmpeg should be installed")
 	}
 
 	t.Run("EncodeMp4Failed", func(t *testing.T) {
-		doneEnv := setEnv("FAIL_WITH", "expect failure")
+		doneEnv := test_utils.SetEnv("FAIL_WITH", "expect failure")
 		defer doneEnv()
 
 		err := executable_worker.Ffmpeg.EncodeMp4("input", "output")
@@ -70,7 +71,7 @@ func TestFfmpeg(t *testing.T) {
 		},
 	}
 	t.Run("EncodeVideoThumbnailMp4Failed", func(t *testing.T) {
-		doneEnv := setEnv("FAIL_WITH", "expect failure")
+		doneEnv := test_utils.SetEnv("FAIL_WITH", "expect failure")
 		defer doneEnv()
 
 		err := executable_worker.Ffmpeg.EncodeVideoThumbnail("input", "output", probeData)
@@ -91,15 +92,15 @@ func TestFfmpeg(t *testing.T) {
 }
 
 func TestFfmpegWithHWAcc(t *testing.T) {
-	doneCodec := setEnv(utils.EnvVideoHardwareAcceleration.GetName(), "qsv")
+	doneCodec := test_utils.SetEnv(utils.EnvVideoHardwareAcceleration.GetName(), "qsv")
 	defer doneCodec()
 
-	donePath := setPathWithCurrent("./testdata/bin")
+	donePath := test_utils.SetPathWithCurrent(testdataBinPath)
 	defer donePath()
 
 	executable_worker.InitializeExecutableWorkers()
 
-	doneEnv := setEnv("FAIL_WITH", "expect failure")
+	doneEnv := test_utils.SetEnv("FAIL_WITH", "expect failure")
 	defer doneEnv()
 
 	err := executable_worker.Ffmpeg.EncodeMp4("input", "output")
@@ -112,15 +113,15 @@ func TestFfmpegWithHWAcc(t *testing.T) {
 }
 
 func TestFfmpegWithCustomCOdec(t *testing.T) {
-	doneCodec := setEnv(utils.EnvVideoHardwareAcceleration.GetName(), "_custom")
+	doneCodec := test_utils.SetEnv(utils.EnvVideoHardwareAcceleration.GetName(), "_custom")
 	defer doneCodec()
 
-	donePath := setPathWithCurrent("./testdata/bin")
+	donePath := test_utils.SetPathWithCurrent(testdataBinPath)
 	defer donePath()
 
 	executable_worker.InitializeExecutableWorkers()
 
-	doneEnv := setEnv("FAIL_WITH", "expect failure")
+	doneEnv := test_utils.SetEnv("FAIL_WITH", "expect failure")
 	defer doneEnv()
 
 	err := executable_worker.Ffmpeg.EncodeMp4("input", "output")

api/scanner/media_encoding/executable_worker/magick_cli_test.go

@@ -5,12 +5,11 @@ import (
 	"testing"
 
 	"github.com/photoview/photoview/api/scanner/media_encoding/executable_worker"
+	"github.com/photoview/photoview/api/test_utils"
 )
 
-const testdataBinPath = "./testdata/bin"
-
 func TestMagickCliNotExist(t *testing.T) {
-	done := setPathWithCurrent()
+	done := test_utils.SetPathWithCurrent()
 	defer done()
 
 	executable_worker.InitializeExecutableWorkers()
@@ -20,10 +19,10 @@ func TestMagickCliNotExist(t *testing.T) {
 }
 
 func TestMagickCliIgnore(t *testing.T) {
-	donePath := setPathWithCurrent(testdataBinPath)
+	donePath := test_utils.SetPathWithCurrent(testdataBinPath)
 	defer donePath()
 
-	doneDisableRaw := setEnv("PHOTOVIEW_DISABLE_RAW_PROCESSING", "true")
+	doneDisableRaw := test_utils.SetEnv("PHOTOVIEW_DISABLE_RAW_PROCESSING", "true")
 	defer doneDisableRaw()
 
 	executable_worker.InitializeExecutableWorkers()
@@ -33,7 +32,7 @@ func TestMagickCliIgnore(t *testing.T) {
 }
 
 func TestMagickCliFail(t *testing.T) {
-	donePath := setPathWithCurrent(testdataBinPath)
+	donePath := test_utils.SetPathWithCurrent(testdataBinPath)
 	defer donePath()
 
 	executable_worker.InitializeExecutableWorkers()
@@ -41,7 +40,7 @@ func TestMagickCliFail(t *testing.T) {
 		t.Fatal("MagickCli should be installed")
 	}
 
-	done := setEnv("FAIL_WITH", "failure")
+	done := test_utils.SetEnv("FAIL_WITH", "failure")
 	defer done()
 
 	err := executable_worker.Magick.EncodeJpeg("input", "output", 70)
@@ -55,7 +54,7 @@ func TestMagickCliFail(t *testing.T) {
 }
 
 func TestMagickCliSucceed(t *testing.T) {
-	donePath := setPathWithCurrent(testdataBinPath)
+	donePath := test_utils.SetPathWithCurrent(testdataBinPath)
 	defer donePath()
 
 	executable_worker.InitializeExecutableWorkers()

api/scanner/scanner_tasks/processing_tasks/counterpart_files_task.go

@@ -10,6 +10,7 @@ import (
 	"github.com/photoview/photoview/api/scanner/media_type"
 	"github.com/photoview/photoview/api/scanner/scanner_task"
 	"github.com/photoview/photoview/api/scanner/scanner_utils"
+	"github.com/photoview/photoview/api/utils"
 	"github.com/pkg/errors"
 )
 
@@ -18,10 +19,27 @@ type CounterpartFilesTask struct {
 }
 
 func (t CounterpartFilesTask) MediaFound(ctx scanner_task.TaskContext, fileInfo fs.FileInfo, mediaPath string) (skip bool, err error) {
+	ext := filepath.Ext(mediaPath)
+	fileExtType, found := media_type.GetExtensionMediaType(ext)
+	if !found {
+		return true, nil
+	}
 
-	// Skip the JPEGs that are compressed version of raw files
+	if utils.EnvDisableRawProcessing.GetBool() {
-	counterpartFile := scanForRawCounterpartFile(mediaPath)
+		if fileExtType.IsRaw() {
-	if counterpartFile != nil {
+			return true, nil
+		}
+
+		// Don't skip the JPEGs if raw processing is disabled. Treat them as standalone files.
+		return false, nil
+	}
+
+	if !fileExtType.IsBasicTypeSupported() {
+		return false, nil
+	}
+
+	rawPath := media_type.RawCounterpart(mediaPath)
+	if rawPath != nil {
 		return true, nil
 	}
 
@@ -67,21 +85,3 @@ func scanForCompressedCounterpartFile(imagePath string) *string {
 
 	return nil
 }
-
-func scanForRawCounterpartFile(imagePath string) *string {
-	ext := filepath.Ext(imagePath)
-	fileExtType, found := media_type.GetExtensionMediaType(ext)
-
-	if found {
-		if !fileExtType.IsBasicTypeSupported() {
-			return nil
-		}
-	}
-
-	rawPath := media_type.RawCounterpart(imagePath)
-	if rawPath != nil {
-		return rawPath
-	}
-
-	return nil
-}

api/scanner/scanner_tasks/processing_tasks/counterpart_files_task_test.go

@@ -0,0 +1,99 @@
+package processing_tasks_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/photoview/photoview/api/scanner/scanner_task"
+	"github.com/photoview/photoview/api/scanner/scanner_tasks/processing_tasks"
+	"github.com/photoview/photoview/api/test_utils"
+	"github.com/photoview/photoview/api/utils"
+)
+
+func TestCounterpartFilesTaskMediaFound(t *testing.T) {
+	tests := []struct {
+		name                 string
+		file                 string
+		disableRawProcessing bool
+		wantSkip             bool
+	}{
+		{
+			name:                 "StandaloneProcessRaw",
+			file:                 "standalone.jpg",
+			disableRawProcessing: false,
+			wantSkip:             false,
+		},
+		{
+			name:                 "StandaloneNotProcessRaw",
+			file:                 "standalone.jpg",
+			disableRawProcessing: true,
+			wantSkip:             false,
+		},
+		{
+			name:                 "RawJpegProcessRaw",
+			file:                 "fujifilm_raw.jpg",
+			disableRawProcessing: false,
+			wantSkip:             true,
+		},
+		{
+			name:                 "RawJpegNotProcessRaw",
+			file:                 "fujifilm_raw.jpg",
+			disableRawProcessing: true,
+			wantSkip:             false,
+		},
+		{
+			name:                 "RawProcessRaw",
+			file:                 "fujifilm_raw.raf",
+			disableRawProcessing: false,
+			wantSkip:             false,
+		},
+		{
+			name:                 "RawNotProcessRaw",
+			file:                 "fujifilm_raw.raf",
+			disableRawProcessing: true,
+			wantSkip:             true,
+		},
+		{
+			name:                 "UnknownProcessRaw",
+			file:                 "file.unknown",
+			disableRawProcessing: false,
+			wantSkip:             true,
+		},
+		{
+			name:                 "UnknownNotProcessRaw",
+			file:                 "file.unknown",
+			disableRawProcessing: true,
+			wantSkip:             true,
+		},
+	}
+
+	mediaPath := test_utils.PathFromAPIRoot("scanner/test_data/fake_media")
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			done := test_utils.SetEnv(string(utils.EnvDisableRawProcessing), fmt.Sprintf("%v", tc.disableRawProcessing))
+			defer done()
+
+			ctx := scanner_task.NewTaskContext(context.Background(), nil, nil, nil)
+
+			fname := filepath.Join(mediaPath, tc.file)
+			fi, err := os.Stat(fname)
+			if err != nil {
+				t.Fatalf("Stat(%q) error: %v", fname, err)
+			}
+
+			var task processing_tasks.CounterpartFilesTask
+			got, err := task.MediaFound(ctx, fi, fname)
+			if err != nil {
+				t.Fatalf("task.MediaFound(ctx, %q) error: %v", fname, err)
+			}
+
+			if got, want := got, tc.wantSkip; got != want {
+				t.Errorf("task.MediaFound(ctx, %q) = (skip)%v, want skip: %v", fname, got, want)
+			}
+		})
+	}
+}

api/test_utils/env.go

@@ -0,0 +1,49 @@
+package test_utils
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+func SetPathWithCurrent(paths ...string) func() {
+	_, file, _, ok := runtime.Caller(1)
+	if !ok {
+		return func() {
+			// Return an empty function in case of error
+		}
+	}
+
+	base := filepath.Dir(file)
+
+	for i, path := range paths {
+		paths[i] = filepath.Join(base, path)
+	}
+
+	originalPath := os.Getenv("PATH")
+	os.Setenv("PATH", strings.Join(paths, ":"))
+
+	return func() {
+		os.Setenv("PATH", originalPath)
+	}
+}
+
+func SetEnv(key, value string) func() {
+	org := os.Getenv(key)
+	os.Setenv(key, value)
+	return func() {
+		os.Setenv(key, org)
+	}
+}
+
+func PathFromAPIRoot(rootRelatedPath string) string {
+	_, file, _, ok := runtime.Caller(0)
+	if !ok {
+		panic("Can't get the path of current function. It should not happen.")
+	}
+
+	base := filepath.Dir(file)
+
+	return filepath.Join(base, "..", rootRelatedPath)
+}

api/test_utils/env_test.go

@@ -0,0 +1,26 @@
+package test_utils
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestPathFromAPIRoot(t *testing.T) {
+	if got, want := PathFromAPIRoot("./server.go"), "/api/server.go"; !strings.HasSuffix(got, want) {
+		t.Fatalf(`PathFromAPIRoot("./server.go") = %q, want a suffix: %q`, got, want)
+	}
+}
+
+func TestSetPathWithCurrent(t *testing.T) {
+	pathDone := SetEnv("PATH", "")
+	defer pathDone()
+
+	testDone := SetPathWithCurrent("./test")
+	defer testDone()
+
+	path := os.Getenv("PATH")
+	if got, want := path, "api/test_utils/test"; !strings.HasSuffix(got, want) {
+		t.Errorf("path = %q, want a suffix: %q", got, want)
+	}
+}