Use hardware acceleration when converting videos. (#1056)

api/example.env

@@ -32,3 +32,8 @@ PHOTOVIEW_SERVE_UI=0
 # Set to 1 to set server in development mode, this enables graphql playground
 # Remove this if running in production
 PHOTOVIEW_DEVELOPMENT_MODE=1
+
+# Set the hardware acceleration when encoding videos.
+# Support `qsv`, `vaapi`, `nvenc`.
+# Only `qsv` is verified with `/dev/dri//dev/dri` devices.
+# PHOTOVIEW_VIDEO_HARDWARE_ACCELERATION=

api/scanner/media_encoding/executable_worker/executable_worker.go

@@ -6,96 +6,38 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/photoview/photoview/api/utils"
-	"github.com/pkg/errors"
 	"gopkg.in/vansante/go-ffprobe.v2"
 )
 
 func InitializeExecutableWorkers() {
 	Magick = newMagickCli()
-	FfmpegCli = newFfmpegWorker()
+	Ffmpeg = newFfmpegCli()
+
+	if err := SetFfprobePath(); err != nil {
+		log.Println("ffprobe init fail:", err)
+	}
 }
 
 var Magick *MagickCli = nil
-var FfmpegCli *FfmpegWorker = nil
+var Ffmpeg *FfmpegCli = nil
 
 type ExecutableWorker interface {
 	Path() string
 }
 
-type FfmpegWorker struct {
-	path string
-}
-
-func newFfmpegWorker() *FfmpegWorker {
-	if utils.EnvDisableVideoEncoding.GetBool() {
-		log.Printf("Executable worker disabled (%s=1): ffmpeg\n", utils.EnvDisableVideoEncoding.GetName())
-		return nil
-	}
-
-	path, err := exec.LookPath("ffmpeg")
+func SetFfprobePath() error {
+	path, err := exec.LookPath("ffprobe")
 	if err != nil {
-		log.Println("Executable worker not found: ffmpeg")
-	} else {
-		version, err := exec.Command(path, "-version").Output()
-		if err != nil {
-			log.Printf("Error getting version of ffmpeg: %s\n", err)
-			return nil
-		}
-
-		log.Printf("Found executable worker: ffmpeg (%s)\n", strings.Split(string(version), "\n")[0])
-
-		return &FfmpegWorker{
-			path: path,
-		}
-	}
-
-	return nil
-}
-
-func (worker *FfmpegWorker) IsInstalled() bool {
-	return worker != nil
-}
-
-func (worker *FfmpegWorker) EncodeMp4(inputPath string, outputPath string) error {
-	args := []string{
-		"-i",
-		inputPath,
-		"-vcodec", "h264",
-		"-acodec", "aac",
-		"-vf", "scale='min(1080,iw)':'min(1080,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2",
-		"-movflags", "+faststart+use_metadata_tags",
-		outputPath,
-	}
-
-	cmd := exec.Command(worker.path, args...)
-
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "encoding video using: %s", worker.path)
-	}
-
-	return nil
-}
-
-func (worker *FfmpegWorker) EncodeVideoThumbnail(inputPath string, outputPath string, probeData *ffprobe.ProbeData) error {
-
-	thumbnailOffsetSeconds := fmt.Sprintf("%d", int(probeData.Format.DurationSeconds*0.25))
-
-	args := []string{
-		"-ss", thumbnailOffsetSeconds, // grab frame at time offset
-		"-i",
-		inputPath,
-		"-vframes", "1", // output one frame
-		"-an", // disable audio
-		"-vf", "scale='min(1024,iw)':'min(1024,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2",
-		outputPath,
-	}
-
-	cmd := exec.Command(worker.path, args...)
-
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "encoding video using: %s", worker.path)
+		return fmt.Errorf("Executable ffprobe not found: %w", err)
 	}
 
+	version, err := exec.Command(path, "-version").Output()
+	if err != nil {
+		return fmt.Errorf("Executable ffprobe(%q) not executable: %w", path, err)
+	}
+
+	log.Println("Found ffprobe:", path, "version:", strings.Split(string(version), "\n")[0])
+	ffprobe.SetFFProbeBinPath(path)
+
 	return nil
 }

api/scanner/media_encoding/executable_worker/executable_worker_test.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/photoview/photoview/api/scanner/media_encoding/executable_worker"
 	"github.com/photoview/photoview/api/test_utils"
 )
 
@@ -43,3 +44,35 @@ func setEnv(key, value string) func() {
 		os.Setenv(key, org)
 	}
 }
+
+func TestInitFfprobePath(t *testing.T) {
+	t.Run("PathFail", func(t *testing.T) {
+		err := executable_worker.SetFfprobePath()
+		if err == nil {
+			t.Fatalf("InitFfprobePath() returns nil, want an error")
+		}
+	})
+
+	t.Run("VersionFail", func(t *testing.T) {
+		donePath := setPathWithCurrent("./testdata/bin")
+		defer donePath()
+
+		doneEnv := setEnv("FAIL_WITH", "expect failure")
+		defer doneEnv()
+
+		err := executable_worker.SetFfprobePath()
+		if err == nil {
+			t.Fatalf("InitFfprobePath() returns nil, want an error")
+		}
+	})
+
+	t.Run("Succeed", func(t *testing.T) {
+		donePath := setPathWithCurrent("./testdata/bin")
+		defer donePath()
+
+		err := executable_worker.SetFfprobePath()
+		if err != nil {
+			t.Fatalf("InitFfprobePath() returns %v, want nil", err)
+		}
+	})
+}

api/scanner/media_encoding/executable_worker/ffmpeg_cli.go

@@ -0,0 +1,108 @@
+package executable_worker
+
+import (
+	"fmt"
+	"log"
+	"os/exec"
+	"strings"
+
+	"github.com/photoview/photoview/api/utils"
+	"gopkg.in/vansante/go-ffprobe.v2"
+)
+
+const defaultCodec = "h264"
+
+var hwAccToCodec = map[string]string{
+	"qsv":   defaultCodec + "_qsv",
+	"vaapi": defaultCodec + "_vaapi",
+	"nvenc": defaultCodec + "_nvenc",
+}
+
+type FfmpegCli struct {
+	path       string
+	videoCodec string
+}
+
+func newFfmpegCli() *FfmpegCli {
+	if utils.EnvDisableVideoEncoding.GetBool() {
+		log.Printf("Executable worker disabled (%s=%q): ffmpeg\n", utils.EnvDisableVideoEncoding.GetName(), utils.EnvDisableVideoEncoding.GetValue())
+		return nil
+	}
+
+	path, err := exec.LookPath("ffmpeg")
+	if err != nil {
+		log.Println("Executable worker not found: ffmpeg")
+		return nil
+	}
+
+	version, err := exec.Command(path, "-version").Output()
+	if err != nil {
+		log.Printf("Error getting version of ffmpeg: %s\n", err)
+		return nil
+	}
+
+	hwAcc := utils.EnvVideoHardwareAcceleration.GetValue()
+	codec, ok := hwAccToCodec[hwAcc]
+	if !ok {
+		if strings.HasPrefix(hwAcc, "_") {
+			// A secret way to set the codec directly.
+			codec = hwAcc[1:]
+		} else {
+			codec = defaultCodec
+		}
+	}
+
+	log.Printf("Found executable worker: ffmpeg (%s) with codec %q\n", strings.Split(string(version), "\n")[0], codec)
+
+	return &FfmpegCli{
+		path:       path,
+		videoCodec: codec,
+	}
+}
+
+func (worker *FfmpegCli) IsInstalled() bool {
+	return worker != nil
+}
+
+func (worker *FfmpegCli) EncodeMp4(inputPath string, outputPath string) error {
+	args := []string{
+		"-i",
+		inputPath,
+		"-vcodec", worker.videoCodec,
+		"-acodec", "aac",
+		"-vf", "scale='min(1080,iw)':'min(1080,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2",
+		"-movflags", "+faststart+use_metadata_tags",
+		outputPath,
+	}
+
+	cmd := exec.Command(worker.path, args...)
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("encoding video with %q %v error: %w", worker.path, args, err)
+	}
+
+	return nil
+}
+
+func (worker *FfmpegCli) EncodeVideoThumbnail(inputPath string, outputPath string, probeData *ffprobe.ProbeData) error {
+
+	thumbnailOffsetSeconds := fmt.Sprintf("%.f", probeData.Format.DurationSeconds*0.25)
+
+	args := []string{
+		"-ss", thumbnailOffsetSeconds, // grab frame at time offset
+		"-i",
+		inputPath,
+		"-vframes", "1", // output one frame
+		"-an", // disable audio
+		"-vf", "scale='min(1024,iw)':'min(1024,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2",
+		outputPath,
+	}
+
+	cmd := exec.Command(worker.path, args...)
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("encoding video thumbnail with %q %v error: %w", worker.path, args, err)
+	}
+
+	return nil
+}

api/scanner/media_encoding/executable_worker/ffmpeg_cli_test.go

@@ -0,0 +1,133 @@
+package executable_worker_test
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/photoview/photoview/api/scanner/media_encoding/executable_worker"
+	"github.com/photoview/photoview/api/utils"
+	"gopkg.in/vansante/go-ffprobe.v2"
+)
+
+func TestFfmpegNotExist(t *testing.T) {
+	done := setPathWithCurrent()
+	defer done()
+
+	executable_worker.InitializeExecutableWorkers()
+
+	if executable_worker.Ffmpeg.IsInstalled() {
+		t.Error("Ffmpeg should not be installed, but is found:", executable_worker.Ffmpeg)
+	}
+}
+
+func TestFfmpegIgnore(t *testing.T) {
+	donePath := setPathWithCurrent("./testdata/bin")
+	defer donePath()
+
+	doneEnv := setEnv("PHOTOVIEW_DISABLE_VIDEO_ENCODING", "true")
+	defer doneEnv()
+
+	executable_worker.InitializeExecutableWorkers()
+
+	if executable_worker.Ffmpeg.IsInstalled() {
+		t.Error("Ffmpeg should be ignored (as it is disabled), but is initialized:", executable_worker.Ffmpeg)
+	}
+}
+
+func TestFfmpeg(t *testing.T) {
+	done := setPathWithCurrent("./testdata/bin")
+	defer done()
+
+	executable_worker.InitializeExecutableWorkers()
+
+	if !executable_worker.Ffmpeg.IsInstalled() {
+		t.Error("Ffmpeg should be installed")
+	}
+
+	t.Run("EncodeMp4Failed", func(t *testing.T) {
+		doneEnv := setEnv("FAIL_WITH", "expect failure")
+		defer doneEnv()
+
+		err := executable_worker.Ffmpeg.EncodeMp4("input", "output")
+		if err == nil {
+			t.Fatalf("Ffmpeg.EncodeMp4(...) = nil, should be an error.")
+		}
+		if got, want := err.Error(), `^encoding video with ".*/testdata/bin/ffmpeg" \[-i input -vcodec h264 .* output\] error: .*$`; !regexp.MustCompile(want).MatchString(got) {
+			t.Errorf("Ffmpeg.EncodeMp4(...) = %q, should be as reg pattern %q", got, want)
+		}
+	})
+
+	t.Run("EncodeMp4Succeeded", func(t *testing.T) {
+		err := executable_worker.Ffmpeg.EncodeMp4("input", "output")
+		if err != nil {
+			t.Fatalf("Ffmpeg.EncodeMp4(...) = %v, should be nil.", err)
+		}
+	})
+
+	probeData := &ffprobe.ProbeData{
+		Format: &ffprobe.Format{
+			DurationSeconds: 10,
+		},
+	}
+	t.Run("EncodeVideoThumbnailMp4Failed", func(t *testing.T) {
+		doneEnv := setEnv("FAIL_WITH", "expect failure")
+		defer doneEnv()
+
+		err := executable_worker.Ffmpeg.EncodeVideoThumbnail("input", "output", probeData)
+		if err == nil {
+			t.Fatalf("Ffmpeg.EncodeVideoThumbnail(...) = nil, should be an error.")
+		}
+		if got, want := err.Error(), `^encoding video thumbnail with ".*/testdata/bin/ffmpeg" \[-ss 2 -i input .* output\] error: .*$`; !regexp.MustCompile(want).MatchString(got) {
+			t.Errorf("Ffmpeg.EncodeVideoThumbnail(...) = %q, should be as reg pattern %q", got, want)
+		}
+	})
+
+	t.Run("EncodeVideoThumbnailSucceeded", func(t *testing.T) {
+		err := executable_worker.Ffmpeg.EncodeVideoThumbnail("input", "output", probeData)
+		if err != nil {
+			t.Fatalf("Ffmpeg.EncodeVideoThumbnail(...) = %v, should be nil.", err)
+		}
+	})
+}
+
+func TestFfmpegWithHWAcc(t *testing.T) {
+	doneCodec := setEnv(utils.EnvVideoHardwareAcceleration.GetName(), "qsv")
+	defer doneCodec()
+
+	donePath := setPathWithCurrent("./testdata/bin")
+	defer donePath()
+
+	executable_worker.InitializeExecutableWorkers()
+
+	doneEnv := setEnv("FAIL_WITH", "expect failure")
+	defer doneEnv()
+
+	err := executable_worker.Ffmpeg.EncodeMp4("input", "output")
+	if err == nil {
+		t.Fatalf("Ffmpeg.EncodeMp4(...) = nil, should be an error.")
+	}
+	if got, want := err.Error(), `^encoding video with ".*/testdata/bin/ffmpeg" \[-i input -vcodec h264_qsv .* output\] error: .*$`; !regexp.MustCompile(want).MatchString(got) {
+		t.Errorf("Ffmpeg.EncodeMp4(...) = %q, should be as reg pattern %q", got, want)
+	}
+}
+
+func TestFfmpegWithCustomCOdec(t *testing.T) {
+	doneCodec := setEnv(utils.EnvVideoHardwareAcceleration.GetName(), "_custom")
+	defer doneCodec()
+
+	donePath := setPathWithCurrent("./testdata/bin")
+	defer donePath()
+
+	executable_worker.InitializeExecutableWorkers()
+
+	doneEnv := setEnv("FAIL_WITH", "expect failure")
+	defer doneEnv()
+
+	err := executable_worker.Ffmpeg.EncodeMp4("input", "output")
+	if err == nil {
+		t.Fatalf("Ffmpeg.EncodeMp4(...) = nil, should be an error.")
+	}
+	if got, want := err.Error(), `^encoding video with ".*/testdata/bin/ffmpeg" \[-i input -vcodec custom .* output\] error: .*$`; !regexp.MustCompile(want).MatchString(got) {
+		t.Errorf("Ffmpeg.EncodeMp4(...) = %q, should be as reg pattern %q", got, want)
+	}
+}

api/scanner/media_encoding/executable_worker/testdata/bin/ffmpeg

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+
+: ${FAIL_WITH=""}
+
+case "$1" in
+  "--version")
+      echo ffmpeg: version fake
+      ;;
+esac
+
+if [ "${FAIL_WITH}" != "" ]
+then
+  echo ${FAIL_WITH}
+  exit -1
+fi
+
+echo $@

api/scanner/media_encoding/executable_worker/testdata/bin/ffprobe

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+
+: ${FAIL_WITH=""}
+
+case "$1" in
+  "--version")
+      echo ffprobe: version fake
+      ;;
+esac
+
+if [ "${FAIL_WITH}" != "" ]
+then
+  echo ${FAIL_WITH}
+  exit -1
+fi
+
+echo $@

api/scanner/media_type/media_type.go

@@ -264,7 +264,7 @@ func (imgType *MediaType) IsSupported() bool {
 		return true
 	}
 
-	if executable_worker.FfmpegCli.IsInstalled() && imgType.IsVideo() {
+	if executable_worker.Ffmpeg.IsInstalled() && imgType.IsVideo() {
 		return true
 	}
 

api/scanner/scanner_tasks/processing_tasks/process_video_task.go

@@ -94,7 +94,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 
 		webVideoPath := path.Join(mediaCachePath, webVideoName)
 
-		err = executable_worker.FfmpegCli.EncodeMp4(video.Path, webVideoPath)
+		err = executable_worker.Ffmpeg.EncodeMp4(video.Path, webVideoPath)
 		if err != nil {
 			return []*models.MediaURL{}, errors.Wrapf(err, "could not encode mp4 video (%s)", video.Path)
 		}
@@ -139,7 +139,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 
 		thumbImagePath := path.Join(mediaCachePath, videoThumbName)
 
-		err = executable_worker.FfmpegCli.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
+		err = executable_worker.Ffmpeg.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
 		if err != nil {
 			return []*models.MediaURL{}, errors.Wrapf(err, "failed to generate thumbnail for video (%s)", video.Title)
 		}
@@ -177,7 +177,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 			fmt.Printf("Video thumbnail found in database but not in cache, re-encoding photo to cache: %s\n", videoThumbnailURL.MediaName)
 			updatedURLs = append(updatedURLs, videoThumbnailURL)
 
-			err = executable_worker.FfmpegCli.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
+			err = executable_worker.Ffmpeg.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
 			if err != nil {
 				return []*models.MediaURL{}, errors.Wrapf(err, "failed to generate thumbnail for video (%s)", video.Title)
 			}

api/utils/environment_variables.go

@@ -35,9 +35,10 @@ const (
 
 // Feature related
 const (
-	EnvDisableFaceRecognition EnvironmentVariable = "PHOTOVIEW_DISABLE_FACE_RECOGNITION"
-	EnvDisableVideoEncoding   EnvironmentVariable = "PHOTOVIEW_DISABLE_VIDEO_ENCODING"
-	EnvDisableRawProcessing   EnvironmentVariable = "PHOTOVIEW_DISABLE_RAW_PROCESSING"
+	EnvDisableFaceRecognition    EnvironmentVariable = "PHOTOVIEW_DISABLE_FACE_RECOGNITION"
+	EnvDisableVideoEncoding      EnvironmentVariable = "PHOTOVIEW_DISABLE_VIDEO_ENCODING"
+	EnvDisableRawProcessing      EnvironmentVariable = "PHOTOVIEW_DISABLE_RAW_PROCESSING"
+	EnvVideoHardwareAcceleration EnvironmentVariable = "PHOTOVIEW_VIDEO_HARDWARE_ACCELERATION"
 )
 
 // GetName returns the name of the environment variable itself