Use hardware acceleration when converting videos. (#1056)

Dockerfile

@@ -56,7 +56,17 @@ COPY scripts/set_compiler_env.sh /app/scripts/
 RUN chmod +x /app/scripts/*.sh \
   && source /app/scripts/set_compiler_env.sh
 
+COPY scripts/install_*.sh /app/scripts/
+# Split values in `/env`
+# hadolint ignore=SC2046
+RUN chmod +x /app/scripts/*.sh \
+  && export $(cat /env) \
+  && /app/scripts/install_build_dependencies.sh \
+  && /app/scripts/install_runtime_dependencies.sh
+
 COPY --from=viktorstrate/dependencies /artifacts.tar.gz /dependencies/
+# Split values in `/env`
+# hadolint ignore=SC2046
 RUN export $(cat /env) \
   && cd /dependencies/ \
   && tar xfv artifacts.tar.gz \
@@ -67,13 +77,9 @@ RUN export $(cat /env) \
   && ldconfig \
   && apt-get install -y ./deb/jellyfin-ffmpeg.deb
 
-COPY scripts/install_*.sh /app/scripts/
-RUN chmod +x /app/scripts/*.sh \
-  && export $(cat /env) \
-  && /app/scripts/install_build_dependencies.sh \
-  && /app/scripts/install_runtime_dependencies.sh
-
 COPY api/go.mod api/go.sum /app/api/
+# Split values in `/env`
+# hadolint ignore=SC2046
 RUN export $(cat /env) \
   && go env \
   && go mod download \
@@ -85,6 +91,8 @@ RUN export $(cat /env) \
     github.com/Kagami/go-face
 
 COPY api /app/api
+# Split values in `/env`
+# hadolint ignore=SC2046
 RUN export $(cat /env) \
   && go env \
   && go build -v -o photoview .

README.md

@@ -159,6 +159,14 @@ Possible ways of securing a self-hosted service might be (but not limited to):
 
 Setting up and configuring of all these protections depends on and requires a lot of info about your local network and self-hosted services. Based on this info, the configuration flow and resulting services architecture might differ a lot between cases. That is why in the scope of this project, we can only provide you with this high-level list of possible ways of webservice protection. You'll need to investigate them, find the best combination and configuration for your case, and take responsibility to configure everything in the correct and consistent way. We cannot provide you support for such highly secured setups, as a lot of things might work differently because of security limitations.
 
+### Hardware Acceleration
+
+It is possible to run the FFmpeg with a codec supproting the hardware acceleration, by defining `PHOTOVIEW_VIDEO_HARDWARE_ACCELERATION`. The value should be one of `qsv`, `vaapi`, `nvenc`.
+
+We only verified the hardware acceleration with `qsv` on an Intel chip. To let it work, it must map `/dev/dri` devices and set a ENV `PHOTOVIEW_VIDEO_HARDWARE_ACCELERATION=qsv`. See [docker-compose.example.yml](./docker-compose example/docker-compose.example.yml).
+
+If you verify other hardware accelerations working well, let us know.
+
 ## Contributing
 
 🎉 First off, thanks for your interest in contribution! 🎉

api/example.env

@@ -32,3 +32,8 @@ PHOTOVIEW_SERVE_UI=0
 # Set to 1 to set server in development mode, this enables graphql playground
 # Remove this if running in production
 PHOTOVIEW_DEVELOPMENT_MODE=1
+
+# Set the hardware acceleration when encoding videos.
+# Support `qsv`, `vaapi`, `nvenc`.
+# Only `qsv` is verified with `/dev/dri//dev/dri` devices.
+# PHOTOVIEW_VIDEO_HARDWARE_ACCELERATION=

api/scanner/media_encoding/executable_worker/executable_worker.go

@@ -6,96 +6,38 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/photoview/photoview/api/utils"
-	"github.com/pkg/errors"
 	"gopkg.in/vansante/go-ffprobe.v2"
 )
 
 func InitializeExecutableWorkers() {
 	Magick = newMagickCli()
-	FfmpegCli = newFfmpegWorker()
+	Ffmpeg = newFfmpegCli()
+
+	if err := SetFfprobePath(); err != nil {
+		log.Println("ffprobe init fail:", err)
+	}
 }
 
 var Magick *MagickCli = nil
-var FfmpegCli *FfmpegWorker = nil
+var Ffmpeg *FfmpegCli = nil
 
 type ExecutableWorker interface {
 	Path() string
 }
 
-type FfmpegWorker struct {
-	path string
-}
-
-func newFfmpegWorker() *FfmpegWorker {
-	if utils.EnvDisableVideoEncoding.GetBool() {
-		log.Printf("Executable worker disabled (%s=1): ffmpeg\n", utils.EnvDisableVideoEncoding.GetName())
-		return nil
+func SetFfprobePath() error {
+	path, err := exec.LookPath("ffprobe")
+	if err != nil {
+		return fmt.Errorf("Executable ffprobe not found: %w", err)
 	}
 
-	path, err := exec.LookPath("ffmpeg")
-	if err != nil {
-		log.Println("Executable worker not found: ffmpeg")
-	} else {
 	version, err := exec.Command(path, "-version").Output()
 	if err != nil {
-			log.Printf("Error getting version of ffmpeg: %s\n", err)
-			return nil
+		return fmt.Errorf("Executable ffprobe(%q) not executable: %w", path, err)
 	}
 
-		log.Printf("Found executable worker: ffmpeg (%s)\n", strings.Split(string(version), "\n")[0])
-
-		return &FfmpegWorker{
-			path: path,
-		}
-	}
-
-	return nil
-}
-
-func (worker *FfmpegWorker) IsInstalled() bool {
-	return worker != nil
-}
-
-func (worker *FfmpegWorker) EncodeMp4(inputPath string, outputPath string) error {
-	args := []string{
-		"-i",
-		inputPath,
-		"-vcodec", "h264",
-		"-acodec", "aac",
-		"-vf", "scale='min(1080,iw)':'min(1080,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2",
-		"-movflags", "+faststart+use_metadata_tags",
-		outputPath,
-	}
-
-	cmd := exec.Command(worker.path, args...)
-
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "encoding video using: %s", worker.path)
-	}
-
-	return nil
-}
-
-func (worker *FfmpegWorker) EncodeVideoThumbnail(inputPath string, outputPath string, probeData *ffprobe.ProbeData) error {
-
-	thumbnailOffsetSeconds := fmt.Sprintf("%d", int(probeData.Format.DurationSeconds*0.25))
-
-	args := []string{
-		"-ss", thumbnailOffsetSeconds, // grab frame at time offset
-		"-i",
-		inputPath,
-		"-vframes", "1", // output one frame
-		"-an", // disable audio
-		"-vf", "scale='min(1024,iw)':'min(1024,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2",
-		outputPath,
-	}
-
-	cmd := exec.Command(worker.path, args...)
-
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "encoding video using: %s", worker.path)
-	}
+	log.Println("Found ffprobe:", path, "version:", strings.Split(string(version), "\n")[0])
+	ffprobe.SetFFProbeBinPath(path)
 
 	return nil
 }

api/scanner/media_encoding/executable_worker/executable_worker_test.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/photoview/photoview/api/scanner/media_encoding/executable_worker"
 	"github.com/photoview/photoview/api/test_utils"
 )
 
@@ -43,3 +44,35 @@ func setEnv(key, value string) func() {
 		os.Setenv(key, org)
 	}
 }
+
+func TestInitFfprobePath(t *testing.T) {
+	t.Run("PathFail", func(t *testing.T) {
+		err := executable_worker.SetFfprobePath()
+		if err == nil {
+			t.Fatalf("InitFfprobePath() returns nil, want an error")
+		}
+	})
+
+	t.Run("VersionFail", func(t *testing.T) {
+		donePath := setPathWithCurrent("./testdata/bin")
+		defer donePath()
+
+		doneEnv := setEnv("FAIL_WITH", "expect failure")
+		defer doneEnv()
+
+		err := executable_worker.SetFfprobePath()
+		if err == nil {
+			t.Fatalf("InitFfprobePath() returns nil, want an error")
+		}
+	})
+
+	t.Run("Succeed", func(t *testing.T) {
+		donePath := setPathWithCurrent("./testdata/bin")
+		defer donePath()
+
+		err := executable_worker.SetFfprobePath()
+		if err != nil {
+			t.Fatalf("InitFfprobePath() returns %v, want nil", err)
+		}
+	})
+}

api/scanner/media_encoding/executable_worker/ffmpeg_cli.go

@@ -0,0 +1,108 @@
+package executable_worker
+
+import (
+	"fmt"
+	"log"
+	"os/exec"
+	"strings"
+
+	"github.com/photoview/photoview/api/utils"
+	"gopkg.in/vansante/go-ffprobe.v2"
+)
+
+const defaultCodec = "h264"
+
+var hwAccToCodec = map[string]string{
+	"qsv":   defaultCodec + "_qsv",
+	"vaapi": defaultCodec + "_vaapi",
+	"nvenc": defaultCodec + "_nvenc",
+}
+
+type FfmpegCli struct {
+	path       string
+	videoCodec string
+}
+
+func newFfmpegCli() *FfmpegCli {
+	if utils.EnvDisableVideoEncoding.GetBool() {
+		log.Printf("Executable worker disabled (%s=%q): ffmpeg\n", utils.EnvDisableVideoEncoding.GetName(), utils.EnvDisableVideoEncoding.GetValue())
+		return nil
+	}
+
+	path, err := exec.LookPath("ffmpeg")
+	if err != nil {
+		log.Println("Executable worker not found: ffmpeg")
+		return nil
+	}
+
+	version, err := exec.Command(path, "-version").Output()
+	if err != nil {
+		log.Printf("Error getting version of ffmpeg: %s\n", err)
+		return nil
+	}
+
+	hwAcc := utils.EnvVideoHardwareAcceleration.GetValue()
+	codec, ok := hwAccToCodec[hwAcc]
+	if !ok {
+		if strings.HasPrefix(hwAcc, "_") {
+			// A secret way to set the codec directly.
+			codec = hwAcc[1:]
+		} else {
+			codec = defaultCodec
+		}
+	}
+
+	log.Printf("Found executable worker: ffmpeg (%s) with codec %q\n", strings.Split(string(version), "\n")[0], codec)
+
+	return &FfmpegCli{
+		path:       path,
+		videoCodec: codec,
+	}
+}
+
+func (worker *FfmpegCli) IsInstalled() bool {
+	return worker != nil
+}
+
+func (worker *FfmpegCli) EncodeMp4(inputPath string, outputPath string) error {
+	args := []string{
+		"-i",
+		inputPath,
+		"-vcodec", worker.videoCodec,
+		"-acodec", "aac",
+		"-vf", "scale='min(1080,iw)':'min(1080,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2",
+		"-movflags", "+faststart+use_metadata_tags",
+		outputPath,
+	}
+
+	cmd := exec.Command(worker.path, args...)
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("encoding video with %q %v error: %w", worker.path, args, err)
+	}
+
+	return nil
+}
+
+func (worker *FfmpegCli) EncodeVideoThumbnail(inputPath string, outputPath string, probeData *ffprobe.ProbeData) error {
+
+	thumbnailOffsetSeconds := fmt.Sprintf("%.f", probeData.Format.DurationSeconds*0.25)
+
+	args := []string{
+		"-ss", thumbnailOffsetSeconds, // grab frame at time offset
+		"-i",
+		inputPath,
+		"-vframes", "1", // output one frame
+		"-an", // disable audio
+		"-vf", "scale='min(1024,iw)':'min(1024,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2",
+		outputPath,
+	}
+
+	cmd := exec.Command(worker.path, args...)
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("encoding video thumbnail with %q %v error: %w", worker.path, args, err)
+	}
+
+	return nil
+}

api/scanner/media_encoding/executable_worker/ffmpeg_cli_test.go

@@ -0,0 +1,133 @@
+package executable_worker_test
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/photoview/photoview/api/scanner/media_encoding/executable_worker"
+	"github.com/photoview/photoview/api/utils"
+	"gopkg.in/vansante/go-ffprobe.v2"
+)
+
+func TestFfmpegNotExist(t *testing.T) {
+	done := setPathWithCurrent()
+	defer done()
+
+	executable_worker.InitializeExecutableWorkers()
+
+	if executable_worker.Ffmpeg.IsInstalled() {
+		t.Error("Ffmpeg should not be installed, but is found:", executable_worker.Ffmpeg)
+	}
+}
+
+func TestFfmpegIgnore(t *testing.T) {
+	donePath := setPathWithCurrent("./testdata/bin")
+	defer donePath()
+
+	doneEnv := setEnv("PHOTOVIEW_DISABLE_VIDEO_ENCODING", "true")
+	defer doneEnv()
+
+	executable_worker.InitializeExecutableWorkers()
+
+	if executable_worker.Ffmpeg.IsInstalled() {
+		t.Error("Ffmpeg should be ignored (as it is disabled), but is initialized:", executable_worker.Ffmpeg)
+	}
+}
+
+func TestFfmpeg(t *testing.T) {
+	done := setPathWithCurrent("./testdata/bin")
+	defer done()
+
+	executable_worker.InitializeExecutableWorkers()
+
+	if !executable_worker.Ffmpeg.IsInstalled() {
+		t.Error("Ffmpeg should be installed")
+	}
+
+	t.Run("EncodeMp4Failed", func(t *testing.T) {
+		doneEnv := setEnv("FAIL_WITH", "expect failure")
+		defer doneEnv()
+
+		err := executable_worker.Ffmpeg.EncodeMp4("input", "output")
+		if err == nil {
+			t.Fatalf("Ffmpeg.EncodeMp4(...) = nil, should be an error.")
+		}
+		if got, want := err.Error(), `^encoding video with ".*/testdata/bin/ffmpeg" \[-i input -vcodec h264 .* output\] error: .*$`; !regexp.MustCompile(want).MatchString(got) {
+			t.Errorf("Ffmpeg.EncodeMp4(...) = %q, should be as reg pattern %q", got, want)
+		}
+	})
+
+	t.Run("EncodeMp4Succeeded", func(t *testing.T) {
+		err := executable_worker.Ffmpeg.EncodeMp4("input", "output")
+		if err != nil {
+			t.Fatalf("Ffmpeg.EncodeMp4(...) = %v, should be nil.", err)
+		}
+	})
+
+	probeData := &ffprobe.ProbeData{
+		Format: &ffprobe.Format{
+			DurationSeconds: 10,
+		},
+	}
+	t.Run("EncodeVideoThumbnailMp4Failed", func(t *testing.T) {
+		doneEnv := setEnv("FAIL_WITH", "expect failure")
+		defer doneEnv()
+
+		err := executable_worker.Ffmpeg.EncodeVideoThumbnail("input", "output", probeData)
+		if err == nil {
+			t.Fatalf("Ffmpeg.EncodeVideoThumbnail(...) = nil, should be an error.")
+		}
+		if got, want := err.Error(), `^encoding video thumbnail with ".*/testdata/bin/ffmpeg" \[-ss 2 -i input .* output\] error: .*$`; !regexp.MustCompile(want).MatchString(got) {
+			t.Errorf("Ffmpeg.EncodeVideoThumbnail(...) = %q, should be as reg pattern %q", got, want)
+		}
+	})
+
+	t.Run("EncodeVideoThumbnailSucceeded", func(t *testing.T) {
+		err := executable_worker.Ffmpeg.EncodeVideoThumbnail("input", "output", probeData)
+		if err != nil {
+			t.Fatalf("Ffmpeg.EncodeVideoThumbnail(...) = %v, should be nil.", err)
+		}
+	})
+}
+
+func TestFfmpegWithHWAcc(t *testing.T) {
+	doneCodec := setEnv(utils.EnvVideoHardwareAcceleration.GetName(), "qsv")
+	defer doneCodec()
+
+	donePath := setPathWithCurrent("./testdata/bin")
+	defer donePath()
+
+	executable_worker.InitializeExecutableWorkers()
+
+	doneEnv := setEnv("FAIL_WITH", "expect failure")
+	defer doneEnv()
+
+	err := executable_worker.Ffmpeg.EncodeMp4("input", "output")
+	if err == nil {
+		t.Fatalf("Ffmpeg.EncodeMp4(...) = nil, should be an error.")
+	}
+	if got, want := err.Error(), `^encoding video with ".*/testdata/bin/ffmpeg" \[-i input -vcodec h264_qsv .* output\] error: .*$`; !regexp.MustCompile(want).MatchString(got) {
+		t.Errorf("Ffmpeg.EncodeMp4(...) = %q, should be as reg pattern %q", got, want)
+	}
+}
+
+func TestFfmpegWithCustomCOdec(t *testing.T) {
+	doneCodec := setEnv(utils.EnvVideoHardwareAcceleration.GetName(), "_custom")
+	defer doneCodec()
+
+	donePath := setPathWithCurrent("./testdata/bin")
+	defer donePath()
+
+	executable_worker.InitializeExecutableWorkers()
+
+	doneEnv := setEnv("FAIL_WITH", "expect failure")
+	defer doneEnv()
+
+	err := executable_worker.Ffmpeg.EncodeMp4("input", "output")
+	if err == nil {
+		t.Fatalf("Ffmpeg.EncodeMp4(...) = nil, should be an error.")
+	}
+	if got, want := err.Error(), `^encoding video with ".*/testdata/bin/ffmpeg" \[-i input -vcodec custom .* output\] error: .*$`; !regexp.MustCompile(want).MatchString(got) {
+		t.Errorf("Ffmpeg.EncodeMp4(...) = %q, should be as reg pattern %q", got, want)
+	}
+}

api/scanner/media_encoding/executable_worker/testdata/bin/ffmpeg

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+
+: ${FAIL_WITH=""}
+
+case "$1" in
+  "--version")
+      echo ffmpeg: version fake
+      ;;
+esac
+
+if [ "${FAIL_WITH}" != "" ]
+then
+  echo ${FAIL_WITH}
+  exit -1
+fi
+
+echo $@

api/scanner/media_encoding/executable_worker/testdata/bin/ffprobe

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+
+: ${FAIL_WITH=""}
+
+case "$1" in
+  "--version")
+      echo ffprobe: version fake
+      ;;
+esac
+
+if [ "${FAIL_WITH}" != "" ]
+then
+  echo ${FAIL_WITH}
+  exit -1
+fi
+
+echo $@

api/scanner/media_type/media_type.go

@@ -264,7 +264,7 @@ func (imgType *MediaType) IsSupported() bool {
 		return true
 	}
 
-	if executable_worker.FfmpegCli.IsInstalled() && imgType.IsVideo() {
+	if executable_worker.Ffmpeg.IsInstalled() && imgType.IsVideo() {
 		return true
 	}
 

api/scanner/scanner_tasks/processing_tasks/process_video_task.go

@@ -94,7 +94,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 
 		webVideoPath := path.Join(mediaCachePath, webVideoName)
 
-		err = executable_worker.FfmpegCli.EncodeMp4(video.Path, webVideoPath)
+		err = executable_worker.Ffmpeg.EncodeMp4(video.Path, webVideoPath)
 		if err != nil {
 			return []*models.MediaURL{}, errors.Wrapf(err, "could not encode mp4 video (%s)", video.Path)
 		}
@@ -139,7 +139,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 
 		thumbImagePath := path.Join(mediaCachePath, videoThumbName)
 
-		err = executable_worker.FfmpegCli.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
+		err = executable_worker.Ffmpeg.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
 		if err != nil {
 			return []*models.MediaURL{}, errors.Wrapf(err, "failed to generate thumbnail for video (%s)", video.Title)
 		}
@@ -177,7 +177,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 			fmt.Printf("Video thumbnail found in database but not in cache, re-encoding photo to cache: %s\n", videoThumbnailURL.MediaName)
 			updatedURLs = append(updatedURLs, videoThumbnailURL)
 
-			err = executable_worker.FfmpegCli.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
+			err = executable_worker.Ffmpeg.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
 			if err != nil {
 				return []*models.MediaURL{}, errors.Wrapf(err, "failed to generate thumbnail for video (%s)", video.Title)
 			}

api/utils/environment_variables.go

@@ -38,6 +38,7 @@ const (
 	EnvDisableFaceRecognition    EnvironmentVariable = "PHOTOVIEW_DISABLE_FACE_RECOGNITION"
 	EnvDisableVideoEncoding      EnvironmentVariable = "PHOTOVIEW_DISABLE_VIDEO_ENCODING"
 	EnvDisableRawProcessing      EnvironmentVariable = "PHOTOVIEW_DISABLE_RAW_PROCESSING"
+	EnvVideoHardwareAcceleration EnvironmentVariable = "PHOTOVIEW_VIDEO_HARDWARE_ACCELERATION"
 )
 
 // GetName returns the name of the environment variable itself

dev-compose.yaml

@@ -1,7 +1,7 @@
 name: photoview
 
 services:
-  dev-ui:
+  ui:
     image: photoview/ui
     build:
       context: .
@@ -21,7 +21,7 @@ services:
         npm ci
         npm run mon
 
-  dev-api:
+  api:
     image: photoview/api
     build:
       context: .

docker-compose example/docker-compose.example.yml

@@ -42,6 +42,10 @@ services:
       ## A token can be generated for free here https://account.mapbox.com/access-tokens/
       ## It's a good idea to limit the scope of the token to your own domain, to prevent others from using it.
       MAPBOX_TOKEN: ${MAPBOX_TOKEN}
+      ## If you want to use it, set the correct value in the .env file.
+      ## Support `qsv`, `vaapi`, `nvenc`.
+      ## Only `qsv` is verified with `/dev/dri` devices (see below `devices`).
+      PHOTOVIEW_VIDEO_HARDWARE_ACCELERATION: ${PHOTOVIEW_VIDEO_HARDWARE_ACCELERATION}
     ## Share hardware devices with FFmpeg (optional):
     # devices:
       ## Uncomment next devices mappings if they are available in your host system

docker-compose example/example.env

@@ -31,6 +31,13 @@ PHOTOVIEW_DATABASE_DRIVER=mysql
 # MAPBOX_TOKEN=yourToken
 ##-----------------------------------##
 
+##----------Video variables----------##
+## Set the hardware acceleration when encoding videos.
+## Support `qsv`, `vaapi`, `nvenc`.
+## Only `qsv` is verified with `/dev/dri` devices.
+# PHOTOVIEW_VIDEO_HARDWARE_ACCELERATION=
+##-----------------------------------##
+
 ##--------MariaDB variables----------##
 ## Comment out these variables if PHOTOVIEW_DATABASE_DRIVER is `sqlite` or `postgres`
 ## Use password generator to generate secret values and replace these defaults