From ad13172a3248dac49f93715b1d603f6df9ba6be6 Mon Sep 17 00:00:00 2001 From: Googol Lee Date: Thu, 29 Aug 2024 10:30:06 +0200 Subject: [PATCH] Switch to Debian testing and `imagemagick`. (#1021) --- .github/workflows/build.yml | 38 +++++------ Dockerfile | 7 ++- README.md | 2 +- api/go.mod | 2 +- api/go.sum | 4 +- api/scanner/media_encoding/encode_photo.go | 18 +++--- .../executable_worker/executable_worker.go | 40 +++++------- .../executable_worker_test.go | 35 +++++++++++ .../executable_worker/magic_worker_test.go | 63 +++++++++++++++++++ .../executable_worker/testdata/bin/convert | 13 ++++ api/scanner/media_type/media_type.go | 2 +- api/utils/{Endpoints.go => endpoints.go} | 0 api/utils/{Throttle.go => throttle.go} | 0 scripts/apt/debian-testing.sources | 13 ++++ scripts/apt/default.pref | 11 ++++ scripts/install_build_dependencies.sh | 2 +- scripts/install_runtime_dependencies.sh | 16 +---- 17 files changed, 191 insertions(+), 75 deletions(-) create mode 100644 api/scanner/media_encoding/executable_worker/executable_worker_test.go create mode 100644 api/scanner/media_encoding/executable_worker/magic_worker_test.go create mode 100755 api/scanner/media_encoding/executable_worker/testdata/bin/convert rename api/utils/{Endpoints.go => endpoints.go} (100%) rename api/utils/{Throttle.go => throttle.go} (100%) create mode 100644 scripts/apt/debian-testing.sources create mode 100644 scripts/apt/default.pref diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 448fa89b..d7601a42 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,43 +12,39 @@ env: DOCKER_USERNAME: viktorstrate DOCKER_IMAGE: viktorstrate/photoview DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 jobs: build: name: Build Docker Image - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ secrets.CACHE_KEY }}-${{ matrix.target_platform }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-${{ secrets.CACHE_KEY }}-${{ matrix.target_platform }}- + - name: Checkout + uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 with: - platforms: ${{ matrix.target_platform }} + platforms: ${{ env.PLATFORMS }} + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Docker Login if: success() && github.event_name != 'pull_request' && github.repository == 'photoview/photoview' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} - name: Docker meta id: docker_meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v5 with: # list of Docker images to use as base name for tags images: ${{ env.DOCKER_IMAGE }} @@ -63,13 +59,19 @@ jobs: type=sha - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 + platforms: ${{ env.PLATFORMS }} + pull: true push: ${{ github.event_name != 'pull_request' && github.repository == 'photoview/photoview' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + sbom: true + provenance: mode=max + annotations: ${{ steps.docker_meta.outputs.annotations }} build-args: | VERSION=${{ github.ref_name }} COMMIT_SHA=${{ github.sha }} diff --git a/Dockerfile b/Dockerfile index 9e6e3fb5..36efb44d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,10 @@ ENV PATH="${GOPATH}/bin:${PATH}" ENV CGO_ENABLED=1 # Download dependencies -COPY scripts/*.sh /app/scripts/ +COPY scripts/apt/debian-testing.sources /etc/apt/sources.list.d/ +COPY scripts/set_compiler_env.sh /app/scripts/ +COPY scripts/install_build_dependencies.sh /app/scripts/ +COPY scripts/install_runtime_dependencies.sh /app/scripts/ RUN chmod +x /app/scripts/*.sh \ && /app/scripts/install_build_dependencies.sh \ && /app/scripts/install_runtime_dependencies.sh @@ -70,7 +73,7 @@ RUN source /app/scripts/set_compiler_env.sh \ && go build -v -o photoview . ### Build release image ### -FROM --platform=${BUILDPLATFORM:-linux/amd64} debian:bookworm-slim AS release +FROM --platform=${BUILDPLATFORM:-linux/amd64} debian:testing-slim AS release ARG TARGETPLATFORM # See for details: https://github.com/hadolint/hadolint/wiki/DL4006 diff --git a/README.md b/README.md index 7f7f9d3c..23b1cddf 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,7 @@ In macOS, install dependencies: ```sh $ brew update # Update the package list -$ brew install golang gcc libheif dlib jpeg # For API +$ brew install golang gcc pkg-config libheif dlib jpeg # For API $ brew install reflex sqlite3 # For API optional tools ``` diff --git a/api/go.mod b/api/go.mod index 3cb2bdd8..ea6ab66a 100644 --- a/api/go.mod +++ b/api/go.mod @@ -18,7 +18,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/stretchr/testify v1.9.0 - github.com/strukturag/libheif v1.15.1 + github.com/strukturag/libheif v1.17.6 github.com/vektah/gqlparser/v2 v2.5.16 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 github.com/xor-gate/goexif2 v1.1.0 diff --git a/api/go.sum b/api/go.sum index c4465e62..71848945 100644 --- a/api/go.sum +++ b/api/go.sum @@ -91,8 +91,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/strukturag/libheif v1.15.1 h1:PWMRTk+9HG0a9avvlV597iI0AdHk25zKVV33lSl6a+I= -github.com/strukturag/libheif v1.15.1/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks= +github.com/strukturag/libheif v1.17.6 h1:UFz4FI7kKLINWyL7bcNEBu4gZxK7rHRkwq49IOzHyvE= +github.com/strukturag/libheif v1.17.6/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks= github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8= diff --git a/api/scanner/media_encoding/encode_photo.go b/api/scanner/media_encoding/encode_photo.go index adf4d8ec..4e02bd99 100644 --- a/api/scanner/media_encoding/encode_photo.go +++ b/api/scanner/media_encoding/encode_photo.go @@ -22,12 +22,12 @@ import ( ) var thumbFilter = map[models.ThumbnailFilter]imaging.ResampleFilter{ - models.ThumbnailFilterNearestNeighbor: imaging.NearestNeighbor, - models.ThumbnailFilterBox: imaging.Box, - models.ThumbnailFilterLinear: imaging.Linear, - models.ThumbnailFilterMitchellNetravali: imaging.MitchellNetravali, - models.ThumbnailFilterCatmullRom: imaging.CatmullRom, - models.ThumbnailFilterLanczos: imaging.Lanczos, + models.ThumbnailFilterNearestNeighbor: imaging.NearestNeighbor, + models.ThumbnailFilterBox: imaging.Box, + models.ThumbnailFilterLinear: imaging.Linear, + models.ThumbnailFilterMitchellNetravali: imaging.MitchellNetravali, + models.ThumbnailFilterCatmullRom: imaging.CatmullRom, + models.ThumbnailFilterLanczos: imaging.Lanczos, } func EncodeThumbnail(db *gorm.DB, inputPath string, outputPath string) (*media_utils.PhotoDimensions, error) { @@ -108,10 +108,10 @@ func (img *EncodeMediaData) EncodeHighRes(outputPath string) error { return errors.New("could not convert photo as file format is not supported") } - // Use darktable if there is no counterpart JPEG file to use instead + // Use ImageMagick if there is no counterpart JPEG file to use instead if contentType.IsRaw() && img.CounterpartPath == nil { - if executable_worker.DarktableCli.IsInstalled() { - err := executable_worker.DarktableCli.EncodeJpeg(img.Media.Path, outputPath, 70) + if executable_worker.MagickCli.IsInstalled() { + err := executable_worker.MagickCli.EncodeJpeg(img.Media.Path, outputPath, 70) if err != nil { return err } diff --git a/api/scanner/media_encoding/executable_worker/executable_worker.go b/api/scanner/media_encoding/executable_worker/executable_worker.go index 2ada358f..c28e716b 100644 --- a/api/scanner/media_encoding/executable_worker/executable_worker.go +++ b/api/scanner/media_encoding/executable_worker/executable_worker.go @@ -2,9 +2,7 @@ package executable_worker import ( "fmt" - "io/ioutil" "log" - "os" "os/exec" "strings" @@ -14,18 +12,18 @@ import ( ) func InitializeExecutableWorkers() { - DarktableCli = newDarktableWorker() + MagickCli = newMagickWorker() FfmpegCli = newFfmpegWorker() } -var DarktableCli *DarktableWorker = nil +var MagickCli *MagickWorker = nil var FfmpegCli *FfmpegWorker = nil type ExecutableWorker interface { Path() string } -type DarktableWorker struct { +type MagickWorker struct { path string } @@ -33,25 +31,25 @@ type FfmpegWorker struct { path string } -func newDarktableWorker() *DarktableWorker { +func newMagickWorker() *MagickWorker { if utils.EnvDisableRawProcessing.GetBool() { - log.Printf("Executable worker disabled (%s=1): darktable\n", utils.EnvDisableRawProcessing.GetName()) + log.Printf("Executable worker disabled (%s=1): ImageMagick\n", utils.EnvDisableRawProcessing.GetName()) return nil } - path, err := exec.LookPath("darktable-cli") + path, err := exec.LookPath("convert") if err != nil { - log.Println("Executable worker not found: darktable") + log.Println("Executable worker not found: ImageMagick convert") } else { version, err := exec.Command(path, "--version").Output() if err != nil { - log.Printf("Error getting version of darktable: %s\n", err) + log.Printf("Error getting version of ImageMagick convert: %s\n", err) return nil } - log.Printf("Found executable worker: darktable (%s)\n", strings.Split(string(version), "\n")[0]) + log.Printf("Found executable worker: ImageMagick convert (%s)\n", strings.Split(string(version), "\n")[0]) - return &DarktableWorker{ + return &MagickWorker{ path: path, } } @@ -85,7 +83,7 @@ func newFfmpegWorker() *FfmpegWorker { return nil } -func (worker *DarktableWorker) IsInstalled() bool { +func (worker *MagickWorker) IsInstalled() bool { return worker != nil } @@ -93,27 +91,17 @@ func (worker *FfmpegWorker) IsInstalled() bool { return worker != nil } -func (worker *DarktableWorker) EncodeJpeg(inputPath string, outputPath string, jpegQuality int) error { - tmpDir, err := ioutil.TempDir("/tmp", "photoview-darktable") - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(tmpDir) - +func (worker *MagickWorker) EncodeJpeg(inputPath string, outputPath string, jpegQuality int) error { args := []string{ inputPath, + "-quality", fmt.Sprintf("%d", jpegQuality), outputPath, - "--core", - "--conf", - fmt.Sprintf("plugins/imageio/format/jpeg/quality=%d", jpegQuality), - "--configdir", - tmpDir, } cmd := exec.Command(worker.path, args...) if err := cmd.Run(); err != nil { - return errors.Wrapf(err, "encoding image using: %s %v", worker.path, args) + return fmt.Errorf("encoding image with \"%s %v\" error: %w", worker.path, args, err) } return nil diff --git a/api/scanner/media_encoding/executable_worker/executable_worker_test.go b/api/scanner/media_encoding/executable_worker/executable_worker_test.go new file mode 100644 index 00000000..8f66a5f6 --- /dev/null +++ b/api/scanner/media_encoding/executable_worker/executable_worker_test.go @@ -0,0 +1,35 @@ +package executable_worker_test + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/photoview/photoview/api/test_utils" +) + +func TestMain(m *testing.M) { + os.Exit(test_utils.IntegrationTestRun(m)) +} + +func setPathWithCurrent(paths ...string) func() { + _, file, _, ok := runtime.Caller(0) + if !ok { + return func() {} + } + + base := filepath.Dir(file) + + for i, path := range paths { + paths[i] = filepath.Join(base, path) + } + + originalPath := os.Getenv("PATH") + os.Setenv("PATH", strings.Join(paths, ":")) + + return func() { + os.Setenv("PATH", originalPath) + } +} diff --git a/api/scanner/media_encoding/executable_worker/magic_worker_test.go b/api/scanner/media_encoding/executable_worker/magic_worker_test.go new file mode 100644 index 00000000..dce9b8e0 --- /dev/null +++ b/api/scanner/media_encoding/executable_worker/magic_worker_test.go @@ -0,0 +1,63 @@ +package executable_worker_test + +import ( + "os" + "regexp" + "testing" + + "github.com/photoview/photoview/api/scanner/media_encoding/executable_worker" +) + +func TestMagickWorkerNotExist(t *testing.T) { + done := setPathWithCurrent() + defer done() + + executable_worker.InitializeExecutableWorkers() + + if executable_worker.MagickCli.IsInstalled() { + t.Error("MagickCli should not be installed, but is found:", executable_worker.MagickCli) + } +} + +func TestMagickWorkerIgnore(t *testing.T) { + done := setPathWithCurrent("./testdata/bin") + defer done() + + org := os.Getenv("PHOTOVIEW_DISABLE_RAW_PROCESSING") + os.Setenv("PHOTOVIEW_DISABLE_RAW_PROCESSING", "true") + defer os.Setenv("PHOTOVIEW_DISABLE_RAW_PROCESSING", org) + + executable_worker.InitializeExecutableWorkers() + + if executable_worker.MagickCli.IsInstalled() { + t.Error("MagickCli should not be installed, but is found:", executable_worker.MagickCli) + } +} + +func TestMagickWorker(t *testing.T) { + done := setPathWithCurrent("./testdata/bin") + defer done() + + executable_worker.InitializeExecutableWorkers() + + if !executable_worker.MagickCli.IsInstalled() { + t.Error("MagickCli should be installed") + } + + t.Run("Failed", func(t *testing.T) { + err := executable_worker.MagickCli.EncodeJpeg("input", "output", 0) + if err == nil { + t.Fatalf("MagickCli.EncodeJpeg(\"input\", \"output\", 0) = nil, should be an error.") + } + if got, want := err.Error(), "^encoding image with \".*?/testdata/bin/convert .*?\" error: .*$"; !regexp.MustCompile(want).MatchString(got) { + t.Errorf("MagickCli.EncodeJpeg(\"input\", \"output\", 0) = %q, should be as reg pattern %q", got, want) + } + }) + + t.Run("Succeeded", func(t *testing.T) { + err := executable_worker.MagickCli.EncodeJpeg("input", "output", 70) + if err != nil { + t.Fatalf("MagickCli.EncodeJpeg(\"input\", \"output\", 0) = %v, should be nil.", err) + } + }) +} diff --git a/api/scanner/media_encoding/executable_worker/testdata/bin/convert b/api/scanner/media_encoding/executable_worker/testdata/bin/convert new file mode 100755 index 00000000..7bcd96fd --- /dev/null +++ b/api/scanner/media_encoding/executable_worker/testdata/bin/convert @@ -0,0 +1,13 @@ +#!/bin/sh + +case "$1" in + "--version") + echo convert: version fake + ;; +esac + +echo $@ +if [ "$3" = "0" ] # quality parameter +then + exit -1 +fi diff --git a/api/scanner/media_type/media_type.go b/api/scanner/media_type/media_type.go index 4b72f27f..f1b42f1f 100644 --- a/api/scanner/media_type/media_type.go +++ b/api/scanner/media_type/media_type.go @@ -260,7 +260,7 @@ func (imgType *MediaType) IsSupported() bool { return true } - if executable_worker.DarktableCli.IsInstalled() && imgType.IsRaw() { + if executable_worker.MagickCli.IsInstalled() && imgType.IsRaw() { return true } diff --git a/api/utils/Endpoints.go b/api/utils/endpoints.go similarity index 100% rename from api/utils/Endpoints.go rename to api/utils/endpoints.go diff --git a/api/utils/Throttle.go b/api/utils/throttle.go similarity index 100% rename from api/utils/Throttle.go rename to api/utils/throttle.go diff --git a/scripts/apt/debian-testing.sources b/scripts/apt/debian-testing.sources new file mode 100644 index 00000000..34b25445 --- /dev/null +++ b/scripts/apt/debian-testing.sources @@ -0,0 +1,13 @@ +Types: deb +# http://snapshot.debian.org/archive/debian/20240722T000000Z +URIs: http://deb.debian.org/debian +Suites: testing testing-updates +Components: main +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg + +Types: deb +# http://snapshot.debian.org/archive/debian-security/20240722T000000Z +URIs: http://deb.debian.org/debian-security +Suites: testing-security +Components: main +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg diff --git a/scripts/apt/default.pref b/scripts/apt/default.pref new file mode 100644 index 00000000..270d1029 --- /dev/null +++ b/scripts/apt/default.pref @@ -0,0 +1,11 @@ +Package: * +Pin: release n=bookworm +Pin-Priority: 700 + +Package: * +Pin: release n=bookworm-updates +Pin-Priority: 700 + +Package: * +Pin: release n=bookworm-security +Pin-Priority: 700 diff --git a/scripts/install_build_dependencies.sh b/scripts/install_build_dependencies.sh index a1d4abb1..2e0ca591 100755 --- a/scripts/install_build_dependencies.sh +++ b/scripts/install_build_dependencies.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -euo pipefail if [ "$TARGETPLATFORM" == "linux/arm64" ]; then dpkg --add-architecture arm64 diff --git a/scripts/install_runtime_dependencies.sh b/scripts/install_runtime_dependencies.sh index 7698b008..e798b293 100755 --- a/scripts/install_runtime_dependencies.sh +++ b/scripts/install_runtime_dependencies.sh @@ -1,22 +1,10 @@ #!/bin/bash +set -euo pipefail apt-get update -apt-get install -y curl libdlib19.1 ffmpeg exiftool libheif1 - -# Install Darktable if building for a supported architecture -if [ "${TARGETPLATFORM}" = "linux/amd64" ] || [ "${TARGETPLATFORM}" = "linux/arm64" ]; then - echo 'deb [trusted=true] https://download.opensuse.org/repositories/graphics:/darktable/Debian_12/ /' > /etc/apt/sources.list.d/darktable.list - # Release key is invalid, just trust the repo - curl -fsSL https://download.opensuse.org/repositories/graphics:/darktable/Debian_12/Release.key \ - | gpg --dearmor -o /etc/apt/trusted.gpg.d/darktable.gpg - gpg --show-keys --with-fingerprint --dry-run /etc/apt/trusted.gpg.d/darktable.gpg - - apt-get update - apt-get install -y darktable -fi +apt-get install -y curl libdlib19.2 ffmpeg exiftool libheif1 imagemagick # Remove build dependencies and cleanup -apt-get purge -y ${BUILD_DEPENDS[@]} apt-get autoremove -y apt-get clean rm -rf /var/lib/apt/lists/*