From a6cbfc76f3d4acbf9dd3e5767a34ba319fc4cd56 Mon Sep 17 00:00:00 2001 From: Googol Lee Date: Fri, 6 Sep 2024 13:32:22 +0200 Subject: [PATCH] Back to `bookworm`. (#1040) --- Dockerfile | 35 ++++---- README.md | 3 +- api/go.mod | 2 +- api/go.sum | 4 +- api/scanner/media_encoding/encode_photo.go | 18 ++-- .../executable_worker/executable_worker.go | 43 ++++++---- .../executable_worker/magick_worker_test.go | 63 -------------- .../executable_worker/testdata/bin/magick | 13 --- api/scanner/media_type/media_type.go | 2 +- dev-compose.yaml | 5 +- scripts/install_build_dependencies.sh | 45 ++++++++-- scripts/install_runtime_dependencies.sh | 27 ++++-- scripts/set_compiler_env.sh | 84 +++++++++++++++++++ 13 files changed, 204 insertions(+), 140 deletions(-) delete mode 100644 api/scanner/media_encoding/executable_worker/magick_worker_test.go delete mode 100755 api/scanner/media_encoding/executable_worker/testdata/bin/magick create mode 100755 scripts/set_compiler_env.sh diff --git a/Dockerfile b/Dockerfile index 15080f9f..9e6e3fb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ ### Build UI ### -FROM --platform=${BUILDPLATFORM:-linux/amd64} node:18-alpine AS ui +FROM --platform=${BUILDPLATFORM:-linux/amd64} node:18 AS ui # See for details: https://github.com/hadolint/hadolint/wiki/DL4006 -SHELL ["/bin/sh", "-euo", "pipefail", "-c"] +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] ARG REACT_APP_API_ENDPOINT ENV REACT_APP_API_ENDPOINT=${REACT_APP_API_ENDPOINT} @@ -36,11 +36,11 @@ RUN if [ "${BUILD_DATE}" = "undefined" ]; then \ npm run build -- --base=$UI_PUBLIC_URL ### Build API ### -FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-alpine AS api +FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-bookworm AS api ARG TARGETPLATFORM # See for details: https://github.com/hadolint/hadolint/wiki/DL4006 -SHELL ["/bin/sh", "-euo", "pipefail", "-c"] +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] WORKDIR /app/api @@ -49,41 +49,40 @@ ENV PATH="${GOPATH}/bin:${PATH}" ENV CGO_ENABLED=1 # Download dependencies -COPY scripts/install_build_dependencies.sh /app/scripts/ -COPY scripts/install_runtime_dependencies.sh /app/scripts/ +COPY scripts/*.sh /app/scripts/ RUN chmod +x /app/scripts/*.sh \ && /app/scripts/install_build_dependencies.sh \ - && mv $(go env GOPATH)/bin/reflex /usr/bin/ \ && /app/scripts/install_runtime_dependencies.sh COPY api/go.mod api/go.sum /app/api/ -RUN go env \ +RUN source /app/scripts/set_compiler_env.sh \ + && go env \ && go mod download \ # Patch go-face && sed -i 's/-march=native//g' ${GOPATH}/pkg/mod/github.com/!kagami/go-face*/face.go \ # Build dependencies that use CGO - && go install github.com/mattn/go-sqlite3 github.com/Kagami/go-face + && go install \ + github.com/mattn/go-sqlite3 \ + github.com/Kagami/go-face COPY api /app/api -RUN go build -v -o photoview . +RUN source /app/scripts/set_compiler_env.sh \ + && go build -v -o photoview . ### Build release image ### -FROM --platform=${BUILDPLATFORM:-linux/amd64} alpine:latest AS release +FROM --platform=${BUILDPLATFORM:-linux/amd64} debian:bookworm-slim AS release ARG TARGETPLATFORM # See for details: https://github.com/hadolint/hadolint/wiki/DL4006 -SHELL ["/bin/sh", "-euo", "pipefail", "-c"] +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] COPY scripts/install_runtime_dependencies.sh /app/scripts/ RUN chmod +x /app/scripts/install_runtime_dependencies.sh \ # Create a user to run Photoview server - && addgroup -g 9999 photoview \ - && adduser -u 9999 -G photoview -D photoview \ + && groupadd -g 999 photoview \ + && useradd -r -u 999 -g photoview -m photoview \ # Required dependencies - && /app/scripts/install_runtime_dependencies.sh \ - # Remove build dependencies and cleanup - && apk cache clean \ - && rm -rf /var/cache/apk/* + && /app/scripts/install_runtime_dependencies.sh WORKDIR /home/photoview diff --git a/README.md b/README.md index 2f82a717..d928a6be 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,6 @@ We can't keep verifying below commands on each environment. People may need to s ### Install dependencies - - API - Required packages: - `golang` >= 1.22 @@ -304,6 +303,8 @@ You can install `node` with other package manager if you like. Then run the following commands: ```bash +# Optional: Set the compiler environment in Debian/Ubuntu +$ source ./scripts/set_compiler_env.sh # Set the compiler environment with `homebrew` $ export CPLUS_INCLUDE_PATH="$(brew --prefix)/opt/jpeg/include:$(brew --prefix)/opt/dlib/include" $ export LD_LIBRARY_PATH="$(brew --prefix)/opt/jpeg/lib:$(brew --prefix)/opt/dlib/lib" diff --git a/api/go.mod b/api/go.mod index ea6ab66a..3cb2bdd8 100644 --- a/api/go.mod +++ b/api/go.mod @@ -18,7 +18,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/stretchr/testify v1.9.0 - github.com/strukturag/libheif v1.17.6 + github.com/strukturag/libheif v1.15.1 github.com/vektah/gqlparser/v2 v2.5.16 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 github.com/xor-gate/goexif2 v1.1.0 diff --git a/api/go.sum b/api/go.sum index 71848945..c4465e62 100644 --- a/api/go.sum +++ b/api/go.sum @@ -91,8 +91,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/strukturag/libheif v1.17.6 h1:UFz4FI7kKLINWyL7bcNEBu4gZxK7rHRkwq49IOzHyvE= -github.com/strukturag/libheif v1.17.6/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks= +github.com/strukturag/libheif v1.15.1 h1:PWMRTk+9HG0a9avvlV597iI0AdHk25zKVV33lSl6a+I= +github.com/strukturag/libheif v1.15.1/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks= github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8= diff --git a/api/scanner/media_encoding/encode_photo.go b/api/scanner/media_encoding/encode_photo.go index 4e02bd99..adf4d8ec 100644 --- a/api/scanner/media_encoding/encode_photo.go +++ b/api/scanner/media_encoding/encode_photo.go @@ -22,12 +22,12 @@ import ( ) var thumbFilter = map[models.ThumbnailFilter]imaging.ResampleFilter{ - models.ThumbnailFilterNearestNeighbor: imaging.NearestNeighbor, - models.ThumbnailFilterBox: imaging.Box, - models.ThumbnailFilterLinear: imaging.Linear, - models.ThumbnailFilterMitchellNetravali: imaging.MitchellNetravali, - models.ThumbnailFilterCatmullRom: imaging.CatmullRom, - models.ThumbnailFilterLanczos: imaging.Lanczos, + models.ThumbnailFilterNearestNeighbor: imaging.NearestNeighbor, + models.ThumbnailFilterBox: imaging.Box, + models.ThumbnailFilterLinear: imaging.Linear, + models.ThumbnailFilterMitchellNetravali: imaging.MitchellNetravali, + models.ThumbnailFilterCatmullRom: imaging.CatmullRom, + models.ThumbnailFilterLanczos: imaging.Lanczos, } func EncodeThumbnail(db *gorm.DB, inputPath string, outputPath string) (*media_utils.PhotoDimensions, error) { @@ -108,10 +108,10 @@ func (img *EncodeMediaData) EncodeHighRes(outputPath string) error { return errors.New("could not convert photo as file format is not supported") } - // Use ImageMagick if there is no counterpart JPEG file to use instead + // Use darktable if there is no counterpart JPEG file to use instead if contentType.IsRaw() && img.CounterpartPath == nil { - if executable_worker.MagickCli.IsInstalled() { - err := executable_worker.MagickCli.EncodeJpeg(img.Media.Path, outputPath, 70) + if executable_worker.DarktableCli.IsInstalled() { + err := executable_worker.DarktableCli.EncodeJpeg(img.Media.Path, outputPath, 70) if err != nil { return err } diff --git a/api/scanner/media_encoding/executable_worker/executable_worker.go b/api/scanner/media_encoding/executable_worker/executable_worker.go index 5e9df818..2ada358f 100644 --- a/api/scanner/media_encoding/executable_worker/executable_worker.go +++ b/api/scanner/media_encoding/executable_worker/executable_worker.go @@ -2,7 +2,9 @@ package executable_worker import ( "fmt" + "io/ioutil" "log" + "os" "os/exec" "strings" @@ -12,18 +14,18 @@ import ( ) func InitializeExecutableWorkers() { - MagickCli = newMagickWorker() + DarktableCli = newDarktableWorker() FfmpegCli = newFfmpegWorker() } -var MagickCli *MagickWorker = nil +var DarktableCli *DarktableWorker = nil var FfmpegCli *FfmpegWorker = nil type ExecutableWorker interface { Path() string } -type MagickWorker struct { +type DarktableWorker struct { path string } @@ -31,25 +33,25 @@ type FfmpegWorker struct { path string } -func newMagickWorker() *MagickWorker { +func newDarktableWorker() *DarktableWorker { if utils.EnvDisableRawProcessing.GetBool() { - log.Printf("Executable worker disabled (%s=1): ImageMagick\n", utils.EnvDisableRawProcessing.GetName()) + log.Printf("Executable worker disabled (%s=1): darktable\n", utils.EnvDisableRawProcessing.GetName()) return nil } - path, err := exec.LookPath("magick") + path, err := exec.LookPath("darktable-cli") if err != nil { - log.Println("Executable worker not found: magick") + log.Println("Executable worker not found: darktable") } else { - version, err := exec.Command(path, "-version").Output() + version, err := exec.Command(path, "--version").Output() if err != nil { - log.Printf("Error getting version of magick: %s\n", err) + log.Printf("Error getting version of darktable: %s\n", err) return nil } - log.Printf("Found executable worker: magick (%s)\n", strings.Split(string(version), "\n")[0]) + log.Printf("Found executable worker: darktable (%s)\n", strings.Split(string(version), "\n")[0]) - return &MagickWorker{ + return &DarktableWorker{ path: path, } } @@ -83,7 +85,7 @@ func newFfmpegWorker() *FfmpegWorker { return nil } -func (worker *MagickWorker) IsInstalled() bool { +func (worker *DarktableWorker) IsInstalled() bool { return worker != nil } @@ -91,18 +93,27 @@ func (worker *FfmpegWorker) IsInstalled() bool { return worker != nil } -func (worker *MagickWorker) EncodeJpeg(inputPath string, outputPath string, jpegQuality int) error { +func (worker *DarktableWorker) EncodeJpeg(inputPath string, outputPath string, jpegQuality int) error { + tmpDir, err := ioutil.TempDir("/tmp", "photoview-darktable") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(tmpDir) + args := []string{ - "convert", inputPath, - "-quality", fmt.Sprintf("%d", jpegQuality), outputPath, + "--core", + "--conf", + fmt.Sprintf("plugins/imageio/format/jpeg/quality=%d", jpegQuality), + "--configdir", + tmpDir, } cmd := exec.Command(worker.path, args...) if err := cmd.Run(); err != nil { - return fmt.Errorf("encoding image with \"%s %v\" error: %w", worker.path, args, err) + return errors.Wrapf(err, "encoding image using: %s %v", worker.path, args) } return nil diff --git a/api/scanner/media_encoding/executable_worker/magick_worker_test.go b/api/scanner/media_encoding/executable_worker/magick_worker_test.go deleted file mode 100644 index b47c8262..00000000 --- a/api/scanner/media_encoding/executable_worker/magick_worker_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package executable_worker_test - -import ( - "os" - "regexp" - "testing" - - "github.com/photoview/photoview/api/scanner/media_encoding/executable_worker" -) - -func TestMagickWorkerNotExist(t *testing.T) { - done := setPathWithCurrent() - defer done() - - executable_worker.InitializeExecutableWorkers() - - if executable_worker.MagickCli.IsInstalled() { - t.Error("MagickCli should not be installed, but is found:", executable_worker.MagickCli) - } -} - -func TestMagickWorkerIgnore(t *testing.T) { - done := setPathWithCurrent("./testdata/bin") - defer done() - - org := os.Getenv("PHOTOVIEW_DISABLE_RAW_PROCESSING") - os.Setenv("PHOTOVIEW_DISABLE_RAW_PROCESSING", "true") - defer os.Setenv("PHOTOVIEW_DISABLE_RAW_PROCESSING", org) - - executable_worker.InitializeExecutableWorkers() - - if executable_worker.MagickCli.IsInstalled() { - t.Error("MagickCli should not be installed, but is found:", executable_worker.MagickCli) - } -} - -func TestMagickWorker(t *testing.T) { - done := setPathWithCurrent("./testdata/bin") - defer done() - - executable_worker.InitializeExecutableWorkers() - - if !executable_worker.MagickCli.IsInstalled() { - t.Error("MagickCli should be installed") - } - - t.Run("Failed", func(t *testing.T) { - err := executable_worker.MagickCli.EncodeJpeg("input", "output", 0) - if err == nil { - t.Fatalf("MagickCli.EncodeJpeg(\"input\", \"output\", 0) = nil, should be an error.") - } - if got, want := err.Error(), "^encoding image with \".*?/testdata/bin/magick .*?\" error: .*$"; !regexp.MustCompile(want).MatchString(got) { - t.Errorf("MagickCli.EncodeJpeg(\"input\", \"output\", 0) = %q, should be as reg pattern %q", got, want) - } - }) - - t.Run("Succeeded", func(t *testing.T) { - err := executable_worker.MagickCli.EncodeJpeg("input", "output", 70) - if err != nil { - t.Fatalf("MagickCli.EncodeJpeg(\"input\", \"output\", 0) = %v, should be nil.", err) - } - }) -} diff --git a/api/scanner/media_encoding/executable_worker/testdata/bin/magick b/api/scanner/media_encoding/executable_worker/testdata/bin/magick deleted file mode 100755 index 8152a4be..00000000 --- a/api/scanner/media_encoding/executable_worker/testdata/bin/magick +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -case "$1" in - "-version") - echo convert: version fake - ;; -esac - -echo $@ -if [ "$4" = "0" ] # quality parameter -then - exit -1 -fi diff --git a/api/scanner/media_type/media_type.go b/api/scanner/media_type/media_type.go index f1b42f1f..4b72f27f 100644 --- a/api/scanner/media_type/media_type.go +++ b/api/scanner/media_type/media_type.go @@ -260,7 +260,7 @@ func (imgType *MediaType) IsSupported() bool { return true } - if executable_worker.MagickCli.IsInstalled() && imgType.IsRaw() { + if executable_worker.DarktableCli.IsInstalled() && imgType.IsRaw() { return true } diff --git a/dev-compose.yaml b/dev-compose.yaml index d3678a3e..20150cab 100644 --- a/dev-compose.yaml +++ b/dev-compose.yaml @@ -13,7 +13,7 @@ services: ports: - 1234:1234 command: - - /bin/sh + - /bin/bash - -c - | npm ci @@ -36,9 +36,10 @@ services: ports: - 4001:4001 command: - - /bin/sh + - /bin/bash - -c - | + source /app/scripts/set_compiler_env.sh reflex -g '*.go' -s -- go run . mariadb: diff --git a/scripts/install_build_dependencies.sh b/scripts/install_build_dependencies.sh index 81d54aa7..a1d4abb1 100755 --- a/scripts/install_build_dependencies.sh +++ b/scripts/install_build_dependencies.sh @@ -1,11 +1,42 @@ -#!/bin/sh -set -euo pipefail +#!/bin/bash +set -e -apk update +if [ "$TARGETPLATFORM" == "linux/arm64" ]; then + dpkg --add-architecture arm64 + DEBIAN_ARCH='arm64' +elif [ "$TARGETPLATFORM" == "linux/arm/v6" ] || [ "$TARGETPLATFORM" == "linux/arm/v7" ]; then + dpkg --add-architecture armhf + DEBIAN_ARCH='armhf' +else + dpkg --add-architecture amd64 + DEBIAN_ARCH='amd64' +fi -apk add go g++ blas-dev cblas lapack-dev jpeg-dev libheif-dev -apk add dlib-dev --repository=https://dl-cdn.alpinelinux.org/alpine/edge/testing +apt-get update + +# Install G++/GCC cross compilers +if [ "$DEBIAN_ARCH" == "arm64" ]; then + apt-get install -y \ + g++-aarch64-linux-gnu \ + libc6-dev-arm64-cross +elif [ "$DEBIAN_ARCH" == "armhf" ]; then + apt-get install -y \ + g++-arm-linux-gnueabihf \ + libc6-dev-armhf-cross +else + apt-get install -y \ + g++-x86-64-linux-gnu \ + libc6-dev-amd64-cross +fi + +# Install go-face dependencies and libheif for HEIF media decoding +apt-get install -y \ + libdlib-dev:${DEBIAN_ARCH} \ + libblas-dev:${DEBIAN_ARCH} \ + libatlas-base-dev:${DEBIAN_ARCH} \ + liblapack-dev:${DEBIAN_ARCH} \ + libjpeg-dev:${DEBIAN_ARCH} \ + libheif-dev:${DEBIAN_ARCH} # Install tools for development -apk add sqlite -go install github.com/cespare/reflex@latest +apt-get install -y reflex sqlite3 diff --git a/scripts/install_runtime_dependencies.sh b/scripts/install_runtime_dependencies.sh index ff426429..7698b008 100755 --- a/scripts/install_runtime_dependencies.sh +++ b/scripts/install_runtime_dependencies.sh @@ -1,9 +1,22 @@ -#!/bin/sh -set -euo pipefail +#!/bin/bash -apk update +apt-get update +apt-get install -y curl libdlib19.1 ffmpeg exiftool libheif1 -apk add curl libheif lapack cblas exiftool -apk add ffmpeg ffmpeg-libs ffmpeg-libavcodec ffmpeg-libavformat -apk add imagemagick imagemagick-libs imagemagick-heic imagemagick-jpeg imagemagick-raw imagemagick-tiff imagemagick-webp imagemagick-svg imagemagick-jxl -apk add dlib --repository=https://dl-cdn.alpinelinux.org/alpine/edge/testing +# Install Darktable if building for a supported architecture +if [ "${TARGETPLATFORM}" = "linux/amd64" ] || [ "${TARGETPLATFORM}" = "linux/arm64" ]; then + echo 'deb [trusted=true] https://download.opensuse.org/repositories/graphics:/darktable/Debian_12/ /' > /etc/apt/sources.list.d/darktable.list + # Release key is invalid, just trust the repo + curl -fsSL https://download.opensuse.org/repositories/graphics:/darktable/Debian_12/Release.key \ + | gpg --dearmor -o /etc/apt/trusted.gpg.d/darktable.gpg + gpg --show-keys --with-fingerprint --dry-run /etc/apt/trusted.gpg.d/darktable.gpg + + apt-get update + apt-get install -y darktable +fi + +# Remove build dependencies and cleanup +apt-get purge -y ${BUILD_DEPENDS[@]} +apt-get autoremove -y +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/scripts/set_compiler_env.sh b/scripts/set_compiler_env.sh new file mode 100755 index 00000000..a7744e6b --- /dev/null +++ b/scripts/set_compiler_env.sh @@ -0,0 +1,84 @@ +#!/bin/sh + +# Script to configure environment variables for Go compiler +# to allow cross compilation + +: ${TARGETPLATFORM=} +: ${TARGETOS=} +: ${TARGETARCH=} +: ${TARGETVARIANT=} + +CGO_ENABLED="$(go env CGO_ENABLED)" +GOARCH="$(go env GOARCH)" +GOOS="$(go env GOOS)" +GOARM="$(go env GOARM)" +GOBIN="$(go env GOBIN)" + +set -eu + +if [ ! -z "$TARGETPLATFORM" ]; then + TARGETOS="$(echo $TARGETPLATFORM | cut -d"/" -f1)" + TARGETARCH="$(echo $TARGETPLATFORM | cut -d"/" -f2)" + TARGETVARIANT="$(echo $TARGETPLATFORM | cut -d"/" -f3)" +fi + +if [ ! -z "$TARGETOS" ]; then + export GOOS="$TARGETOS" +fi + +if [ ! -z "$TARGETARCH" ]; then + export GOARCH="$TARGETARCH" +fi + +if [ "$TARGETARCH" = "arm" ]; then + if [ ! -z "$TARGETVARIANT" ]; then + case "$TARGETVARIANT" in + "v5") + export GOARM="5" + ;; + "v6") + export GOARM="6" + ;; + *) + export GOARM="7" + ;; + esac + else + export GOARM="7" + fi +fi + +if [ "$CGO_ENABLED" = "1" ]; then + case "$GOARCH" in + "amd64") + export COMPILER_ARCH="x86_64-linux-gnu" + ;; + "ppc64le") + export COMPILER_ARCH="powerpc64le-linux-gnu" + ;; + "s390x") + export COMPILER_ARCH="s390x-linux-gnu" + ;; + "arm64") + export COMPILER_ARCH="aarch64-linux-gnu" + ;; + "arm") + case "$GOARM" in + "5") + export COMPILER_ARCH="arm-linux-gnueabi" + ;; + *) + export COMPILER_ARCH="arm-linux-gnueabihf" + ;; + esac + ;; + esac +fi + +export CC="${COMPILER_ARCH}-gcc" +export CXX="${COMPILER_ARCH}-g++" +export PKG_CONFIG_PATH="/usr/lib/${COMPILER_ARCH}/pkgconfig/" + +if [ -z "$GOBIN" ] && [ -n "$GOPATH" ] && [ -n "$GOARCH" ] && [ -n "$GOOS" ]; then + export PATH=${GOPATH}/bin/${GOOS}_${GOARCH}:${PATH} +fi