Refactor Dockerfile (qdrant#1594)

* Refactor Dockerfile - fix cross-compilation - improve caching TODO: - check if `lld` is used/works for linkage (and enable if not used, or remove if doesn't work) * Remove `aarch64` linker config from `.cargo/config.toml` (seems to be unnecessary) * Expose `LINKER` argument and enable `lld` linker * Add `mold` linker support * Document Dockerfile * Add closing ` in the Dockerfile comments Co-authored-by: Tim Visée <tim+github@visee.me> --------- Co-authored-by: Tim Visée <tim+github@visee.me>
AKJUS · Jun 19, 2023 · 5e2d6ac · 5e2d6ac
1 parent 75f2dec
commit 5e2d6ac
Show file tree

Hide file tree

Showing 10 changed files with 86 additions and 69 deletions.
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -1,3 +0,0 @@
-[target.aarch64-unknown-linux-gnu]
-linker = "aarch64-linux-gnu-gcc"
-

diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -36,7 +36,7 @@ jobs:
 
         GITHUB_TAG="docker.pkg.github.com/qdrant/qdrant/qdrant:${{ github.ref_name }}"
 
-        docker buildx build --build-arg='RUST_BUILD_PROFILE=release' --platform='linux/amd64,linux/arm64' $TAGS --push .
+        docker buildx build --platform='linux/amd64,linux/arm64' $TAGS --push .
 
         docker pull $DOCKERHUB_TAG
 

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -87,7 +87,7 @@ jobs:
           cache-to: type=gha,mode=max,scope=${{ github.ref }}
           load: true
           build-args: |
-            RUST_BUILD_PROFILE=ci
+            PROFILE=ci
       - name: Run integration tests - multiple peers - compose
         run: ./test_restart.sh
         working-directory: ./tests/consensus_tests
@@ -111,7 +111,7 @@ jobs:
           cache-to: type=gha,mode=max,scope=${{ github.ref }}
           load: true
           build-args: |
-            RUST_BUILD_PROFILE=ci
+            PROFILE=ci
       - name: Run integration tests
         run: ./test_tls.sh
         working-directory: ./tests/tls
@@ -141,7 +141,7 @@ jobs:
           cache-to: type=gha,mode=max,scope=${{ github.ref }}-schema2openapi
           load: true
           build-args: |
-            RUST_BUILD_PROFILE=ci
+            PROFILE=ci
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
@@ -180,7 +180,7 @@ jobs:
           cache-to: type=gha,mode=max,scope=${{ github.ref }}
           load: true
           build-args: |
-            RUST_BUILD_PROFILE=ci
+            PROFILE=ci
       - name: Run low RAM test
         working-directory: ./tests/low-ram
         shell: bash

diff --git a/Dockerfile b/Dockerfile
@@ -1,70 +1,101 @@
-# Leveraging the pre-built Docker images with
-# cargo-chef and the Rust toolchain
+# Cross-compiling using Docker multi-platform builds/images and `xx`.
+#
+# https://docs.docker.com/build/building/multi-platform/
+# https://github.com/tonistiigi/xx
+FROM --platform=${BUILDPLATFORM:-linux/amd64} tonistiigi/xx AS xx
+
+# Utilizing Docker layer caching with `cargo-chef`.
+#
 # https://www.lpalmieri.com/posts/fast-rust-docker-builds/
-FROM --platform=${BUILDPLATFORM:-linux/amd64} lukemathwalker/cargo-chef:latest-rust-1.69.0 AS chef
-WORKDIR /qdrant
-
-ARG RUST_BUILD_PROFILE=release
-
-ARG MOLD_VERSION=1.11.0
+FROM --platform=${BUILDPLATFORM:-linux/amd64} lukemathwalker/cargo-chef:latest-rust-1.68.1 AS chef
 
-# # Choose MOLD arch based on TARGETARCH: amd64 -> x86_64, arm64 -> aarch64
 
 FROM chef AS planner
+WORKDIR /qdrant
 COPY . .
 RUN cargo chef prepare --recipe-path recipe.json
 
+
 FROM chef as builder
+WORKDIR /qdrant
 
-# based on https://github.com/docker/buildx/issues/510
-ARG TARGETARCH
-ENV TARGETARCH=${TARGETARCH:-amd64}
+COPY --from=xx / /
 
-WORKDIR /qdrant
+# Relative order of `ARG` and `RUN` commands in the Dockerfile matters.
+#
+# If you pass a different `ARG` to `docker build`, it would invalidate Docker layer cache
+# for the next steps. (E.g., the following steps may depend on a new `ARG` value, so Docker would
+# have to re-execute them instead of using a cached layer from a previous run.)
+#
+# Steps in this stage are ordered in a way that should maximize Docker layer cache utilization,
+# so, please, don't reorder them without prior consideration. 🥲
 
-COPY ./tools/mold_arch.sh ./mold_arch.sh
+RUN apt-get update \
+    && apt-get install -y clang lld cmake protobuf-compiler \
+    && rustup component add rustfmt
 
-RUN wget https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/mold-${MOLD_VERSION}-$(bash mold_arch.sh)-linux.tar.gz \
-    && tar -xf mold-${MOLD_VERSION}-$(bash mold_arch.sh)-linux.tar.gz  \
-    && mv mold-${MOLD_VERSION}-$(bash mold_arch.sh)-linux /qdrant/mold \
-    && chmod +x /qdrant/mold/bin/mold
+# `ARG`/`ENV` pair is a workaround for `docker build` backward-compatibility.
+#
+# https://github.com/docker/buildx/issues/510
+ARG BUILDPLATFORM
+ENV BUILDPLATFORM=${BUILDPLATFORM:-linux/amd64}
 
-COPY ./tools/target_arch.sh ./target_arch.sh
-COPY ./tools/cargo_cmd_arch.sh ./cargo_cmd_arch.sh
-RUN echo "Building for $TARGETARCH, arch: $(bash target_arch.sh)"
+ARG MOLD_VERSION=1.11.0
 
-COPY --from=planner /qdrant/recipe.json recipe.json
+RUN case "$BUILDPLATFORM" in \
+        */amd64 ) PLATFORM=x86_64 ;; \
+        */arm64 | */arm64/* ) PLATFORM=aarch64 ;; \
+        * ) echo "Unexpected BUILDPLATFORM '$BUILDPLATFORM'" >&2; exit 1 ;; \
+    esac; \
+    \
+    mkdir -p /opt/mold; \
+    cd /opt/mold; \
+    \
+    TARBALL="mold-$MOLD_VERSION-$PLATFORM-linux.tar.gz"; \
+    curl -sSLO "https://github.com/rui314/mold/releases/download/v$MOLD_VERSION/$TARBALL"; \
+    tar -xf "$TARBALL" --strip-components 1; \
+    rm "$TARBALL"
 
-RUN apt-get update \
-    && ( apt-get install -y gcc-multilib || echo "Warning: not installing gcc-multilib" ) \
-    && apt-get install -y clang cmake gcc-aarch64-linux-gnu g++-aarch64-linux-gnu protobuf-compiler \
-    && rustup component add rustfmt
+# `ARG`/`ENV` pair is a workaround for `docker build` backward-compatibility.
+#
+# https://github.com/docker/buildx/issues/510
+ARG TARGETPLATFORM
+ENV TARGETPLATFORM=${TARGETPLATFORM:-linux/amd64}
 
+RUN xx-apt-get install -y gcc g++ libc6-dev
 
-RUN rustup target add $(bash target_arch.sh)
+# Select Cargo profile (e.g., `release`, `dev` or `ci`)
+ARG PROFILE=release
 
-# Build dependencies - this is the caching Docker layer!
-RUN $(bash cargo_cmd_arch.sh) chef cook --profile=${RUST_BUILD_PROFILE} --target $(bash target_arch.sh) --recipe-path recipe.json
+# Enable crate features
+ARG FEATURES
 
-COPY . .
+# Pass custom `RUSTFLAGS` (e.g., `--cfg tokio_unstable` to enable Tokio tracing/`tokio-console`)
+ARG RUSTFLAGS
+
+# Select linker (e.g., `mold`, `lld` or an empty string for the default linker)
+ARG LINKER=mold
 
-# Build actual target here
-RUN $(bash cargo_cmd_arch.sh) build --profile=${RUST_BUILD_PROFILE} --target $(bash target_arch.sh) --bin qdrant
+COPY --from=planner /qdrant/recipe.json recipe.json
+RUN PATH="$PATH:/opt/mold/bin" \
+    RUSTFLAGS="${LINKER:+-C link-arg=-fuse-ld=}$LINKER $RUSTFLAGS" \
+    xx-cargo chef cook --profile $PROFILE ${FEATURES:+--features} $FEATURES --recipe-path recipe.json
+
+COPY . .
+RUN PATH="$PATH:/opt/mold/bin" \
+    RUSTFLAGS="${LINKER:+-C link-arg=-fuse-ld=}$LINKER $RUSTFLAGS" \
+    xx-cargo build --profile $PROFILE ${FEATURES:+--features} $FEATURES --bin qdrant \
+    && PROFILE_DIR=$(if [ "$PROFILE" = dev ]; then echo debug; else echo $PROFILE; fi) \
+    && mv target/$(xx-cargo --print-target-triple)/$PROFILE_DIR/qdrant /qdrant/qdrant
 
-RUN mv target/$(bash target_arch.sh)/${RUST_BUILD_PROFILE}/qdrant /qdrant/qdrant
 
-FROM debian:11-slim
-ARG APP=/qdrant
+FROM debian:11-slim AS qdrant
 
 RUN apt-get update \
     && apt-get install -y ca-certificates tzdata \
     && rm -rf /var/lib/apt/lists/*
 
-EXPOSE 6333
-EXPOSE 6334
-
-ENV TZ=Etc/UTC \
-    RUN_MODE=production
+ARG APP=/qdrant
 
 RUN mkdir -p ${APP}
 
@@ -74,4 +105,10 @@ COPY --from=builder /qdrant/tools/entrypoint.sh ${APP}/entrypoint.sh
 
 WORKDIR ${APP}
 
+ENV TZ=Etc/UTC \
+    RUN_MODE=production
+
+EXPOSE 6333
+EXPOSE 6334
+
 CMD ["./entrypoint.sh"]
diff --git a/tests/consensus_tests/run_tests.sh b/tests/consensus_tests/run_tests.sh
@@ -11,7 +11,7 @@ function clear_after_tests()
 }
 
 # Prevent double building in docker-compose
-docker buildx build --build-arg='RUST_BUILD_PROFILE=ci' --load ../../ --tag=qdrant_consensus
+docker buildx build --build-arg=PROFILE=ci --load ../../ --tag=qdrant_consensus
 docker-compose up -d --force-recreate
 trap clear_after_tests EXIT
 

diff --git a/tests/consensus_tests/test_restart.sh b/tests/consensus_tests/test_restart.sh
@@ -11,7 +11,7 @@ function clear_after_tests()
 }
 
 # Prevent double building in docker-compose
-docker buildx build --build-arg='RUST_BUILD_PROFILE=ci' --load ../../ --tag=qdrant_consensus
+docker buildx build --build-arg=PROFILE=ci --load ../../ --tag=qdrant_consensus
 docker compose down --volumes
 docker compose up -d --force-recreate
 trap clear_after_tests EXIT

diff --git a/tests/low-ram/low-ram.sh b/tests/low-ram/low-ram.sh
@@ -6,7 +6,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 
 declare DOCKER_IMAGE_NAME=qdrant-recovery
 
-docker buildx build --build-arg='RUST_BUILD_PROFILE=ci' --load ../../ --tag=$DOCKER_IMAGE_NAME
+docker buildx build --build-arg=PROFILE=ci --load ../../ --tag=$DOCKER_IMAGE_NAME
 
 if [[ ! -e storage ]]; then
     git lfs pull

diff --git a/tests/tls/test_tls.sh b/tests/tls/test_tls.sh
@@ -35,7 +35,7 @@ function run_with_retry()
 }
 
 # Prevent double building in docker-compose
-docker buildx build --build-arg='RUST_BUILD_PROFILE=ci' --load ../../ --tag=qdrant_tls
+docker buildx build --build-arg=PROFILE=ci --load ../../ --tag=qdrant_tls
 docker compose down --volumes
 docker compose up -d --force-recreate
 trap clear_after_tests EXIT

diff --git a/tools/mold_arch.sh b/tools/mold_arch.sh
diff --git a/tools/target_arch.sh b/tools/target_arch.sh
Original file line number	Diff line number	Diff line change
		@@ -1,3 +0,0 @@
		[target.aarch64-unknown-linux-gnu]
		linker = "aarch64-linux-gnu-gcc"