Add caching of docker layers in CI (qdrant#1856)

* Add caching of docker layers in CI Build required docker images for CI in a workflow step using buildkit's gha cache type. This will populate the local layer cache from github actions' cache. Builds in subsequent CI steps will be nearly instant, because all layers can be reused. * add minor change to see if build time is any faster --------- Co-authored-by: Andrey Vasnetsov <andrey@vasnetsov.com>
jankieAI · May 17, 2023 · b11729e · b11729e
1 parent 313ad8e
commit b11729e
Show file tree

Hide file tree

Showing 9 changed files with 56 additions and 6 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -8,3 +8,4 @@ lib/collection/target
 lib/storage/target
 openapi/tests/
 *.tar
+.git
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -74,6 +74,18 @@ jobs:
           python-version: '3.10'
       - name: Install requests python lib
         run: pip install requests
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Docker build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          tags: qdrant_consensus
+          cache-from: |
+            type=gha,scope=${{ github.ref }}
+            type=gha,scope=${{ github.base_ref }}
+          cache-to: type=gha,mode=max,scope=${{ github.ref }}
+          load: true
       - name: Run integration tests - multiple peers - compose
         run: ./test_restart.sh
         working-directory: ./tests/consensus_tests
@@ -84,6 +96,18 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Docker build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          tags: qdrant_tls
+          cache-from: |
+            type=gha,scope=${{ github.ref }}
+            type=gha,scope=${{ github.base_ref }}
+          cache-to: type=gha,mode=max,scope=${{ github.ref }}
+          load: true
       - name: Run integration tests
         run: ./test_tls.sh
         working-directory: ./tests/tls
@@ -100,6 +124,18 @@ jobs:
           toolchain: stable
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Docker build
+        uses: docker/build-push-action@v4
+        with:
+          context: tools/schema2openapi
+          tags: schema2openapi
+          cache-from: |
+            type=gha,scope=${{ github.ref }}-schema2openapi
+            type=gha,scope=${{ github.base_ref }}-schema2openapi
+          cache-to: type=gha,mode=max,scope=${{ github.ref }}-schema2openapi
+          load: true
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
@@ -125,6 +161,18 @@ jobs:
           toolchain: stable
       - uses: Swatinem/rust-cache@v2
       - uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Docker build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          tags: qdrant-recovery
+          cache-from: |
+            type=gha,scope=${{ github.ref }}
+            type=gha,scope=${{ github.base_ref }}
+          cache-to: type=gha,mode=max,scope=${{ github.ref }}
+          load: true
       - name: Run low RAM test
         working-directory: ./tests/low-ram
         shell: bash

diff --git a/lib/segment/src/index/struct_payload_index.rs b/lib/segment/src/index/struct_payload_index.rs
@@ -42,6 +42,7 @@ pub const PAYLOAD_FIELD_INDEX_PATH: &str = "fields";
 pub struct StructPayloadIndex {
     /// Payload storage
     payload: Arc<AtomicRefCell<PayloadStorageEnum>>,
+    /// Used for `has_id` condition and estimating cardinality
     id_tracker: Arc<AtomicRefCell<IdTrackerSS>>,
     /// Indexes, associated with fields
     pub field_indexes: IndexesMap,

diff --git a/tests/consensus_tests/run_tests.sh b/tests/consensus_tests/run_tests.sh
@@ -11,7 +11,7 @@ function clear_after_tests()
 }
 
 # Prevent double building in docker-compose
-docker build ../../ --tag=qdrant_consensus
+docker buildx build --load ../../ --tag=qdrant_consensus
 docker-compose up -d --force-recreate
 trap clear_after_tests EXIT
 

diff --git a/tests/consensus_tests/test_restart.sh b/tests/consensus_tests/test_restart.sh
@@ -11,7 +11,7 @@ function clear_after_tests()
 }
 
 # Prevent double building in docker-compose
-docker build ../../ --tag=qdrant_consensus
+docker buildx build --load ../../ --tag=qdrant_consensus
 docker compose down --volumes
 docker compose up -d --force-recreate
 trap clear_after_tests EXIT

diff --git a/tests/low-ram/low-ram.sh b/tests/low-ram/low-ram.sh
@@ -6,7 +6,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 
 declare DOCKER_IMAGE_NAME=qdrant-recovery
 
-docker build ../../ --tag=$DOCKER_IMAGE_NAME
+docker buildx build --load ../../ --tag=$DOCKER_IMAGE_NAME
 
 if [[ ! -e storage ]]; then
     git lfs pull

diff --git a/tests/openapi_integration_test.sh b/tests/openapi_integration_test.sh
@@ -11,4 +11,4 @@ docker run --rm \
             --network=host \
             -e OPENAPI_FILE='openapi.json' \
             -v "${PWD}"/openapi/tests:/code \
-            "$(docker build -q ./openapi/tests)" sh -c /code/run_docker.sh
+            "$(docker buildx build --load -q ./openapi/tests)" sh -c /code/run_docker.sh
diff --git a/tests/tls/test_tls.sh b/tests/tls/test_tls.sh
@@ -35,7 +35,7 @@ function run_with_retry()
 }
 
 # Prevent double building in docker-compose
-docker build ../../ --tag=qdrant_tls
+docker buildx build --load ../../ --tag=qdrant_tls
 docker compose down --volumes
 docker compose up -d --force-recreate
 trap clear_after_tests EXIT

diff --git a/tools/generate_openapi_models.sh b/tools/generate_openapi_models.sh
@@ -25,7 +25,7 @@ cargo run --package qdrant --bin schema_generator > ./openapi/schemas/AllDefinit
 
 (
     cd tools/schema2openapi/
-    docker build . --tag schema2openapi
+    docker buildx build --load . --tag schema2openapi
 )
 
 docker run --rm \
-Original file line number
+Diff line change
@@ Expand Up / @@ -8,3 +8,4 @@ lib/collection/target @@
     lib/storage/target
     openapi/tests/
     *.tar
+    .git