From b11729e77b81f0e2fb1e10a8256232d8ad3077d3 Mon Sep 17 00:00:00 2001 From: Jesse Date: Mon, 8 May 2023 17:02:29 +0200 Subject: [PATCH] Add caching of docker layers in CI (#1856) * Add caching of docker layers in CI Build required docker images for CI in a workflow step using buildkit's gha cache type. This will populate the local layer cache from github actions' cache. Builds in subsequent CI steps will be nearly instant, because all layers can be reused. * add minor change to see if build time is any faster --------- Co-authored-by: Andrey Vasnetsov --- .dockerignore | 1 + .github/workflows/integration-tests.yml | 48 +++++++++++++++++++ lib/segment/src/index/struct_payload_index.rs | 1 + tests/consensus_tests/run_tests.sh | 2 +- tests/consensus_tests/test_restart.sh | 2 +- tests/low-ram/low-ram.sh | 2 +- tests/openapi_integration_test.sh | 2 +- tests/tls/test_tls.sh | 2 +- tools/generate_openapi_models.sh | 2 +- 9 files changed, 56 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index cff4e1a2746..61f7381a058 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,4 @@ lib/collection/target lib/storage/target openapi/tests/ *.tar +.git diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6883dfd8201..0f022c78097 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -74,6 +74,18 @@ jobs: python-version: '3.10' - name: Install requests python lib run: pip install requests + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Docker build + uses: docker/build-push-action@v4 + with: + context: . + tags: qdrant_consensus + cache-from: | + type=gha,scope=${{ github.ref }} + type=gha,scope=${{ github.base_ref }} + cache-to: type=gha,mode=max,scope=${{ github.ref }} + load: true - name: Run integration tests - multiple peers - compose run: ./test_restart.sh working-directory: ./tests/consensus_tests @@ -84,6 +96,18 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Docker build + uses: docker/build-push-action@v4 + with: + context: . + tags: qdrant_tls + cache-from: | + type=gha,scope=${{ github.ref }} + type=gha,scope=${{ github.base_ref }} + cache-to: type=gha,mode=max,scope=${{ github.ref }} + load: true - name: Run integration tests run: ./test_tls.sh working-directory: ./tests/tls @@ -100,6 +124,18 @@ jobs: toolchain: stable - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Docker build + uses: docker/build-push-action@v4 + with: + context: tools/schema2openapi + tags: schema2openapi + cache-from: | + type=gha,scope=${{ github.ref }}-schema2openapi + type=gha,scope=${{ github.base_ref }}-schema2openapi + cache-to: type=gha,mode=max,scope=${{ github.ref }}-schema2openapi + load: true - name: Install Protoc uses: arduino/setup-protoc@v1 with: @@ -125,6 +161,18 @@ jobs: toolchain: stable - uses: Swatinem/rust-cache@v2 - uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Docker build + uses: docker/build-push-action@v4 + with: + context: . + tags: qdrant-recovery + cache-from: | + type=gha,scope=${{ github.ref }} + type=gha,scope=${{ github.base_ref }} + cache-to: type=gha,mode=max,scope=${{ github.ref }} + load: true - name: Run low RAM test working-directory: ./tests/low-ram shell: bash diff --git a/lib/segment/src/index/struct_payload_index.rs b/lib/segment/src/index/struct_payload_index.rs index a202cfeb66b..014ae6062e0 100644 --- a/lib/segment/src/index/struct_payload_index.rs +++ b/lib/segment/src/index/struct_payload_index.rs @@ -42,6 +42,7 @@ pub const PAYLOAD_FIELD_INDEX_PATH: &str = "fields"; pub struct StructPayloadIndex { /// Payload storage payload: Arc>, + /// Used for `has_id` condition and estimating cardinality id_tracker: Arc>, /// Indexes, associated with fields pub field_indexes: IndexesMap, diff --git a/tests/consensus_tests/run_tests.sh b/tests/consensus_tests/run_tests.sh index 6bf639982f2..571a55cc5d7 100755 --- a/tests/consensus_tests/run_tests.sh +++ b/tests/consensus_tests/run_tests.sh @@ -11,7 +11,7 @@ function clear_after_tests() } # Prevent double building in docker-compose -docker build ../../ --tag=qdrant_consensus +docker buildx build --load ../../ --tag=qdrant_consensus docker-compose up -d --force-recreate trap clear_after_tests EXIT diff --git a/tests/consensus_tests/test_restart.sh b/tests/consensus_tests/test_restart.sh index 9a8a6c6c901..ba2d466f903 100755 --- a/tests/consensus_tests/test_restart.sh +++ b/tests/consensus_tests/test_restart.sh @@ -11,7 +11,7 @@ function clear_after_tests() } # Prevent double building in docker-compose -docker build ../../ --tag=qdrant_consensus +docker buildx build --load ../../ --tag=qdrant_consensus docker compose down --volumes docker compose up -d --force-recreate trap clear_after_tests EXIT diff --git a/tests/low-ram/low-ram.sh b/tests/low-ram/low-ram.sh index f4c1af29e2e..a8bfd008b46 100755 --- a/tests/low-ram/low-ram.sh +++ b/tests/low-ram/low-ram.sh @@ -6,7 +6,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" declare DOCKER_IMAGE_NAME=qdrant-recovery -docker build ../../ --tag=$DOCKER_IMAGE_NAME +docker buildx build --load ../../ --tag=$DOCKER_IMAGE_NAME if [[ ! -e storage ]]; then git lfs pull diff --git a/tests/openapi_integration_test.sh b/tests/openapi_integration_test.sh index 45b04714bd1..0981cecf755 100755 --- a/tests/openapi_integration_test.sh +++ b/tests/openapi_integration_test.sh @@ -11,4 +11,4 @@ docker run --rm \ --network=host \ -e OPENAPI_FILE='openapi.json' \ -v "${PWD}"/openapi/tests:/code \ - "$(docker build -q ./openapi/tests)" sh -c /code/run_docker.sh + "$(docker buildx build --load -q ./openapi/tests)" sh -c /code/run_docker.sh diff --git a/tests/tls/test_tls.sh b/tests/tls/test_tls.sh index de183d5d99a..207c3010ec5 100755 --- a/tests/tls/test_tls.sh +++ b/tests/tls/test_tls.sh @@ -35,7 +35,7 @@ function run_with_retry() } # Prevent double building in docker-compose -docker build ../../ --tag=qdrant_tls +docker buildx build --load ../../ --tag=qdrant_tls docker compose down --volumes docker compose up -d --force-recreate trap clear_after_tests EXIT diff --git a/tools/generate_openapi_models.sh b/tools/generate_openapi_models.sh index 520cbf5ef83..8dd351ec6e9 100755 --- a/tools/generate_openapi_models.sh +++ b/tools/generate_openapi_models.sh @@ -25,7 +25,7 @@ cargo run --package qdrant --bin schema_generator > ./openapi/schemas/AllDefinit ( cd tools/schema2openapi/ - docker build . --tag schema2openapi + docker buildx build --load . --tag schema2openapi ) docker run --rm \