add performance regression testing to ci

duckdb · Sep 3, 2024 · 97e8a9b · 97e8a9b
1 parent f4cfb12
commit 97e8a9b
Show file tree

Hide file tree

Showing 104 changed files with 768 additions and 558 deletions.
diff --git a/.github/regression/tpcds_sf1_local.csv b/.github/regression/tpcds_sf1_local.csv
@@ -0,0 +1,100 @@
+benchmark/tpcds/sf1/local/delta/q01.benchmark
+benchmark/tpcds/sf1/local/delta/q02.benchmark
+benchmark/tpcds/sf1/local/delta/q03.benchmark
+benchmark/tpcds/sf1/local/delta/q04.benchmark
+benchmark/tpcds/sf1/local/delta/q05.benchmark
+benchmark/tpcds/sf1/local/delta/q06.benchmark
+benchmark/tpcds/sf1/local/delta/q07.benchmark
+benchmark/tpcds/sf1/local/delta/q08.benchmark
+benchmark/tpcds/sf1/local/delta/q09.benchmark
+benchmark/tpcds/sf1/local/delta/q10.benchmark
+benchmark/tpcds/sf1/local/delta/q11.benchmark
+benchmark/tpcds/sf1/local/delta/q12.benchmark
+benchmark/tpcds/sf1/local/delta/q13.benchmark
+benchmark/tpcds/sf1/local/delta/q14.benchmark
+benchmark/tpcds/sf1/local/delta/q15.benchmark
+benchmark/tpcds/sf1/local/delta/q16.benchmark
+benchmark/tpcds/sf1/local/delta/q17.benchmark
+benchmark/tpcds/sf1/local/delta/q18.benchmark
+benchmark/tpcds/sf1/local/delta/q19.benchmark
+benchmark/tpcds/sf1/local/delta/q20.benchmark
+benchmark/tpcds/sf1/local/delta/q21.benchmark
+benchmark/tpcds/sf1/local/delta/q22.benchmark
+benchmark/tpcds/sf1/local/delta/q23.benchmark
+benchmark/tpcds/sf1/local/delta/q24.benchmark
+benchmark/tpcds/sf1/local/delta/q25.benchmark
+benchmark/tpcds/sf1/local/delta/q26.benchmark
+benchmark/tpcds/sf1/local/delta/q27.benchmark
+benchmark/tpcds/sf1/local/delta/q28.benchmark
+
+benchmark/tpcds/sf1/local/delta/q29.benchmark
+benchmark/tpcds/sf1/local/delta/q30.benchmark
+benchmark/tpcds/sf1/local/delta/q31.benchmark
+benchmark/tpcds/sf1/local/delta/q32.benchmark
+benchmark/tpcds/sf1/local/delta/q33.benchmark
+benchmark/tpcds/sf1/local/delta/q34.benchmark
+benchmark/tpcds/sf1/local/delta/q35.benchmark
+benchmark/tpcds/sf1/local/delta/q36.benchmark
+benchmark/tpcds/sf1/local/delta/q37.benchmark
+benchmark/tpcds/sf1/local/delta/q38.benchmark
+benchmark/tpcds/sf1/local/delta/q39.benchmark
+benchmark/tpcds/sf1/local/delta/q40.benchmark
+benchmark/tpcds/sf1/local/delta/q41.benchmark
+benchmark/tpcds/sf1/local/delta/q42.benchmark
+benchmark/tpcds/sf1/local/delta/q43.benchmark
+benchmark/tpcds/sf1/local/delta/q44.benchmark
+benchmark/tpcds/sf1/local/delta/q45.benchmark
+benchmark/tpcds/sf1/local/delta/q46.benchmark
+benchmark/tpcds/sf1/local/delta/q47.benchmark
+benchmark/tpcds/sf1/local/delta/q48.benchmark
+benchmark/tpcds/sf1/local/delta/q49.benchmark
+benchmark/tpcds/sf1/local/delta/q50.benchmark
+benchmark/tpcds/sf1/local/delta/q51.benchmark
+benchmark/tpcds/sf1/local/delta/q52.benchmark
+benchmark/tpcds/sf1/local/delta/q53.benchmark
+benchmark/tpcds/sf1/local/delta/q54.benchmark
+benchmark/tpcds/sf1/local/delta/q55.benchmark
+benchmark/tpcds/sf1/local/delta/q56.benchmark
+benchmark/tpcds/sf1/local/delta/q57.benchmark
+benchmark/tpcds/sf1/local/delta/q58.benchmark
+benchmark/tpcds/sf1/local/delta/q59.benchmark
+benchmark/tpcds/sf1/local/delta/q60.benchmark
+benchmark/tpcds/sf1/local/delta/q61.benchmark
+benchmark/tpcds/sf1/local/delta/q62.benchmark
+benchmark/tpcds/sf1/local/delta/q63.benchmark
+benchmark/tpcds/sf1/local/delta/q64.benchmark
+benchmark/tpcds/sf1/local/delta/q65.benchmark
+benchmark/tpcds/sf1/local/delta/q66.benchmark
+benchmark/tpcds/sf1/local/delta/q67.benchmark
+benchmark/tpcds/sf1/local/delta/q68.benchmark
+benchmark/tpcds/sf1/local/delta/q69.benchmark
+benchmark/tpcds/sf1/local/delta/q70.benchmark
+benchmark/tpcds/sf1/local/delta/q71.benchmark
+benchmark/tpcds/sf1/local/delta/q72.benchmark
+benchmark/tpcds/sf1/local/delta/q73.benchmark
+benchmark/tpcds/sf1/local/delta/q74.benchmark
+benchmark/tpcds/sf1/local/delta/q75.benchmark
+benchmark/tpcds/sf1/local/delta/q76.benchmark
+benchmark/tpcds/sf1/local/delta/q77.benchmark
+benchmark/tpcds/sf1/local/delta/q78.benchmark
+benchmark/tpcds/sf1/local/delta/q79.benchmark
+benchmark/tpcds/sf1/local/delta/q80.benchmark
+benchmark/tpcds/sf1/local/delta/q81.benchmark
+benchmark/tpcds/sf1/local/delta/q82.benchmark
+benchmark/tpcds/sf1/local/delta/q83.benchmark
+benchmark/tpcds/sf1/local/delta/q84.benchmark
+benchmark/tpcds/sf1/local/delta/q85.benchmark
+benchmark/tpcds/sf1/local/delta/q86.benchmark
+benchmark/tpcds/sf1/local/delta/q87.benchmark
+benchmark/tpcds/sf1/local/delta/q88.benchmark
+benchmark/tpcds/sf1/local/delta/q89.benchmark
+benchmark/tpcds/sf1/local/delta/q90.benchmark
+benchmark/tpcds/sf1/local/delta/q91.benchmark
+benchmark/tpcds/sf1/local/delta/q92.benchmark
+benchmark/tpcds/sf1/local/delta/q93.benchmark
+benchmark/tpcds/sf1/local/delta/q94.benchmark
+benchmark/tpcds/sf1/local/delta/q95.benchmark
+benchmark/tpcds/sf1/local/delta/q96.benchmark
+benchmark/tpcds/sf1/local/delta/q97.benchmark
+benchmark/tpcds/sf1/local/delta/q98.benchmark
+benchmark/tpcds/sf1/local/delta/q99.benchmark
diff --git a/.github/regression/tpch_sf1_local.csv b/.github/regression/tpch_sf1_local.csv
@@ -0,0 +1,22 @@
+benchmark/tpch/sf1/delta/q01.benchmark
+benchmark/tpch/sf1/delta/q02.benchmark
+benchmark/tpch/sf1/delta/q03.benchmark
+benchmark/tpch/sf1/delta/q04.benchmark
+benchmark/tpch/sf1/delta/q05.benchmark
+benchmark/tpch/sf1/delta/q06.benchmark
+benchmark/tpch/sf1/delta/q07.benchmark
+benchmark/tpch/sf1/delta/q08.benchmark
+benchmark/tpch/sf1/delta/q09.benchmark
+benchmark/tpch/sf1/delta/q10.benchmark
+benchmark/tpch/sf1/delta/q11.benchmark
+benchmark/tpch/sf1/delta/q12.benchmark
+benchmark/tpch/sf1/delta/q13.benchmark
+benchmark/tpch/sf1/delta/q14.benchmark
+benchmark/tpch/sf1/delta/q15.benchmark
+benchmark/tpch/sf1/delta/q16.benchmark
+benchmark/tpch/sf1/delta/q17.benchmark
+benchmark/tpch/sf1/delta/q18.benchmark
+benchmark/tpch/sf1/delta/q19.benchmark
+benchmark/tpch/sf1/delta/q20.benchmark
+benchmark/tpch/sf1/delta/q21.benchmark
+benchmark/tpch/sf1/delta/q22.benchmark
diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml
@@ -7,6 +7,9 @@ defaults:
   run:
     shell: bash
 
+env:
+  BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }}
+
 jobs:
   azurite-tests-linux:
     name: Azurite (local azure test server) tests (Linux)
@@ -213,4 +216,75 @@ jobs:
       - name: Test
         shell: bash
         run: |
-          GENERATED_DATA_AVAILABLE=1 make test
+          GENERATED_DATA_AVAILABLE=1 make test
+
+  regression-test-benchmark-runner:
+    name: Performance Regression Tests
+    runs-on: ubuntu-latest
+    env:
+      GEN: ninja
+      BUILD_BENCHMARK: 1
+      VCPKG_TARGET_TRIPLET: x64-linux
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        with:
+          key: ${{ github.job }}
+
+      - name: Setup vcpkg
+        uses: lukka/run-vcpkg@v11.1
+        with:
+          vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+
+      - name: Configure OpenSSL for Rust
+        run: |
+          echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
+
+      - name: Build
+        shell: bash
+        run: |
+          make
+          git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb_delta.git --depth=1 
+          cd duckdb_delta
+          git submodule init
+          git submodule update
+          make
+          cd ..
+
+      - name: Generate test data
+        shell: bash
+        run: make generate-data
+
+      - name: Regression Test TPC-H
+        if: always()
+        shell: bash
+        run: |
+          python3 duckdb/scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpch_sf1_local.csv --verbose --threads=2 --root-dir=.
+
+      - name: Regression Test TPC-DS
+        if: always()
+        shell: bash
+        run: |
+          python duckdb/scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpcds_sf1_local.csv --verbose --threads=2 --root-dir=.
+
+      - name: Test benchmark makefile
+        shell: bash
+        run: |
+          make bench-run-tpch-sf1
+          make bench-run-tpcds-sf1
diff --git a/benchmark/benchmark.Makefile b/benchmark/benchmark.Makefile
@@ -8,7 +8,6 @@ ifeq ("$(IO_MODE)a", "a")
     IO_MODE:=local
 endif
 
-
 bench-output-dir:
 	mkdir -p benchmark_results
 
@@ -18,38 +17,24 @@ clean_benchmark:
 plot:
 	python3 scripts/plot.py
 
-
 ############### BENCHMARK TARGETS ###############
 
 ###
-# TPCH LOCAL
+# TPCH
 ###
 
 # TPCH SF1 on delta table
 bench-run-tpch-sf1-delta: bench-output-dir
-	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpch/sf1/delta/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-delta.csv
+	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpch/sf1/local/delta/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-delta.csv
 # TPCH SF1 on parquet files
 bench-run-tpch-sf1-parquet: bench-output-dir
-	./build/release/benchmark/benchmark_runner 'benchmark/tpch/sf1/parquet/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-parquet.csv
+	./build/release/benchmark/benchmark_runner 'benchmark/tpch/sf1-parquet/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-parquet.csv
 # TPCH SF1 on duckdb file
 bench-run-tpch-sf1-duckdb: bench-output-dir
-	./build/release/benchmark/benchmark_runner 'benchmark/tpch/sf1/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-duckdb.csv
-# COMPARES TPCH SF1 on parquet file vs on delta files
+	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpch/sf1/local/duckdb/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-duckdb.csv
+# COMPARES TPCH SF1 on parquet file vs on delta files vs on duckdb files
 bench-run-tpch-sf1: bench-run-tpch-sf1-delta bench-run-tpch-sf1-parquet
 
-###
-# TPCH REMOTE
-###
-
-# TPCH on remote delta table (set BENCHMARK_DATA_S3_LINEITEM_SF1)
-bench-run-tpch-sf1-remote-delta: bench-output-dir
-	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpch/sf1/delta-remote/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-remote-delta.csv
-# TPCH on remote parquet table (set BENCHMARK_DATA_S3_LINEITEM_SF1)
-bench-run-tpch-sf1-remote-parquet: bench-output-dir
-	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpch/sf1/parquet-remote/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-remote-parquet.csv
-# COMPARES TPCH SF1 on parquet file vs on delta files
-bench-run-tpch-sf1-remote: bench-run-tpch-sf1-remote-parquet bench-run-tpch-sf1-remote-delta
-
 ###
 # TPCDS
 ###

diff --git a/benchmark/tpcds/sf1/local/delta/load.sql b/benchmark/tpcds/sf1/local/delta/load.sql
@@ -1,4 +1,4 @@
-SET VARIABLE delta_path = '/mount/delta_benchmarking/tpcds_sf1_pyspark';
+SET VARIABLE delta_path = './data/generated/tpcds_sf1';
 
 create view call_center as from delta_scan(getvariable('delta_path') || '/call_center/delta_lake');
 create view catalog_page as from delta_scan(getvariable('delta_path') || '/catalog_page/delta_lake');

diff --git a/benchmark/tpcds/sf1/local/duckdb/tpcds_sf1.benchmark.in b/benchmark/tpcds/sf1/local/duckdb/tpcds_sf1.benchmark.in
@@ -13,7 +13,7 @@ require parquet
 require tpcds
 
 run
-attach '/mount/delta_benchmarking/tpcds_sf1_pyspark/duckdb.db' as tpcds;
+attach './data/generated/tpcds_sf1/duckdb.db' as tpcds;
 use tpcds;
 pragma tpcds(${QUERY_NUMBER})
 

diff --git a/benchmark/tpcds/sf1/local/parquet/load.sql b/benchmark/tpcds/sf1/local/parquet/load.sql
@@ -1,4 +1,4 @@
-SET VARIABLE parquet_path = '/mount/delta_benchmarking/tpcds_sf1_pyspark';
+SET VARIABLE parquet_path = './data/generated/tpcds_sf1';
 
 create view call_center as from parquet_scan(getvariable('parquet_path') || '/call_center/parquet/**/*.parquet');
 create view catalog_page as from parquet_scan(getvariable('parquet_path') || '/catalog_page/parquet/**/*.parquet');

diff --git a/benchmark/tpch/sf1/delta-remote/load.sql b/benchmark/tpch/sf1/delta-remote/load.sql
diff --git a/benchmark/tpch/sf1/delta-remote/tpch_sf1_delta.benchmark.in b/benchmark/tpch/sf1/delta-remote/tpch_sf1_delta.benchmark.in
diff --git a/benchmark/tpch/sf1/delta/q01.benchmark b/benchmark/tpch/sf1/delta/q01.benchmark
diff --git a/benchmark/tpch/sf1/delta/q02.benchmark b/benchmark/tpch/sf1/delta/q02.benchmark
diff --git a/benchmark/tpch/sf1/delta/q03.benchmark b/benchmark/tpch/sf1/delta/q03.benchmark
diff --git a/benchmark/tpch/sf1/delta/q04.benchmark b/benchmark/tpch/sf1/delta/q04.benchmark
diff --git a/benchmark/tpch/sf1/delta/q05.benchmark b/benchmark/tpch/sf1/delta/q05.benchmark
diff --git a/benchmark/tpch/sf1/delta/q06.benchmark b/benchmark/tpch/sf1/delta/q06.benchmark
diff --git a/benchmark/tpch/sf1/delta/q07.benchmark b/benchmark/tpch/sf1/delta/q07.benchmark
diff --git a/benchmark/tpch/sf1/delta/q08.benchmark b/benchmark/tpch/sf1/delta/q08.benchmark
diff --git a/benchmark/tpch/sf1/delta/q09.benchmark b/benchmark/tpch/sf1/delta/q09.benchmark
diff --git a/benchmark/tpch/sf1/delta/q10.benchmark b/benchmark/tpch/sf1/delta/q10.benchmark