Improve Dockerfile: llvm16 workarounds + doc + local test (#243)

* improve dockerfile and add a doc - improve Dockerfile ( new Alpine versions; workarounds LLVM 16 ) - add .dockerignore - add simple doc doc/docker_usage.md - add simple local ci ci/local_docker_matrix.sh * fix docker doc * add simple docker build test matrix * improve text in dockertest.yml
orioledb · Jun 11, 2023 · f282011 · f282011
1 parent 828e94b
commit f282011
Show file tree

Hide file tree

Showing 6 changed files with 416 additions and 6 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,46 @@
+# Global excludes across all subdirectories
+**/*.o
+**/*.obj
+**/*.bc
+**/*.so
+**/*.so.[0-9]
+**/*.so.[0-9].[0-9]
+**/*.so.[0-9].[0-9][0-9]
+**/*.sl
+**/*.sl.[0-9]
+**/*.sl.[0-9].[0-9]
+**/*.sl.[0-9].[0-9][0-9]
+**/*.dylib
+**/*.dll
+**/*.exp
+**/*.a
+**/*.mo
+**/*.pot
+**/objfiles.txt
+**/.deps/
+**/*.gcno
+**/*.gcda
+**/*.gcov
+**/*.gcov.out
+**/lcov*.info
+**/coverage/
+**/coverage-html-stamp
+**/*.vcproj
+**/*.vcxproj
+**/win32ver.rc
+**/*.exe
+**/lib*dll.def
+**/lib*.pc
+
+# Local excludes in root directory
+t/__pycache__/
+log/
+log_docker_build/
+results/
+tmp_check/
+tmp_check_iso/
+output_iso/
+include/utils/stopevents_defs.h
+include/utils/stopevents_data.h
+orioledb.typedefs
+!**/ci/antithesis/libvoidstar.so
diff --git a/.github/workflows/dockertest.yml b/.github/workflows/dockertest.yml
@@ -0,0 +1,50 @@
+name: dockerTEST
+
+on:
+  push:
+  pull_request:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  test-docker-builds:
+    strategy:
+      fail-fast: true
+      matrix:
+        postgres: [16, 13]
+        compiler: [clang, gcc]
+        alpine:  [edge, 3.18, 3.13]
+
+    name: docker ${{ matrix.postgres }}-${{ matrix.compiler }}-alpine${{ matrix.alpine }}
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.alpine == 'edge' }}
+
+    steps:
+      - name: Set DOCKER_PG_LLVM_DEPS
+        run: |
+          if [[ "${{ matrix.alpine }}" == "3.18" ]] || [[ "${{ matrix.alpine }}" == "edge" ]]; \
+          then echo "DOCKER_PG_LLVM_DEPS=llvm15-dev clang15" >> $GITHUB_ENV; \
+          else echo "DOCKER_PG_LLVM_DEPS=llvm-dev clang"     >> $GITHUB_ENV; \
+          fi
+
+      - name: Echo DOCKER_PG_LLVM_DEP
+        run: echo "DOCKER_PG_LLVM_DEP = ${{ env.DOCKER_PG_LLVM_DEPS }} "
+
+      - name: Checkout source
+        uses: actions/checkout@v3
+
+      - name: docker build orioletest:${{ matrix.postgres }}-${{ matrix.compiler }}-alpine${{ matrix.alpine }}
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: linux/amd64
+          push: false
+          pull: true
+          tags: orioletest:${{ matrix.postgres }}-${{ matrix.compiler }}-alpine${{ matrix.alpine }}
+          build-args: |
+            ALPINE_VERSION=${{ matrix.alpine }}
+            BUILD_CC_COMPILER=${{ matrix.compiler }}
+            PG_MAJOR=${{ matrix.postgres }}
+            DOCKER_PG_LLVM_DEPS=${{ env.DOCKER_PG_LLVM_DEPS }}
diff --git a/.gitignore b/.gitignore
@@ -35,6 +35,7 @@ lib*.pc
 # Local excludes in root directory
 /t/__pycache__/
 /log/
+/log_docker_build/
 /results/
 /tmp_check/
 /tmp_check_iso/

diff --git a/Dockerfile b/Dockerfile
@@ -1,19 +1,27 @@
 # This is slightly adjusted Dockerfile from
 # https://github.com/docker-library/postgres
 
-# set ALPINE_VERSION= [ edge 3.17 3.16 3.15 3.14 3.13 ]
+# set ALPINE_VERSION= [ edge 3.18 3.17 3.16 3.15 3.14 3.13 ]
 ARG ALPINE_VERSION=3.17
 FROM alpine:${ALPINE_VERSION}
 
 ARG ALPINE_VERSION
-# Set PG_MAJOR = [ 15 14 13 ]
+# Set PG_MAJOR = [16 15 14 13 ]
 ARG PG_MAJOR=14
 ENV PG_MAJOR ${PG_MAJOR}
 
 # set compiler: [ clang gcc ]
 ARG BUILD_CC_COMPILER=clang
 ENV BUILD_CC_COMPILER ${BUILD_CC_COMPILER}
 
+# Define build dependencies for LLVM [ llvm-dev clang ]
+# These include the specific versions of 'llvm-dev' and 'clang' suitable for the current version of PostgreSQL.
+# They are useful for building downstream extensions using the same LLVM, like PostGIS alpine https://github.com/postgis/docker-postgis
+# Note: PostgreSQL does not support LLVM 16. Therefore, for Alpine >=3.18, please use "llvm15-dev clang15".
+# Reference: https://github.com/docker-library/postgres/pull/1077
+ARG DOCKER_PG_LLVM_DEPS="llvm-dev clang"
+ENV DOCKER_PG_LLVM_DEPS ${DOCKER_PG_LLVM_DEPS}
+
 # 70 is the standard uid/gid for "postgres" in Alpine
 # https://git.alpinelinux.org/aports/tree/main/postgresql/postgresql.pre-install?h=3.12-stable
 RUN set -eux; \
@@ -45,7 +53,7 @@ RUN set -eux; \
 # https://wiki.alpinelinux.org/wiki/Release_Notes_for_Alpine_3.16.0#ICU_data_split
 # https://github.com/docker-library/postgres/issues/327#issuecomment-1201582069
 	case "$ALPINE_VERSION" in 3.13 | 3.14 | 3.15 )  EXTRA_ICU_PACKAGES='' ;; \
-		3.16 | 3.17 | 3.18* ) EXTRA_ICU_PACKAGES=icu-data-full ;; \
+		3.16 | 3.17 | 3.18 | 3.19* ) EXTRA_ICU_PACKAGES=icu-data-full ;; \
 		*) : ;; \
 	esac ; \
 	\
@@ -56,22 +64,51 @@ RUN set -eux; \
 	echo "ORIOLEDB_BUILDTIME=$ORIOLEDB_BUILDTIME" ; \
 	echo "ALPINE_VERSION=$ALPINE_VERSION" ; \
 	echo "EXTRA_ICU_PACKAGES=$EXTRA_ICU_PACKAGES" ; \
+	echo "DOCKER_PG_LLVM_DEPS=$DOCKER_PG_LLVM_DEPS" ; \	
+	\
+# check if the custom llvm version is set, and if so, set the LLVM_CONFIG and CLANG variables
+	CUSTOM_LLVM_VERSION=$(echo "$DOCKER_PG_LLVM_DEPS" | sed -n 's/.*llvm\([0-9]*\).*/\1/p') ; \
+	if [ ! -z "${CUSTOM_LLVM_VERSION}" ];  then \
+		echo "CUSTOM_LLVM_VERSION=$CUSTOM_LLVM_VERSION" ; \
+		export LLVM_CONFIG="/usr/lib/llvm${CUSTOM_LLVM_VERSION}/bin/llvm-config" ; \
+		export CLANG=clang-${CUSTOM_LLVM_VERSION} ; \
+		if [[ "${BUILD_CC_COMPILER}" == "clang" ]]; then \
+			export BUILD_CC_COMPILER=clang-${CUSTOM_LLVM_VERSION} ; \
+			echo "fix: BUILD_CC_COMPILER=clang-${CUSTOM_LLVM_VERSION}" ; \
+		fi ; \
+	fi ; \
+	\
+# temporary not allowing LLVM 16 to be used
+# reason: we can't overwrite and fix the DOCKER_PG_LLVM_DEPS
+# and the future downstream extensions like PostGIS need a correct build information (DOCKER_PG_LLVM_DEPS)
+	if \
+	# if the custom llvm version is set ( via DOCKER_PG_LLVM_DEPS ), and it is 16, then halt operation
+		( [ ! -z "${CUSTOM_LLVM_VERSION}" ] && [ "$CUSTOM_LLVM_VERSION" == "16" ] ) \
+	# or - if the custom llvm version is not set, and the Alpine version is >=3.18, then halt operation
+	  ||  ( [ -z "${CUSTOM_LLVM_VERSION}" ] && ( [ "$ALPINE_VERSION" == "3.18" ] || [ "$ALPINE_VERSION" == "3.19" ]) ) \
+	  ;  then \
+			set +x ; \	  
+			echo "------------------------------" ; \
+			echo "Error: The LLVM 16 is not compatible with the current PostgreSQL! Halting operation." ; \
+			echo "Suggested workarounds: use --build-arg DOCKER_PG_LLVM_DEPS='llvm15-dev clang15' " ; \
+			exit 1; \
+	fi ; \
 	\
 	apk add --no-cache --virtual .build-deps \
+		${DOCKER_PG_LLVM_DEPS} \
 		bison \
-		clang \
 		coreutils \
 		curl \
 		dpkg-dev dpkg \
 		flex \
+		g++ \
 		gcc \
 		krb5-dev \
 		libc-dev \
 		libedit-dev \
 		libxml2-dev \
 		libxslt-dev \
 		linux-headers \
-		llvm-dev clang g++ \
 		make \
 		openldap-dev \
 		openssl-dev \
@@ -134,7 +171,6 @@ RUN set -eux; \
 		--prefix=/usr/local \
 		--with-includes=/usr/local/include \
 		--with-libraries=/usr/local/lib \
-		--with-krb5 \
 		--with-gssapi \
 		--with-ldap \
 		--with-tcl \

diff --git a/ci/local_docker_matrix.sh b/ci/local_docker_matrix.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -Eeo pipefail
+
+# Testing all possible docker builds on a local machine
+# run from project root: ./ci/local_docker_matrix.sh
+# and check the logs in ./log_docker_build/*.log
+
+# set and prepare $logpath for build logs
+logpath=./log_docker_build
+mkdir -p $logpath
+rm -f    ${logpath}/*.log
+
+# simple local build matxix - for build all images
+for alpine in edge 3.18 3.17 3.16 3.15 3.14 3.13 ; do
+  # Refresh alpine images!
+  docker pull alpine:$alpine
+  for pg_major in 16 15 14 13; do
+    for compiler in clang gcc; do
+
+      # LLVM 16 hack for setting the correct DOCKER_PG_LLVM_DEPS
+      case "$alpine" in 3.18 | edge )  pg_llvm_deps='llvm15-dev clang15' ;; \
+                                  * )  pg_llvm_deps='llvm-dev   clang'   ;; \
+      esac ; \
+
+      docker_tag="${pg_major}-${compiler}-alpine${alpine}"
+      echo "------------ $docker_tag ------------------"
+      echo "params: ALPINE_VERSION=$alpine"
+      echo "params: BUILD_CC_COMPILER=$compiler"
+      echo "params: PG_MAJOR=$pg_major"
+      echo "params: DOCKER_PG_LLVM_DEPS=$pg_llvm_deps"
+      echo "----------------------------------------"
+
+      rm -f ${logpath}/${docker_tag}.log
+      time docker build --network=host --progress=plain \
+          --build-arg ALPINE_VERSION="$alpine" \
+          --build-arg BUILD_CC_COMPILER="$compiler" \
+          --build-arg PG_MAJOR="$pg_major" \
+          --build-arg DOCKER_PG_LLVM_DEPS="$pg_llvm_deps" \
+          -t orioletest:${docker_tag} . 2>&1 | tee ${logpath}/${docker_tag}.log
+
+    done
+  done
+done
+
+docker images orioletest:* | sort