From f2820112c73a18b94e10cf0c667b401898260a22 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 11 Jun 2023 02:03:37 +0200 Subject: [PATCH] Improve Dockerfile: llvm16 workarounds + doc + local test (#243) * improve dockerfile and add a doc - improve Dockerfile ( new Alpine versions; workarounds LLVM 16 ) - add .dockerignore - add simple doc doc/docker_usage.md - add simple local ci ci/local_docker_matrix.sh * fix docker doc * add simple docker build test matrix * improve text in dockertest.yml --- .dockerignore | 46 ++++++ .github/workflows/dockertest.yml | 50 +++++++ .gitignore | 1 + Dockerfile | 48 ++++++- ci/local_docker_matrix.sh | 45 ++++++ doc/docker_usage.md | 232 +++++++++++++++++++++++++++++++ 6 files changed, 416 insertions(+), 6 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/dockertest.yml create mode 100755 ci/local_docker_matrix.sh create mode 100644 doc/docker_usage.md diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..cd53c679 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,46 @@ +# Global excludes across all subdirectories +**/*.o +**/*.obj +**/*.bc +**/*.so +**/*.so.[0-9] +**/*.so.[0-9].[0-9] +**/*.so.[0-9].[0-9][0-9] +**/*.sl +**/*.sl.[0-9] +**/*.sl.[0-9].[0-9] +**/*.sl.[0-9].[0-9][0-9] +**/*.dylib +**/*.dll +**/*.exp +**/*.a +**/*.mo +**/*.pot +**/objfiles.txt +**/.deps/ +**/*.gcno +**/*.gcda +**/*.gcov +**/*.gcov.out +**/lcov*.info +**/coverage/ +**/coverage-html-stamp +**/*.vcproj +**/*.vcxproj +**/win32ver.rc +**/*.exe +**/lib*dll.def +**/lib*.pc + +# Local excludes in root directory +t/__pycache__/ +log/ +log_docker_build/ +results/ +tmp_check/ +tmp_check_iso/ +output_iso/ +include/utils/stopevents_defs.h +include/utils/stopevents_data.h +orioledb.typedefs +!**/ci/antithesis/libvoidstar.so \ No newline at end of file diff --git a/.github/workflows/dockertest.yml b/.github/workflows/dockertest.yml new file mode 100644 index 00000000..4903468c --- /dev/null +++ b/.github/workflows/dockertest.yml @@ -0,0 +1,50 @@ +name: dockerTEST + +on: + push: + pull_request: + +defaults: + run: + shell: bash + +jobs: + test-docker-builds: + strategy: + fail-fast: true + matrix: + postgres: [16, 13] + compiler: [clang, gcc] + alpine: [edge, 3.18, 3.13] + + name: docker ${{ matrix.postgres }}-${{ matrix.compiler }}-alpine${{ matrix.alpine }} + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.alpine == 'edge' }} + + steps: + - name: Set DOCKER_PG_LLVM_DEPS + run: | + if [[ "${{ matrix.alpine }}" == "3.18" ]] || [[ "${{ matrix.alpine }}" == "edge" ]]; \ + then echo "DOCKER_PG_LLVM_DEPS=llvm15-dev clang15" >> $GITHUB_ENV; \ + else echo "DOCKER_PG_LLVM_DEPS=llvm-dev clang" >> $GITHUB_ENV; \ + fi + + - name: Echo DOCKER_PG_LLVM_DEP + run: echo "DOCKER_PG_LLVM_DEP = ${{ env.DOCKER_PG_LLVM_DEPS }} " + + - name: Checkout source + uses: actions/checkout@v3 + + - name: docker build orioletest:${{ matrix.postgres }}-${{ matrix.compiler }}-alpine${{ matrix.alpine }} + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64 + push: false + pull: true + tags: orioletest:${{ matrix.postgres }}-${{ matrix.compiler }}-alpine${{ matrix.alpine }} + build-args: | + ALPINE_VERSION=${{ matrix.alpine }} + BUILD_CC_COMPILER=${{ matrix.compiler }} + PG_MAJOR=${{ matrix.postgres }} + DOCKER_PG_LLVM_DEPS=${{ env.DOCKER_PG_LLVM_DEPS }} diff --git a/.gitignore b/.gitignore index 741a60fd..b8502f67 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ lib*.pc # Local excludes in root directory /t/__pycache__/ /log/ +/log_docker_build/ /results/ /tmp_check/ /tmp_check_iso/ diff --git a/Dockerfile b/Dockerfile index 4f8433d7..f5e6f07d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ # This is slightly adjusted Dockerfile from # https://github.com/docker-library/postgres -# set ALPINE_VERSION= [ edge 3.17 3.16 3.15 3.14 3.13 ] +# set ALPINE_VERSION= [ edge 3.18 3.17 3.16 3.15 3.14 3.13 ] ARG ALPINE_VERSION=3.17 FROM alpine:${ALPINE_VERSION} ARG ALPINE_VERSION -# Set PG_MAJOR = [ 15 14 13 ] +# Set PG_MAJOR = [16 15 14 13 ] ARG PG_MAJOR=14 ENV PG_MAJOR ${PG_MAJOR} @@ -14,6 +14,14 @@ ENV PG_MAJOR ${PG_MAJOR} ARG BUILD_CC_COMPILER=clang ENV BUILD_CC_COMPILER ${BUILD_CC_COMPILER} +# Define build dependencies for LLVM [ llvm-dev clang ] +# These include the specific versions of 'llvm-dev' and 'clang' suitable for the current version of PostgreSQL. +# They are useful for building downstream extensions using the same LLVM, like PostGIS alpine https://github.com/postgis/docker-postgis +# Note: PostgreSQL does not support LLVM 16. Therefore, for Alpine >=3.18, please use "llvm15-dev clang15". +# Reference: https://github.com/docker-library/postgres/pull/1077 +ARG DOCKER_PG_LLVM_DEPS="llvm-dev clang" +ENV DOCKER_PG_LLVM_DEPS ${DOCKER_PG_LLVM_DEPS} + # 70 is the standard uid/gid for "postgres" in Alpine # https://git.alpinelinux.org/aports/tree/main/postgresql/postgresql.pre-install?h=3.12-stable RUN set -eux; \ @@ -45,7 +53,7 @@ RUN set -eux; \ # https://wiki.alpinelinux.org/wiki/Release_Notes_for_Alpine_3.16.0#ICU_data_split # https://github.com/docker-library/postgres/issues/327#issuecomment-1201582069 case "$ALPINE_VERSION" in 3.13 | 3.14 | 3.15 ) EXTRA_ICU_PACKAGES='' ;; \ - 3.16 | 3.17 | 3.18* ) EXTRA_ICU_PACKAGES=icu-data-full ;; \ + 3.16 | 3.17 | 3.18 | 3.19* ) EXTRA_ICU_PACKAGES=icu-data-full ;; \ *) : ;; \ esac ; \ \ @@ -56,14 +64,44 @@ RUN set -eux; \ echo "ORIOLEDB_BUILDTIME=$ORIOLEDB_BUILDTIME" ; \ echo "ALPINE_VERSION=$ALPINE_VERSION" ; \ echo "EXTRA_ICU_PACKAGES=$EXTRA_ICU_PACKAGES" ; \ + echo "DOCKER_PG_LLVM_DEPS=$DOCKER_PG_LLVM_DEPS" ; \ + \ +# check if the custom llvm version is set, and if so, set the LLVM_CONFIG and CLANG variables + CUSTOM_LLVM_VERSION=$(echo "$DOCKER_PG_LLVM_DEPS" | sed -n 's/.*llvm\([0-9]*\).*/\1/p') ; \ + if [ ! -z "${CUSTOM_LLVM_VERSION}" ]; then \ + echo "CUSTOM_LLVM_VERSION=$CUSTOM_LLVM_VERSION" ; \ + export LLVM_CONFIG="/usr/lib/llvm${CUSTOM_LLVM_VERSION}/bin/llvm-config" ; \ + export CLANG=clang-${CUSTOM_LLVM_VERSION} ; \ + if [[ "${BUILD_CC_COMPILER}" == "clang" ]]; then \ + export BUILD_CC_COMPILER=clang-${CUSTOM_LLVM_VERSION} ; \ + echo "fix: BUILD_CC_COMPILER=clang-${CUSTOM_LLVM_VERSION}" ; \ + fi ; \ + fi ; \ + \ +# temporary not allowing LLVM 16 to be used +# reason: we can't overwrite and fix the DOCKER_PG_LLVM_DEPS +# and the future downstream extensions like PostGIS need a correct build information (DOCKER_PG_LLVM_DEPS) + if \ + # if the custom llvm version is set ( via DOCKER_PG_LLVM_DEPS ), and it is 16, then halt operation + ( [ ! -z "${CUSTOM_LLVM_VERSION}" ] && [ "$CUSTOM_LLVM_VERSION" == "16" ] ) \ + # or - if the custom llvm version is not set, and the Alpine version is >=3.18, then halt operation + || ( [ -z "${CUSTOM_LLVM_VERSION}" ] && ( [ "$ALPINE_VERSION" == "3.18" ] || [ "$ALPINE_VERSION" == "3.19" ]) ) \ + ; then \ + set +x ; \ + echo "------------------------------" ; \ + echo "Error: The LLVM 16 is not compatible with the current PostgreSQL! Halting operation." ; \ + echo "Suggested workarounds: use --build-arg DOCKER_PG_LLVM_DEPS='llvm15-dev clang15' " ; \ + exit 1; \ + fi ; \ \ apk add --no-cache --virtual .build-deps \ + ${DOCKER_PG_LLVM_DEPS} \ bison \ - clang \ coreutils \ curl \ dpkg-dev dpkg \ flex \ + g++ \ gcc \ krb5-dev \ libc-dev \ @@ -71,7 +109,6 @@ RUN set -eux; \ libxml2-dev \ libxslt-dev \ linux-headers \ - llvm-dev clang g++ \ make \ openldap-dev \ openssl-dev \ @@ -134,7 +171,6 @@ RUN set -eux; \ --prefix=/usr/local \ --with-includes=/usr/local/include \ --with-libraries=/usr/local/lib \ - --with-krb5 \ --with-gssapi \ --with-ldap \ --with-tcl \ diff --git a/ci/local_docker_matrix.sh b/ci/local_docker_matrix.sh new file mode 100755 index 00000000..0e2331dd --- /dev/null +++ b/ci/local_docker_matrix.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -Eeo pipefail + +# Testing all possible docker builds on a local machine +# run from project root: ./ci/local_docker_matrix.sh +# and check the logs in ./log_docker_build/*.log + +# set and prepare $logpath for build logs +logpath=./log_docker_build +mkdir -p $logpath +rm -f ${logpath}/*.log + +# simple local build matxix - for build all images +for alpine in edge 3.18 3.17 3.16 3.15 3.14 3.13 ; do + # Refresh alpine images! + docker pull alpine:$alpine + for pg_major in 16 15 14 13; do + for compiler in clang gcc; do + + # LLVM 16 hack for setting the correct DOCKER_PG_LLVM_DEPS + case "$alpine" in 3.18 | edge ) pg_llvm_deps='llvm15-dev clang15' ;; \ + * ) pg_llvm_deps='llvm-dev clang' ;; \ + esac ; \ + + docker_tag="${pg_major}-${compiler}-alpine${alpine}" + echo "------------ $docker_tag ------------------" + echo "params: ALPINE_VERSION=$alpine" + echo "params: BUILD_CC_COMPILER=$compiler" + echo "params: PG_MAJOR=$pg_major" + echo "params: DOCKER_PG_LLVM_DEPS=$pg_llvm_deps" + echo "----------------------------------------" + + rm -f ${logpath}/${docker_tag}.log + time docker build --network=host --progress=plain \ + --build-arg ALPINE_VERSION="$alpine" \ + --build-arg BUILD_CC_COMPILER="$compiler" \ + --build-arg PG_MAJOR="$pg_major" \ + --build-arg DOCKER_PG_LLVM_DEPS="$pg_llvm_deps" \ + -t orioletest:${docker_tag} . 2>&1 | tee ${logpath}/${docker_tag}.log + + done + done +done + +docker images orioletest:* | sort diff --git a/doc/docker_usage.md b/doc/docker_usage.md new file mode 100644 index 00000000..d67d72af --- /dev/null +++ b/doc/docker_usage.md @@ -0,0 +1,232 @@ +# Docker Documentation for OrioleDB + +This document provides instructions on how to build Docker images for OrioleDB, and how to test them. + +#### Prerequisites + +Before you begin, make sure you have Docker installed on your local machine. If not, you can download and install it from the Docker official website. https://docs.docker.com/get-docker/ + +* `docker -v` + +## Quickstart + +Open a terminal and navigate to the OrioleDB project directory, if you are not already in it: +* `cd path/to/orioledb` + +Build PostgreSQL 15 + OrioleDB extension: + +* `docker build -t orioletest:15 --pull --network=host --progress=plain --build-arg PG_MAJOR="15" .` + +Start server: + +* `docker run --name oriolest15 -v orioletest15data:/var/lib/postgresql/data -e POSTGRES_PASSWORD=oriole123 -d orioletest:15` + +Connect to the server via psql: + +* `docker exec -ti oriolest15 psql -U postgres` + +You should expect a similar psql message: +``` +psql (15.2 OrioleDB public alpha 13 PGTAG=patches15_13 alpine:3.17+clang build:2023-06-02T19:08:58+00:00) +Type "help" for help. + +postgres=# +``` + +Enable orioledb extension: + +* `create extension if not exists orioledb;` + +Test some commands: +``` +postgres=# \d+ + List of relations + Schema | Name | Type | Owner | Persistence | Access method | Size | Description +--------+----------------------+------+----------+-------------+---------------+---------+------------- + public | orioledb_index | view | postgres | permanent | | 0 bytes | + public | orioledb_index_descr | view | postgres | permanent | | 0 bytes | + public | orioledb_table | view | postgres | permanent | | 0 bytes | + public | orioledb_table_descr | view | postgres | permanent | | 0 bytes | +(4 rows) + +postgres=# \dx + List of installed extensions + Name | Version | Schema | Description +----------+---------+------------+------------------------------------------------------ + orioledb | 1.0 | public | OrioleDB -- the next generation transactional engine + plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language +(2 rows) + +postgres=# \dx+ orioledb + Objects in extension "orioledb" + Object description +------------------------------------------------------------------------- + access method orioledb + function orioledb_commit_hash() + function orioledb_compression_max_level() + function orioledb_evict_pages(oid,integer) + function orioledb_get_index_descrs() + function orioledb_get_table_descrs() + function orioledb_has_retained_undo() + function orioledb_idx_structure(oid,text,character varying,integer) + function orioledb_index_description(oid,oid,oid,text) + function orioledb_index_oids() + function orioledb_index_rows(oid) + function orioledb_page_stats() + function orioledb_parallel_debug_start() + function orioledb_parallel_debug_stop() + function orioledb_recovery_synchronized() + function orioledb_relation_size(oid) + function orioledb_sys_tree_check(integer,boolean) + function orioledb_sys_tree_rows(integer) + function orioledb_sys_tree_structure(integer,character varying,integer) + function orioledb_table_description(oid) + function orioledb_table_description(oid,oid,oid) + function orioledb_table_oids() + function orioledb_table_pages(oid) + function orioledb_tableam_handler(internal) + function orioledb_tbl_are_indices_equal(regclass,regclass) + function orioledb_tbl_check(oid,boolean) + function orioledb_tbl_compression_check(bigint,oid,integer[]) + function orioledb_tbl_indices(oid) + function orioledb_tbl_structure(oid,character varying,integer) + function orioledb_ucm_check() + function orioledb_version() + function orioledb_write_pages(oid) + function pg_stopevent_reset(text) + function pg_stopevent_set(text,jsonpath) + function pg_stopevents() + view orioledb_index + view orioledb_index_descr + view orioledb_table + view orioledb_table_descr +(39 rows) + +postgres=# select orioledb_version(); + orioledb_version +-------------------------- + OrioleDB public alpha 13 +(1 row) + + +``` + +Quit from the database: `\q` + + +Stop the server: + +* `docker stop oriolest15` + +Remove container: + +* `docker container rm oriolest15` + +Remove docker image: + +* `docker rmi orioletest:15` + +Remove the data volume: + +* `docker volume rm orioletest15data` + + + +## Building Docker Images + +To build a Docker image, use one of the following commands: + +#### To build PostgreSQL 16 + OrieleDB extension +``` +docker build -t orioletest:16 --pull --network=host --progress=plain --build-arg PG_MAJOR="16" . +``` + +#### To build PostgreSQL 15 + OrieleDB extension +``` +docker build -t orioletest:15 --pull --network=host --progress=plain --build-arg PG_MAJOR="15" . +``` + +#### To build PostgreSQL 14 + OrieleDB extension +``` +docker build -t orioletest:14 --pull --network=host --progress=plain --build-arg PG_MAJOR="14" . +``` + +#### To build PostgreSQL 13 + OrieleDB extension +``` +docker build -t orioletest:13 --pull --network=host --progress=plain --build-arg PG_MAJOR="13" . +``` + +## Supported environment variables: + +This project aims to maintain compatibility with the Docker Official PostgreSQL image, and therefore, it also supports the environmental variables found there: + +* `POSTGRES_PASSWORD` +* `POSTGRES_USER` +* `POSTGRES_DB` +* `POSTGRES_INITDB_ARGS` +* `POSTGRES_INITDB_WALDIR` +* `POSTGRES_HOST_AUTH_METHOD` +* `PGDATA` + +Read more: https://github.com/docker-library/docs/blob/master/postgres/README.md + +## Available Docker build args + +* `--build-arg ALPINE_VERSION="3.18"` + * Choose which version of Alpine Linux to use. Default is `3.17`. + * You can choose from `edge`, `3.18`, `3.17`, `3.16`, `3.15`, `3.14`, `3.13`. +* `--build-arg BUILD_CC_COMPILER="gcc"` + * Choose the C compiler. Default is `clang`. + * You can choose either `clang` or `gcc`. + +* `--build-arg PG_MAJOR="16"` + * Choose the main version of PostgreSQL. Default is `14`. + * You can choose from `16`, `15`, `14`, `13`. +* `--build-arg DOCKER_PG_LLVM_DEPS='lvm15-dev clang15'` + * Choose the LLVM build environment. Default is `llvm-dev clang`. + * If you're using Alpine version `3.18` or higher, use `llvm15` because `llvm16` is not supported yet. + +For example, to build an image using Alpine version `3.17`, the `gcc` compiler and PostgreSQL version `14`, use the following command: + +``` +docker build --network=host --progress=plain \ + --build-arg ALPINE_VERSION="3.17" \ + --build-arg BUILD_CC_COMPILER="gcc" \ + --build-arg PG_MAJOR="14" \ + -t orioletest:14-gcc-alpine3.17 . +``` +This command will build the Docker image and tag it as `orioletest:14-gcc-alpine3.17.` + +## Experimental OrioleDB + PostGIS Extension build: + +Known limitations: +- It only works with Alpine `3.18`. This is due to a Docker `postgis/docker-postgis` limitation. The build script expects the `sfcgal` package, which is only available in Alpine 3.18 or later versions. +- OrioleDB `gist`, `sp-gist`, and other related indexes are not yet supported. + + +#### Step 1: create image: `orioletest:15-gcc-alpine3.18` + +``` +docker build --pull --network=host --progress=plain \ + --build-arg ALPINE_VERSION="3.18" \ + --build-arg BUILD_CC_COMPILER="gcc" \ + --build-arg PG_MAJOR="15" \ + --build-arg DOCKER_PG_LLVM_DEPS="llvm15-dev clang15" \ + -t orioletest:15-gcc-alpine3.18 . +``` + +#### Step2: Build the `oriolegis:15-3.3` image. + +in a new directory, run this commands: + +``` +git clone https://github.com/postgis/docker-postgis.git +cd ./docker-postgis/15-3.3/alpine +docker build --network=host --progress=plain \ + --build-arg BASE_IMAGE=orioletest:15-gcc-alpine3.18 \ + -t oriolegis:15-3.3 . +``` +## Developer notes: + +To build all Docker image variations on a local machine, run the following command: +* `./ci/local_docker_matrix.sh`