update dockerfiles

MeteoSwiss · ninaburg · Nov 17, 2023 · Jan 4, 2024 · Jan 4, 2024 · Oct 1, 2024
commit e2bfca8e84904acbb8aeeca7cb27b05b03090a9c
diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,14 @@
-# Dockerfile to build Flexpart-IFS into an image
-ARG container_registry=dockerhub.apps.cp.meteoswiss.ch
-ARG base_tag=v0.2.1
+# Dockerfile to build Flexpart-IFS into an image, also containing the python module to prepare the input data.
+FROM dockerhub.apps.cp.meteoswiss.ch/mch/python/builder AS python-builder
+
+COPY utils/poetry.lock utils/pyproject.toml /scratch
+
+RUN cd /scratch \
+    && poetry export --without-hashes -o requirements.txt \
+    && poetry export --without-hashes --with dev -o requirements_dev.txt
 
 #  FROM ${container_registry}/numericalweatherpredictions/dispersionmodelling/flexpart-ifs/flexpart-base:${base_tag} as spack-builder
-FROM my-base-image:local as spack-builder
+FROM localhost/base-0612:latest as spack-builder
 
 ARG TOKEN
 ENV TOKEN=$TOKEN 
@@ -12,15 +17,19 @@ ARG COMMIT
 ENV COMMIT=$COMMIT
 
 # Add Flexpart-IFS source code
-RUN git clone git@github.com:MeteoSwiss/flexpart.git /scratch/flexpart && cd /scratch/flexpart && git checkout $COMMIT 
+# RUN git clone git@github.com:MeteoSwiss/flexpart.git /scratch/flexpart && cd /scratch/flexpart && git checkout $COMMIT 
+COPY  --chown=default_user:default_user ../flexpart-ifs /scratch/flexpart
+RUN chmod -R 777 /scratch/flexpart
+RUN cd /scratch
 
 # Install Flexpart-IFS
 RUN cd spack-env && \
     . $SPACK_ROOT/share/spack/setup-env.sh && \
     spack -e . -vv install -j1 --fail-fast && \
-    # Spack garbage collection 
     spack gc -y
 
+RUN git clone --depth 1 --branch 2.25.0 https://github.com/ecmwf/eccodes.git
+
 FROM docker-all-nexus.meteoswiss.ch/mch/ubuntu-jammy AS runner
 
 RUN apt-get -yqq update \
@@ -38,24 +47,35 @@ RUN apt-get -yqq update \
     flex \
     bison
 
+RUN mkdir -p \
+    /scratch/output/ \
+    /scratch/flexpart_ifs_utils/ \
+    /scratch/jobs/ \
+    /scratch/eccodes/definitions \
+    /scratch/db/
+
+COPY --from=python-builder /scratch/requirements.txt /scratch/requirements.txt
 COPY --from=spack-builder /scratch/spack-root/ /scratch/spack-root/
 COPY --from=spack-builder /scratch/spack-view/ /scratch/spack-view/
+COPY --from=spack-builder /scratch/eccodes/definitions /scratch/eccodes/definitions
+
 
+ENV GRIB_DEFINITION_PATH=/scratch/eccodes/definitions
 ENV PATH="/scratch/spack-view/bin:$PATH"
-ENV GRIB_DEFINITION_PATH=/scratch/eccodes-cosmo-resources/definitions:/scratch/eccodes/definitions
 ENV JOBS_DIR=/scratch/jobs/
-ENV FLEXPART_PREFIX=/scratch/spack-root/flexpart-cosmo/
+ENV FLEXPART_PREFIX=/scratch/spack-root/flexpart-ifs/
 
 WORKDIR /scratch
 
-COPY utils/flexpart_utils/ flexpart_utils/
+COPY utils/flexpart_ifs_utils/ flexpart_ifs_utils/
 COPY entrypoint.sh entrypoint.sh
+COPY data/IGBP_int1.dat $JOBS_DIR
 
 COPY utils/pip.conf /etc/pip.conf
 
 COPY utils/pyproject.toml /scratch
-# RUN python3.11 -m pip install -r requirements.txt && \
-#  python3.11 -m pip install .
+RUN python3.11 -m pip install -r requirements.txt && \
+    python3.11 -m pip install .
 
 RUN chmod -R a+rwx /scratch
 

diff --git a/Dockerfile.base b/Dockerfile.base
@@ -4,9 +4,9 @@ FROM docker-all-nexus.meteoswiss.ch/mch/ubuntu-jammy AS spack
 ENV DOCKERFILE_BASE=ubuntu            \
     DOCKERFILE_DISTRO=ubuntu          \
     DOCKERFILE_DISTRO_VERSION=20.04   \
-    SPACK_PREFIX=/root/c2sm-spack     \
-    SPACK_SYSTEM_CONFIG_PATH=/root/c2sm-spack/sysconfigs/unknown \
-    SPACK_ROOT=/root/c2sm-spack/spack \
+    SPACK_PREFIX=/opt/c2sm-spack     \
+    SPACK_SYSTEM_CONFIG_PATH=/opt/c2sm-spack/sysconfigs/unknown \
+    SPACK_ROOT=/opt/c2sm-spack/spack \
     SPACK_TAG=v0.20.1.3               \
     DEBIAN_FRONTEND=noninteractive    \
     CURRENTLY_BUILDING_DOCKER_IMAGE=1
@@ -52,7 +52,7 @@ ENV LANGUAGE=en_US.UTF-8 \
 
 RUN git clone --depth 1 --recurse-submodules --shallow-submodules -b ${SPACK_TAG} https://github.com/C2SM/spack-c2sm.git ${SPACK_PREFIX}
 RUN cd ${SPACK_PREFIX} && git rev-parse --verify HEAD && . ./setup-env.sh && spack compiler find && spack external find --all
-RUN chmod -R 777 ${SPACK_PREFIX}/setup-env.sh && echo $SPACK_SYSTEM_CONFIG_PATH
+RUN chmod 777 ${SPACK_PREFIX}/setup-env.sh  && echo $SPACK_SYSTEM_CONFIG_PATH
 
 RUN mkdir -p $SPACK_ROOT
 
@@ -63,10 +63,10 @@ RUN ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
     && ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
     /usr/local/bin/spack-env
 
-RUN mkdir -p /root/.spack \
+RUN mkdir -p /opt/.spack \
     && cp $SPACK_ROOT/share/spack/docker/modules.yaml \
-    /root/.spack/modules.yaml \
-    && rm -rf /root/*.* /run/nologin $SPACK_ROOT/.git
+    /opt/.spack/modules.yaml \
+    && rm -rf /opt/*.* /run/nologin $SPACK_ROOT/.git
 
 FROM spack AS builder
 
@@ -76,27 +76,34 @@ RUN apt-get -yqq update \
     && spack external find
 
 # Make scratch
-RUN mkdir -p /scratch/flexpart/ /scratch/spack-root 
+RUN mkdir -p /scratch/flexpart/ /scratch/spack-root && chmod -R 777 /scratch/flexpart
 WORKDIR /scratch
 
 # Create spack environment and install dependencies
 RUN mkdir spack-env && \
     cd spack-env && \
     . $SPACK_ROOT/share/spack/setup-env.sh && \
-    spack env create -d . && \
-    spack -e . concretize --force || { echo "Error concretizing environment"; exit 1; } && \
-    spack -e . add flexpart-ifs@main  target=x86_64 || { echo "Error adding spec"; exit 1; } && \
-    # Configure development path for the package
+    spack env create -d . --with-view /scratch/spack-view/ && \
+    spack -e . add flexpart-ifs@main target=x86_64 || { echo "Error adding spec"; exit 1; } && \
     spack -e . develop --no-clone --path /scratch/flexpart flexpart-ifs@main || { echo "Error adding develop"; exit 1; } && \
     spack -e . config --scope env:/scratch/spack-env add -f /scratch/spack-env/spack.yaml config:install_tree:root:/scratch/spack-root && \
-    spack -e . config --scope env:/scratch/spack-env add -f /scratch/spack-env/spack.yaml config:install_path_scheme:"\"{name}\"" && \ 
+    spack -e . config --scope env:/scratch/spack-env add -f /scratch/spack-env/spack.yaml config:install_path_scheme:"\"{name}\""
+
+# Install dependencies
+RUN cd spack-env && \
+    . $SPACK_ROOT/share/spack/setup-env.sh && \
     spack -e . concretize && \
     spack -e . install --only=dependencies --fail-fast
 
 ARG USERNAME=default_user
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
+
 # Create the user
 RUN groupadd --gid $USER_GID $USERNAME \
     && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME
+
+RUN chown -R $USERNAME:$USERNAME /scratch /opt
+
+# Switch to default_user for subsequent steps
 USER $USERNAME