Skip to content

Commit

Permalink
fix: notebook server images with non-root SecurityContext (#7622)
Browse files Browse the repository at this point in the history
* fix: notebook server images with non-root SecurityContext

Signed-off-by: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com>

* fix: set `S6_BEHAVIOUR_IF_STAGE2_FAILS` to `2`

Signed-off-by: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com>

* fix: permissions for gid 0

Signed-off-by: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com>

* fix: rstudio with random uid

Signed-off-by: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com>

* fix: capture STDERR in jupyter and code-server

Signed-off-by: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com>

---------

Signed-off-by: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com>
  • Loading branch information
thesuperzapper authored Jul 4, 2024
1 parent 2898d0f commit 32ff054
Show file tree
Hide file tree
Showing 20 changed files with 135 additions and 47 deletions.
44 changes: 35 additions & 9 deletions components/example-notebook-servers/base/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,23 +10,35 @@ ARG TARGETARCH
# common environemnt variables
ENV NB_USER jovyan
ENV NB_UID 1000
# WARNING: the primary GID of 'jovyan' MUST be 0!
# this allows any UID to be used as all important folders are owned by GID 0.
# some Kubernetes environments run containers as a random UID (e.g. OpenShift).
# note, having GID 0 (root) does NOT give you root permissions, so this is not a security issue.
ENV NB_GID 0
ENV NB_PREFIX /
ENV HOME /home/$NB_USER
ENV SHELL /bin/bash

# the GID of the 'users' group
ENV USERS_GID 100

# we copy the contents of $HOME_TMP to $HOME on startup
# this is to work around the fact that a PVC will be mounted to $HOME
# but we still want to have some default files in $HOME
# see `s6/cont-init.d/01-copy-tmp-home`
ENV HOME_TMP /tmp_home/$NB_USER

# s3 only gives 5 seconds by default, which is too small for slow PVC storage backends
# s6-overlay only gives 5 seconds by default, which is too small for slow PVC storage backends
# when running `/etc/cont-inid.d/01-copy-tmp-home` (note, this is in milliseconds)
ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME 300000

# s6-overlay does not fail by default if the `/etc/cont-init.d/` or `/etc/services.d/` scripts fail
# this is not the desired behavior, so we set it to fail
ENV S6_BEHAVIOUR_IF_STAGE2_FAILS 2

# args - software versions
ARG KUBECTL_VERSION=v1.27.14
ARG S6_VERSION=v3.1.6.2
ARG S6_VERSION=v3.2.0.0

# set shell to bash
SHELL ["/bin/bash", "-c"]
Expand Down Expand Up @@ -76,6 +88,10 @@ RUN case "${TARGETARCH}" in \
/tmp/s6-overlay-${S6_ARCH}.tar.xz \
/tmp/s6-overlay-${S6_ARCH}.tar.xz.sha256

# fix permissions of '/run' folder for s6
# https://github.com/just-containers/s6-overlay/blob/v3.2.0.0/layout/rootfs-overlay/package/admin/s6-overlay-%40VERSION%40/libexec/preinit#L86
RUN chmod 0775 /run

# install - kubectl
RUN curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl" -o /usr/local/bin/kubectl \
&& curl -fsSL "https://dl.k8s.io/${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl.sha256" -o /tmp/kubectl.sha256 \
Expand All @@ -84,12 +100,22 @@ RUN curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${TARGETA
&& chmod +x /usr/local/bin/kubectl

# create user and set required ownership
RUN useradd -M -s /bin/bash -N -u ${NB_UID} ${NB_USER} \
&& mkdir -p ${HOME} \
&& mkdir -p ${HOME_TMP} \
&& chown -R ${NB_USER}:users ${HOME} \
&& chown -R ${NB_USER}:users ${HOME_TMP} \
&& chown -R ${NB_USER}:users /usr/local/bin
RUN useradd -M -N \
--shell /bin/bash \
--home ${HOME} \
--uid ${NB_UID} \
--gid ${NB_GID} \
--groups ${USERS_GID} \
${NB_USER} \
&& mkdir -pv ${HOME} \
&& mkdir -pv ${HOME_TMP} \
# in the interest of backwards compatibility we have the 'users' group owns the home directory
# we also set the SGID bit so that new files and directories are created with the 'users' group
&& chmod 2775 ${HOME} \
&& chmod 2775 ${HOME_TMP} \
&& chown -R ${NB_USER}:${USERS_GID} ${HOME} \
&& chown -R ${NB_USER}:${USERS_GID} ${HOME_TMP} \
&& chown -R ${NB_USER}:${NB_GID} /usr/local/bin

# set locale configs
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
Expand All @@ -99,7 +125,7 @@ ENV LANGUAGE en_US.UTF-8
ENV LC_ALL en_US.UTF-8

# s6 - copy scripts
COPY --chown=${NB_USER}:users --chmod=755 s6/ /etc
COPY --chown=${NB_USER}:${NB_GID} --chmod=755 s6/ /etc

USER $NB_UID

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
#!/command/with-contenv bash

# setting umask to 0002 makes copied files/folders writable by group
# this is needed to run the container as an arbitrary UID
umask 0002

# the home directory is usually a PVC
# we need to copy the contents of $HOME_TMP that we populated during the build
# NOTE: -n prevents overwriting existing files
Expand Down
13 changes: 8 additions & 5 deletions components/example-notebook-servers/codeserver-python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@ ARG PYTHON_VERSION=3.11.9
# setup environment for conda
ENV CONDA_DIR /opt/conda
ENV PATH "${CONDA_DIR}/bin:${PATH}"
RUN mkdir -p ${CONDA_DIR} \
RUN mkdir -pv ${CONDA_DIR} \
&& chmod 2775 ${CONDA_DIR} \
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> ${HOME}/.bashrc \
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> /etc/profile \
&& echo "conda activate base" >> ${HOME}/.bashrc \
&& echo "conda activate base" >> /etc/profile \
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
&& chown -R ${NB_USER}:${NB_GID} ${CONDA_DIR} \
&& chown -R ${NB_USER}:${USERS_GID} ${HOME}

USER $NB_UID

Expand All @@ -51,7 +52,7 @@ RUN case "${TARGETARCH}" in \
&& conda clean -a -f -y

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

Expand All @@ -63,4 +64,6 @@ RUN code-server --install-extension "ms-python.python@${CODESERVER_PYTHON_VERSIO
# NOTE: the contents of $HOME_TMP are copied to $HOME at runtime
# this is a workaround because a PVC will be mounted at $HOME
# and the contents of $HOME will be hidden
RUN cp -r -T "${HOME}" "${HOME_TMP}"
RUN cp -p -r -T "${HOME}" "${HOME_TMP}" \
# give group same access as user (needed for OpenShift)
&& chmod -R g=u "${HOME_TMP}"
2 changes: 1 addition & 1 deletion components/example-notebook-servers/codeserver/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ RUN curl -fsSL "https://github.com/coder/code-server/releases/download/${CODESER
&& rm -f /tmp/code-server.deb

# s6 - copy scripts
COPY --chown=${NB_USER}:users --chmod=755 s6/ /etc
COPY --chown=${NB_USER}:${NB_GID} --chmod=755 s6/ /etc

USER $NB_UID

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
#!/command/with-contenv bash

cd "${HOME}"
echo "INFO: starting code-server..."
exec 2>&1
exec /usr/bin/code-server \
--bind-addr 0.0.0.0:8888 \
--disable-telemetry \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ RUN mamba install -y -q \
&& mamba clean -a -f -y

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp/requirements.txt
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ RUN python3 -m pip install --quiet --no-cache-dir --index-url https://download.p
torchvision==${TORCHVISION_VERSION}

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ RUN mamba install -y -q \
&& mamba clean -a -f -y

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp/requirements.txt
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,6 @@ RUN python3 -m pip install --quiet --no-cache-dir --index-url https://download.p
torchvision==${TORCHVISION_VERSION}

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,6 @@ RUN mamba install -y -q \
&& mamba clean -a -f -y

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ RUN mamba install -y -q \
&& mamba clean -a -f -y

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp/requirements.txt
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,6 @@ RUN ln -s ${TENSORRT_LIBS}/libnvinfer.so.${TENSORRT_LIBS_VERSION%%.*} ${TENSORRT
ENV LD_LIBRARY_PATH ${LD_LIBRARY_PATH}/nvidia/cudnn/lib:${TENSORRT_LIBS}

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ RUN mamba install -y -q \
&& mamba clean -a -f -y

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp/requirements.txt
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,6 @@ RUN python3 -m pip install --quiet --no-cache-dir \
tensorflow==${TENSORFLOW_VERSION}

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
19 changes: 11 additions & 8 deletions components/example-notebook-servers/jupyter/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,19 @@ RUN export DEBIAN_FRONTEND=noninteractive \
# setup environment for conda
ENV CONDA_DIR /opt/conda
ENV PATH "${CONDA_DIR}/bin:${PATH}"
RUN mkdir -p ${CONDA_DIR} \
RUN mkdir -pv ${CONDA_DIR} \
&& chmod 2775 ${CONDA_DIR} \
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> ${HOME}/.bashrc \
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> /etc/profile \
&& echo "conda activate base" >> ${HOME}/.bashrc \
&& echo "conda activate base" >> /etc/profile \
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
&& chown -R ${NB_USER}:${NB_GID} ${CONDA_DIR} \
&& chown -R ${NB_USER}:${USERS_GID} ${HOME}

# create the SYSTEM_CONFIG_PATH for jupyter, and make it owned by NB_USER
# this is needed for jupyter to write `--level=system` configs
RUN mkdir -p /usr/local/etc/jupyter \
&& chown -R ${NB_USER}:users /usr/local/etc/jupyter
RUN mkdir -pv /usr/local/etc/jupyter \
&& chown -R ${NB_USER}:${NB_GID} /usr/local/etc/jupyter

# switch to NB_UID for installs
USER $NB_UID
Expand Down Expand Up @@ -77,12 +78,12 @@ RUN echo "jupyterlab ==${JUPYTERLAB_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned
&& conda clean -a -f -y

# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

# s6 - copy scripts
COPY --chown=${NB_USER}:users --chmod=755 s6/ /etc
COPY --chown=${NB_USER}:${NB_GID} --chmod=755 s6/ /etc

# configure - jupyter
# NOTE: we use `--level=system` to write these configs at `/usr/local/etc/jupyter` because it defaults
Expand All @@ -94,6 +95,8 @@ RUN jupyter labextension disable --level=system "@jupyterlab/apputils-extension:
# NOTE: the contents of $HOME_TMP are copied to $HOME at runtime
# this is a workaround because a PVC will be mounted at $HOME
# and the contents of $HOME will be hidden
RUN cp -r -T "${HOME}" "${HOME_TMP}"
RUN cp -p -r -T "${HOME}" "${HOME_TMP}" \
# give group same access as user (needed for OpenShift)
&& chmod -R g=u "${HOME_TMP}"

EXPOSE 8888
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
export JUPYTER_RUNTIME_DIR="/tmp/jupyter_runtime"

cd "${HOME}"
echo "INFO: starting jupyter..."
exec 2>&1
exec /opt/conda/bin/jupyter lab \
--notebook-dir="${HOME}" \
--ip=0.0.0.0 \
Expand Down
51 changes: 38 additions & 13 deletions components/example-notebook-servers/rstudio/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,27 @@ RUN apt-get -yq update \
# setup environment for conda
ENV CONDA_DIR /opt/conda
ENV PATH "${CONDA_DIR}/bin:${PATH}"
RUN mkdir -p ${CONDA_DIR} \
RUN mkdir -pv ${CONDA_DIR} \
&& chmod 2775 ${CONDA_DIR} \
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> ${HOME}/.bashrc \
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> /etc/profile \
&& echo "conda activate base" >> ${HOME}/.bashrc \
&& echo "conda activate base" >> /etc/profile \
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
&& chown -R ${NB_USER}:${NB_GID} ${CONDA_DIR} \
&& chown -R ${NB_USER}:${USERS_GID} ${HOME}

# setup environment for R
ENV R_HOME ${CONDA_DIR}/lib/R

# rstudio logs to a file by default, but we need to see them in the container logs
ENV RS_LOG_LEVEL WARN
ENV RS_LOGGER_TYPE stderr

# rstudio must run as a named user.
# we change the UID of 'jovyan' at runtime to match the UID of the container.
# this is important for environments like OpenShift where the container has a random UID.
RUN chmod g+w /etc/passwd

USER $NB_UID

# install - conda, pip, python
Expand Down Expand Up @@ -100,36 +110,51 @@ RUN case "${TARGETARCH}" in \
*) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \
esac \
&& curl -fsSL "${RSTUDIO_DEB_URL}" -o /tmp/rstudio-server.deb \
# add rstudio public code-signing keys
# validate the build signature
&& export GNUPGHOME="$(mktemp -d)" \
&& gpg --keyserver keys.gnupg.net --keyserver pgp.surfnet.nl --recv-keys 3F32EE77E331692F \
&& gpg --keyserver keys.openpgp.org --recv-keys 51C0B5BB19F92D60 \
# validate the build signature
&& dpkg-sig --verify /tmp/rstudio-server.deb \
&& rm -rf "${GNUPGHOME}" \
# install rstudio-server
&& dpkg -i /tmp/rstudio-server.deb \
&& rm -f /tmp/rstudio-server.deb \
# the default DB path '/var/lib/rstudio-server' causes permission issues when running as a random UID (like in OpenShift)
&& echo "provider=sqlite" > /etc/rstudio/database.conf \
&& echo "directory=/tmp/rstudio_db" >> /etc/rstudio/database.conf \
# use advisory file-locks to improve PVC support
&& echo "lock-type=advisory" > /etc/rstudio/file-locks \
# allow kubeflow to display rstudio in an iframe
&& echo "www-frame-origin=same" >> /etc/rstudio/rserver.conf \
# allows the non-root NB_USER to run rstudio
&& chown -R ${NB_USER}:users /etc/rstudio \
&& chown -R ${NB_USER}:users /run/rstudio-server* \
&& chown -R ${NB_USER}:users /usr/lib/rstudio-server \
&& chown -R ${NB_USER}:users /var/lib/rstudio-server
&& chown -R ${NB_USER}:${NB_GID} ${R_HOME} \
&& chown -R ${NB_USER}:${NB_GID} /etc/rstudio \
&& chown -R ${NB_USER}:${NB_GID} /run/rstudio-server* \
&& chown -R ${NB_USER}:${NB_GID} /usr/lib/rstudio-server \
&& chown -R ${NB_USER}:${NB_GID} /var/lib/rstudio-server \
&& chown -R ${NB_USER}:${NB_GID} /var/log/rstudio \
# give group same access as user (needed for OpenShift)
&& chmod -R g=u ${R_HOME} \
&& chmod -R g=u /etc/rstudio \
&& chmod -R g=u /run/rstudio-server* \
&& chmod -R g=u /usr/lib/rstudio-server \
&& chmod -R g=u /var/lib/rstudio-server \
&& chmod -R g=u /var/log/rstudio

# tell rstudio to use conda python by setting `RETICULATE_PYTHON` with `--rsession-path=/opt/rsession.sh`
COPY --chown=${NB_USER}:users --chmod=755 rsession.sh /opt
COPY --chown=${NB_USER}:${NB_GID} --chmod=755 rsession.sh /opt
RUN chmod +x /opt/rsession.sh

# s6 - copy scripts
COPY --chown=${NB_USER}:users --chmod=755 s6/ /etc
COPY --chown=${NB_USER}:${NB_GID} --chmod=755 s6/ /etc

# s6 - 01-copy-tmp-home
# NOTE: the contents of $HOME_TMP are copied to $HOME at runtime
# this is a workaround because a PVC will be mounted at $HOME
# and the contents of $HOME will be hidden
RUN cp -r -T "${HOME}" "${HOME_TMP}" \
&& chown -R ${NB_USER}:users ${HOME_TMP}
RUN cp -p -r -T "${HOME}" "${HOME_TMP}" \
# give group same access as user (needed for OpenShift)
&& chmod -R g=u "${HOME_TMP}"

USER $NB_UID

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/command/with-contenv bash

# rstudio terminal cant see environment variables set by the container runtime
# (which breaks kubectl, to fix this we store the KUBERNETES_* env vars in Renviron.site)
env | grep KUBERNETES_ >> ${R_HOME}/etc/Renviron.site

This file was deleted.

Loading

0 comments on commit 32ff054

Please sign in to comment.