diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efa92cdb4fba..c0e76f4dea25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -219,6 +219,7 @@ jobs: fail-fast: false matrix: os: [windows-2022, windows-2019] + cgroup_driver: [cgroupfs] defaults: run: @@ -345,12 +346,14 @@ jobs: - name: CRI Integration Test env: TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml + CGROUP_DRIVER: ${{ matrix.cgroup_driver }} run: | make cri-integration - name: cri-tools critest env: CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml + CGROUP_DRIVER: ${{ matrix.cgroup_driver }} shell: powershell run: | Start-Process -FilePath containerd.exe -NoNewWindow -RedirectStandardError true -PassThru @@ -367,7 +370,7 @@ jobs: - uses: actions/upload-artifact@v4 if: always() with: - name: TestResults ${{ matrix.os }} + name: TestResults ${{ matrix.os }} ${{ matrix.cgroup_driver }} path: | ${{github.workspace}}/*-junit.xml ${{github.workspace}}/*-gotest.json @@ -388,6 +391,7 @@ jobs: os: [ubuntu-22.04, ubuntu-24.04, actuated-arm64-4cpu-16gb] exclude: - os: ${{ github.repository != 'containerd/containerd' && 'actuated-arm64-4cpu-16gb' }} + cgroup_driver: [cgroupfs, systemd] env: GOTEST: gotestsum -- @@ -476,7 +480,15 @@ jobs: - name: CRI Integration Test env: TEST_RUNTIME: ${{ matrix.runtime }} + CGROUP_DRIVER: ${{ matrix.cgroup_driver }} + RUNC_FLAVOR: ${{ matrix.runc }} run: | + cat /sys/fs/cgroup/cgroup.controllers + systemctl status + [ "${RUNC_FLAVOR}" == "crun" ] && { + export XDG_RUNTIME_DIR=/run/user/$(id -u) + } + runc --version CONTAINERD_RUNTIME=$TEST_RUNTIME make cri-integration - name: Fix up for actuated @@ -488,6 +500,7 @@ jobs: - name: cri-tools critest env: TEST_RUNTIME: ${{ matrix.runtime }} + CGROUP_DRIVER: ${{ matrix.cgroup_driver }} run: | env sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report" @@ -510,7 +523,7 @@ jobs: - uses: actions/upload-artifact@v4 if: always() with: - name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} ${{ matrix.os }} + name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} ${{ matrix.os }} ${{ matrix.cgroup_driver }} path: | *-junit.xml *-gotest.json @@ -534,9 +547,10 @@ jobs: # https://github.com/containerd/containerd/pull/10297 - almalinux/8 - rockylinux/9@4.0.0 + cgroup_driver: [cgroupfs, systemd] env: BOX: ${{ matrix.box }} - + CGROUP_DRIVER: ${{ matrix.cgroup_driver }} steps: - name: Show the host info run: | @@ -586,9 +600,9 @@ jobs: - name: test-integration run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration - name: test-cri-integration - run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration + run: sudo BOX=$BOX CGROUP_DRIVER=$CGROUP_DRIVER vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration - name: test-cri - run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri + run: sudo BOX=$BOX CGROUP_DRIVER=$CGROUP_DRIVER vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri tests-cri-in-userns: name: "CRI-in-UserNS" @@ -597,6 +611,11 @@ jobs: timeout-minutes: 40 needs: [project, linters, protos, man] + strategy: + fail-fast: false + matrix: + cgroup_driver: [cgroupfs, systemd] + steps: - uses: actions/checkout@v4 - name: Set up cgroup v2 delegation @@ -608,11 +627,35 @@ jobs: EOF sudo systemctl daemon-reload - name: Build cri-in-userns image - run: podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test . + env: + CGROUP_DRIVER: ${{ matrix.cgroup_driver }} + run: | + if [ "$CGROUP_DRIVER" = "systemd" ];then + podman build --target cri-in-userns-systemd -t cri-in-userns-systemd -f ./contrib/Dockerfile.test . + else + podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test . + fi + - name: Run cri-in-userns image + env: + CGROUP_DRIVER: ${{ matrix.cgroup_driver }} # Rootless Podman is used for testing CRI-in-UserNS # (We could use rootless Docker or rootless nerdctl, but we are using Podman here because it is preinstalled) - run: podman run --rm --privileged cri-in-userns + run: | + if [ "$CGROUP_DRIVER" = "systemd" ];then + set +e + touch ./critest_exit_code.txt + podman run --rm --privileged --group-add keep-groups -v ./critest_exit_code.txt:/tmp/critest_exit_code.txt cri-in-userns-systemd + exit_code=`cat ./critest_exit_code.txt` + echo "exit_code:"$exit_code + if [ "$exit_code" -gt 0 ]; then + exit 1 + else + exit 0 + fi + else + podman run --rm --privileged cri-in-userns + fi tests-mac-os: name: MacOS unit tests diff --git a/Vagrantfile b/Vagrantfile index 5e0b38bb40d7..42bad1a4dc87 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -272,6 +272,7 @@ EOF 'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'], 'GOTESTSUM_JSONFILE': ENV['GOTESTSUM_JSONFILE'], 'GITHUB_WORKSPACE': '', + 'CGROUP_DRIVER': ENV['CGROUP_DRIVER'], } sh.inline = <<~SHELL #!/usr/bin/env bash @@ -299,6 +300,7 @@ EOF sh.env = { 'GOTEST': ENV['GOTEST'] || "go test", 'REPORT_DIR': ENV['REPORT_DIR'], + 'CGROUP_DRIVER': ENV['CGROUP_DRIVER'], } sh.inline = <<~SHELL #!/usr/bin/env bash diff --git a/contrib/Dockerfile.test b/contrib/Dockerfile.test index 1e7688684d34..324a94626e57 100644 --- a/contrib/Dockerfile.test +++ b/contrib/Dockerfile.test @@ -27,6 +27,11 @@ # "cri-in-userns": for running critest with "CRI-in-UserNS" mode; needs Rootless Docker/Podman/nerdctl: # docker build -t containerd-test -f Dockerfile.test --target cri-in-userns ../ # docker run --privileged containerd-test +# +# "cri-in-userns-systemd": for running critest with "CRI-in-UserNS-Systemd" mode; needs Rootless Docker/Podman/nerdctl: +# docker build -t containerd-test -f Dockerfile.test --target cri-in-userns-systemd ../ +# touch ./critest_exit_code.txt +# docker run --privileged --group-add keep-groups -v ./critest_exit_code.txt:/tmp/critest_exit_code.txt containerd-test # ------------------------------------------------------------------------------ ARG GOLANG_VERSION=1.23.1 @@ -111,6 +116,7 @@ CMD ["script/critest.sh", "/tmp"] # Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/ # (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves) FROM critest AS cri-in-userns +ENV IS_SYSTEMD_CGROUP=false COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker-entrypoint.sh ENTRYPOINT ["/docker-entrypoint.sh"] @@ -119,6 +125,24 @@ ENTRYPOINT ["/docker-entrypoint.sh"] # Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:} stream:stdout log:Effective uid: 1000) }] CMD ["critest", "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"] +# cri-in-userns-systemd stage is for testing "CRI-in-UserNS-Systemd", which should be used in conjunction with +# "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless +# This feature is mostly expected to be used for `kind` and `minikube`. +# +# Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/ +# (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves) +FROM critest AS cri-in-userns-systemd +ENV IS_SYSTEMD_CGROUP=true +RUN apt-get update && apt-get install -y systemd sysvinit-core +COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml +COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker-entrypoint.sh +COPY contrib/Dockerfile.test.d/critest.sh /critest.sh +# Skip "runtime should support unsafe sysctls": `container init caused: write sysctl key fs.mqueue.msg_max: open /proc/sys/fs/mqueue/msg_max: permission denied` +# Skip "runtime should support safe sysctls": `container init caused: write sysctl key kernel.shm_rmid_forced: open /proc/sys/kernel/shm_rmid_forced: permission denied` +# Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:} stream:stdout log:Effective uid: 1000) }] +ENTRYPOINT ["/critest.sh","start"] + + # Install proto3 FROM golang AS proto3 ARG DESTDIR=/build diff --git a/contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh b/contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh index e7b5882bbde3..e9fe4cc6a5db 100755 --- a/contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh +++ b/contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh @@ -41,6 +41,13 @@ xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ > /sys/fs/cgroup/cgroup.subtree_control +if [ ! -z "$IS_SYSTEMD_CGROUP" ] && [ "$IS_SYSTEMD_CGROUP" = true ];then + cat >> /etc/containerd/config.toml <&2 "Running containerd in background" containerd & @@ -48,4 +55,9 @@ containerd & echo >&2 "Waiting for containerd" until ctr plugins list; do sleep 3; done -exec "$@" +if [ ! -z "$IS_SYSTEMD_CGROUP" ] && [ "$IS_SYSTEMD_CGROUP" = true ];then + critest "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false" + /bin/bash /critest.sh exit +else + exec "$@" +fi diff --git a/contrib/Dockerfile.test.d/critest.sh b/contrib/Dockerfile.test.d/critest.sh new file mode 100755 index 000000000000..d913dc60e1b8 --- /dev/null +++ b/contrib/Dockerfile.test.d/critest.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +cat > /etc/systemd/system/critest.service << EOF +[Unit] +Description=critest script +[Service] +Type=simple +Environment="IS_SYSTEMD_CGROUP=true" +RemainAfterExit=yes +ExecStart=/bin/bash /docker-entrypoint.sh +StandardOutput=/dev/stdout +StandardError=/dev/stderr +[Install] +WantedBy=default.target +EOF + + +function echo_exit_code() { + sleep 30 + log_str=`systemctl status critest.service|grep "SUCCESS!"` + if [ -z "$log_str" ]; then + echo 1 > /tmp/critest_exit_code.txt + /bin/systemctl poweroff + fi + failed_count=$(echo "$log_str" | awk '{for(i=1;i<=NF;i++) if($i=="Failed") {print $(i-1); exit}}') + if [ "$failed_count" -gt 0 ]; then + echo 1 > /tmp/critest_exit_code.txt + else + echo 0 > /tmp/critest_exit_code.txt + fi + /bin/systemctl poweroff +} + +function start(){ + systemctl enable critest.service + journalctl -f & + exec /lib/systemd/systemd +} + +case $1 in + start) + start + ;; + exit) + echo_exit_code + ;; +esac diff --git a/integration/nri_test.go b/integration/nri_test.go index ca04c37db643..b56d173bd90b 100644 --- a/integration/nri_test.go +++ b/integration/nri_test.go @@ -401,6 +401,7 @@ func TestNriLinuxCpusetAdjustmentUpdate(t *testing.T) { ) t.Log("Test that NRI plugins can update linux cpusets of existing containers.") + t.Logf("availableCpuset values is %v", availableCpuset) var ( out = t.TempDir() @@ -423,10 +424,12 @@ func TestNriLinuxCpusetAdjustmentUpdate(t *testing.T) { Type: "bind", Options: []string{"bind"}, }) + t.Logf("ctr0 availableCpuset values is %v", availableCpuset) adjust.SetLinuxCPUSetCPUs(availableCpuset[0]) } else { update = []*api.ContainerUpdate{{}} update[0].SetContainerId(ctr0) + t.Logf("ctr1 availableCpuset values is %v", availableCpuset) update[0].SetLinuxCPUSetCPUs(availableCpuset[1]) } return adjust, update, nil diff --git a/script/critest.sh b/script/critest.sh index 17bdf3b663ab..267d2082fdd0 100755 --- a/script/critest.sh +++ b/script/critest.sh @@ -62,6 +62,14 @@ runtime_type = "${TEST_RUNTIME}" # This is safely ignored for kernel >= 5.19. slow_chown = true EOF + +if [ ! -z "$CGROUP_DRIVER" ] && [ "$CGROUP_DRIVER" = "systemd" ];then + cat >> ${BDIR}/config.toml <> ${CONTAINERD_CONFIG_FILE} << EOF +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true +EOF +fi + # CONTAINERD_TEST_SUFFIX is the suffix appended to the root/state directory used # by test containerd. CONTAINERD_TEST_SUFFIX=${CONTAINERD_TEST_SUFFIX:-"-test"}