Skip to content

Commit

Permalink
add use systemd cgroup e2e
Browse files Browse the repository at this point in the history
Signed-off-by: lengrongfu <lenronfu@gmail.com>
Signed-off-by: rongfu.leng <lenronfu@gmail.com>
  • Loading branch information
lengrongfu committed Oct 2, 2024
1 parent 06dfa0c commit 095131a
Show file tree
Hide file tree
Showing 8 changed files with 154 additions and 8 deletions.
57 changes: 50 additions & 7 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ jobs:
fail-fast: false
matrix:
os: [windows-2022, windows-2019]
cgroup_driver: [cgroupfs]

defaults:
run:
Expand Down Expand Up @@ -345,12 +346,14 @@ jobs:
- name: CRI Integration Test
env:
TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
run: |
make cri-integration
- name: cri-tools critest
env:
CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml
CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
shell: powershell
run: |
Start-Process -FilePath containerd.exe -NoNewWindow -RedirectStandardError true -PassThru
Expand All @@ -367,7 +370,7 @@ jobs:
- uses: actions/upload-artifact@v4
if: always()
with:
name: TestResults ${{ matrix.os }}
name: TestResults ${{ matrix.os }} ${{ matrix.cgroup_driver }}
path: |
${{github.workspace}}/*-junit.xml
${{github.workspace}}/*-gotest.json
Expand All @@ -388,6 +391,7 @@ jobs:
os: [ubuntu-22.04, ubuntu-24.04, actuated-arm64-4cpu-16gb]
exclude:
- os: ${{ github.repository != 'containerd/containerd' && 'actuated-arm64-4cpu-16gb' }}
cgroup_driver: [cgroupfs, systemd]

env:
GOTEST: gotestsum --
Expand Down Expand Up @@ -476,7 +480,15 @@ jobs:
- name: CRI Integration Test
env:
TEST_RUNTIME: ${{ matrix.runtime }}
CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
RUNC_FLAVOR: ${{ matrix.runc }}
run: |
cat /sys/fs/cgroup/cgroup.controllers
systemctl status
[ "${RUNC_FLAVOR}" == "crun" ] && {
export XDG_RUNTIME_DIR=/run/user/$(id -u)
}
runc --version
CONTAINERD_RUNTIME=$TEST_RUNTIME make cri-integration
- name: Fix up for actuated
Expand All @@ -488,6 +500,7 @@ jobs:
- name: cri-tools critest
env:
TEST_RUNTIME: ${{ matrix.runtime }}
CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
run: |
env
sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report"
Expand All @@ -510,7 +523,7 @@ jobs:
- uses: actions/upload-artifact@v4
if: always()
with:
name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} ${{ matrix.os }}
name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} ${{ matrix.os }} ${{ matrix.cgroup_driver }}
path: |
*-junit.xml
*-gotest.json
Expand All @@ -534,9 +547,10 @@ jobs:
# https://github.com/containerd/containerd/pull/10297
- almalinux/8
- rockylinux/9@4.0.0
cgroup_driver: [cgroupfs, systemd]
env:
BOX: ${{ matrix.box }}

CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
steps:
- name: Show the host info
run: |
Expand Down Expand Up @@ -586,9 +600,9 @@ jobs:
- name: test-integration
run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
- name: test-cri-integration
run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration
run: sudo BOX=$BOX CGROUP_DRIVER=$CGROUP_DRIVER vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration
- name: test-cri
run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
run: sudo BOX=$BOX CGROUP_DRIVER=$CGROUP_DRIVER vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri

tests-cri-in-userns:
name: "CRI-in-UserNS"
Expand All @@ -597,6 +611,11 @@ jobs:
timeout-minutes: 40
needs: [project, linters, protos, man]

strategy:
fail-fast: false
matrix:
cgroup_driver: [cgroupfs, systemd]

steps:
- uses: actions/checkout@v4
- name: Set up cgroup v2 delegation
Expand All @@ -608,11 +627,35 @@ jobs:
EOF
sudo systemctl daemon-reload
- name: Build cri-in-userns image
run: podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test .
env:
CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
run: |
if [ "$CGROUP_DRIVER" = "systemd" ];then
podman build --target cri-in-userns-systemd -t cri-in-userns-systemd -f ./contrib/Dockerfile.test .
else
podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test .
fi
- name: Run cri-in-userns image
env:
CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
# Rootless Podman is used for testing CRI-in-UserNS
# (We could use rootless Docker or rootless nerdctl, but we are using Podman here because it is preinstalled)
run: podman run --rm --privileged cri-in-userns
run: |
if [ "$CGROUP_DRIVER" = "systemd" ];then
set +e
touch ./critest_exit_code.txt
podman run --rm --privileged --group-add keep-groups -v ./critest_exit_code.txt:/tmp/critest_exit_code.txt cri-in-userns-systemd
exit_code=`cat ./critest_exit_code.txt`
echo "exit_code:"$exit_code
if [ "$exit_code" -gt 0 ]; then
exit 1
else
exit 0
fi
else
podman run --rm --privileged cri-in-userns
fi
tests-mac-os:
name: MacOS unit tests
Expand Down
2 changes: 2 additions & 0 deletions Vagrantfile
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ EOF
'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
'GOTESTSUM_JSONFILE': ENV['GOTESTSUM_JSONFILE'],
'GITHUB_WORKSPACE': '',
'CGROUP_DRIVER': ENV['CGROUP_DRIVER'],
}
sh.inline = <<~SHELL
#!/usr/bin/env bash
Expand Down Expand Up @@ -299,6 +300,7 @@ EOF
sh.env = {
'GOTEST': ENV['GOTEST'] || "go test",
'REPORT_DIR': ENV['REPORT_DIR'],
'CGROUP_DRIVER': ENV['CGROUP_DRIVER'],
}
sh.inline = <<~SHELL
#!/usr/bin/env bash
Expand Down
24 changes: 24 additions & 0 deletions contrib/Dockerfile.test
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@
# "cri-in-userns": for running critest with "CRI-in-UserNS" mode; needs Rootless Docker/Podman/nerdctl:
# docker build -t containerd-test -f Dockerfile.test --target cri-in-userns ../
# docker run --privileged containerd-test
#
# "cri-in-userns-systemd": for running critest with "CRI-in-UserNS-Systemd" mode; needs Rootless Docker/Podman/nerdctl:
# docker build -t containerd-test -f Dockerfile.test --target cri-in-userns-systemd ../
# touch ./critest_exit_code.txt
# docker run --privileged --group-add keep-groups -v ./critest_exit_code.txt:/tmp/critest_exit_code.txt containerd-test
# ------------------------------------------------------------------------------

ARG GOLANG_VERSION=1.23.1
Expand Down Expand Up @@ -111,6 +116,7 @@ CMD ["script/critest.sh", "/tmp"]
# Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/
# (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves)
FROM critest AS cri-in-userns
ENV IS_SYSTEMD_CGROUP=false
COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml
COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker-entrypoint.sh
ENTRYPOINT ["/docker-entrypoint.sh"]
Expand All @@ -119,6 +125,24 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
# Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:<nil>} stream:stdout log:Effective uid: 1000) }]
CMD ["critest", "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"]

# cri-in-userns-systemd stage is for testing "CRI-in-UserNS-Systemd", which should be used in conjunction with
# "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless
# This feature is mostly expected to be used for `kind` and `minikube`.
#
# Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/
# (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves)
FROM critest AS cri-in-userns-systemd
ENV IS_SYSTEMD_CGROUP=true
RUN apt-get update && apt-get install -y systemd sysvinit-core
COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml
COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh /docker-entrypoint.sh
COPY contrib/Dockerfile.test.d/critest.sh /critest.sh
# Skip "runtime should support unsafe sysctls": `container init caused: write sysctl key fs.mqueue.msg_max: open /proc/sys/fs/mqueue/msg_max: permission denied`
# Skip "runtime should support safe sysctls": `container init caused: write sysctl key kernel.shm_rmid_forced: open /proc/sys/kernel/shm_rmid_forced: permission denied`
# Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:<nil>} stream:stdout log:Effective uid: 1000) }]
ENTRYPOINT ["/critest.sh","start"]


# Install proto3
FROM golang AS proto3
ARG DESTDIR=/build
Expand Down
14 changes: 13 additions & 1 deletion contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,23 @@ xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
> /sys/fs/cgroup/cgroup.subtree_control

if [ ! -z "$IS_SYSTEMD_CGROUP" ] && [ "$IS_SYSTEMD_CGROUP" = true ];then
cat >> /etc/containerd/config.toml <<EOF
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
EOF
fi

set -x
echo >&2 "Running containerd in background"
containerd &

echo >&2 "Waiting for containerd"
until ctr plugins list; do sleep 3; done

exec "$@"
if [ ! -z "$IS_SYSTEMD_CGROUP" ] && [ "$IS_SYSTEMD_CGROUP" = true ];then
critest "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"
/bin/bash /critest.sh exit
else
exec "$@"
fi
47 changes: 47 additions & 0 deletions contrib/Dockerfile.test.d/critest.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/usr/bin/env bash

cat > /etc/systemd/system/critest.service << EOF
[Unit]
Description=critest script
[Service]
Type=simple
Environment="IS_SYSTEMD_CGROUP=true"
RemainAfterExit=yes
ExecStart=/bin/bash /docker-entrypoint.sh
StandardOutput=/dev/stdout
StandardError=/dev/stderr
[Install]
WantedBy=default.target
EOF


function echo_exit_code() {
sleep 30
log_str=`systemctl status critest.service|grep "SUCCESS!"`
if [ -z "$log_str" ]; then
echo 1 > /tmp/critest_exit_code.txt
/bin/systemctl poweroff
fi
failed_count=$(echo "$log_str" | awk '{for(i=1;i<=NF;i++) if($i=="Failed") {print $(i-1); exit}}')
if [ "$failed_count" -gt 0 ]; then
echo 1 > /tmp/critest_exit_code.txt
else
echo 0 > /tmp/critest_exit_code.txt
fi
/bin/systemctl poweroff
}

function start(){
systemctl enable critest.service
journalctl -f &
exec /lib/systemd/systemd
}

case $1 in
start)
start
;;
exit)
echo_exit_code
;;
esac
3 changes: 3 additions & 0 deletions integration/nri_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,7 @@ func TestNriLinuxCpusetAdjustmentUpdate(t *testing.T) {
)

t.Log("Test that NRI plugins can update linux cpusets of existing containers.")
t.Logf("availableCpuset values is %v", availableCpuset)

var (
out = t.TempDir()
Expand All @@ -423,10 +424,12 @@ func TestNriLinuxCpusetAdjustmentUpdate(t *testing.T) {
Type: "bind",
Options: []string{"bind"},
})
t.Logf("ctr0 availableCpuset values is %v", availableCpuset)
adjust.SetLinuxCPUSetCPUs(availableCpuset[0])
} else {
update = []*api.ContainerUpdate{{}}
update[0].SetContainerId(ctr0)
t.Logf("ctr1 availableCpuset values is %v", availableCpuset)
update[0].SetLinuxCPUSetCPUs(availableCpuset[1])
}
return adjust, update, nil
Expand Down
8 changes: 8 additions & 0 deletions script/critest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,14 @@ runtime_type = "${TEST_RUNTIME}"
# This is safely ignored for kernel >= 5.19.
slow_chown = true
EOF

if [ ! -z "$CGROUP_DRIVER" ] && [ "$CGROUP_DRIVER" = "systemd" ];then
cat >> ${BDIR}/config.toml <<EOF
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
EOF
fi

ls /etc/cni/net.d

/usr/local/bin/containerd \
Expand Down
7 changes: 7 additions & 0 deletions script/test/utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,13 @@ version = 2
EOF
fi

if [ $IS_WINDOWS -eq 0 ] && [ ! -z "$CGROUP_DRIVER" ] && [ "$CGROUP_DRIVER" = "systemd" ];then
cat >> ${CONTAINERD_CONFIG_FILE} << EOF
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
EOF
fi

# CONTAINERD_TEST_SUFFIX is the suffix appended to the root/state directory used
# by test containerd.
CONTAINERD_TEST_SUFFIX=${CONTAINERD_TEST_SUFFIX:-"-test"}
Expand Down

0 comments on commit 095131a

Please sign in to comment.