Merge pull request #4479 from giuseppe/cgroupv2-unified

server: support setting raw unified cgroupv2 settings
cri-o · Feb 5, 2021 · f39e3d0 · f39e3d0
2 parents e77f586 + cf1e612
commit f39e3d0
Show file tree

Hide file tree

Showing 14 changed files with 144 additions and 21 deletions.
diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md
@@ -262,6 +262,7 @@ The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.  Th
   "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod.
   "io.kubernetes.cri-o.Devices" for configuring devices for the pod.
   "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm.
+  "io.kubernetes.cri-o.UnifiedCgroup.$CTR_NAME" for configuring the cgroup v2 unified block for a container.
 
 ## CRIO.IMAGE TABLE
 The `crio.image` table contains settings pertaining to the management of OCI images.

diff --git a/go.mod b/go.mod
@@ -43,8 +43,8 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.2-0.20200206005212-79b036d80240
 	github.com/opencontainers/runc v1.0.0-rc92
-	github.com/opencontainers/runtime-spec v1.0.3-0.20200728170252-4d89ac9fbff6
-	github.com/opencontainers/runtime-tools v0.9.1-0.20200714183735-07406c5828aa
+	github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1
+	github.com/opencontainers/runtime-tools v0.9.1-0.20200121211434-d1bf3e66ff0a
 	github.com/opencontainers/selinux v1.8.0
 	github.com/pkg/errors v0.9.1
 	github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 // indirect
@@ -74,7 +74,7 @@ require (
 replace (
 	github.com/golang/protobuf => github.com/golang/protobuf v1.3.5
 	github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc90
-	github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20200710190001-3e4195d92445
+	github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1
 	// Pinning the syndtr/gocapability until https://github.com/opencontainers/runc/commit/6dfbe9b80707b1ca188255e8def15263348e0f9a
 	// is included in the runc release
 	github.com/syndtr/gocapability => github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2

diff --git a/go.sum b/go.sum
@@ -815,12 +815,12 @@ github.com/opencontainers/image-spec v1.0.2-0.20200206005212-79b036d80240 h1:SCj
 github.com/opencontainers/image-spec v1.0.2-0.20200206005212-79b036d80240/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/runc v1.0.0-rc90 h1:4+xo8mtWixbHoEm451+WJNUrq12o2/tDsyK9Vgc/NcA=
 github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runtime-spec v1.0.3-0.20200710190001-3e4195d92445 h1:y8cfsJRmn8g3VkM4IDpusKSgMUZEXhudm/BuYANLozE=
-github.com/opencontainers/runtime-spec v1.0.3-0.20200710190001-3e4195d92445/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1 h1:UAfI7SOCo1CNIu3RevW9B4HQyf7SY5aSzcSeoC7OPs0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
 github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
-github.com/opencontainers/runtime-tools v0.9.1-0.20200714183735-07406c5828aa h1:iyj+fFHVBn0xOalz9UChYzSU1K0HJ+d75b4YqShBRhI=
-github.com/opencontainers/runtime-tools v0.9.1-0.20200714183735-07406c5828aa/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
+github.com/opencontainers/runtime-tools v0.9.1-0.20200121211434-d1bf3e66ff0a h1:sf61qNtb7rsTAzYjwV7sqSXoksDyazZn2uHi8nj4GlM=
+github.com/opencontainers/runtime-tools v0.9.1-0.20200121211434-d1bf3e66ff0a/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
 github.com/opencontainers/selinux v1.3.0/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs=
 github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=
 github.com/opencontainers/selinux v1.5.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=

diff --git a/internal/oci/oci.go b/internal/oci/oci.go
@@ -203,6 +203,12 @@ func (r *Runtime) AllowDevicesAnnotation(handler string) (bool, error) {
 	return r.allowAnnotation(handler, annotations.DevicesAnnotation)
 }
 
+// AllowUnifiedCgroupAnnotation searches through the AllowedAnnotations for
+// the devices annotation, checking whether this runtime allows processing of "io.kubernetes.cri-o.UnifiedCgroup"
+func (r *Runtime) AllowUnifiedCgroupAnnotation(handler string) (bool, error) {
+	return r.allowAnnotation(handler, annotations.UnifiedCgroupAnnotation)
+}
+
 // AllowCPULoadBalancingAnnotation searches through the AllowedAnnotations for
 // the CPU load balancing annotation, checking whether this runtime allows processing of  "cpu-load-balancing.crio.io"
 func (r *Runtime) AllowCPULoadBalancingAnnotation(handler string) (bool, error) {

diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go
@@ -4,6 +4,9 @@ const (
 	// UsernsMode is the user namespace mode to use
 	UsernsModeAnnotation = "io.kubernetes.cri-o.userns-mode"
 
+	// UnifiedCgroupAnnotation specifies the unified configuration for cgroup v2
+	UnifiedCgroupAnnotation = "io.kubernetes.cri-o.UnifiedCgroup"
+
 	// SpoofedContainer indicates a container was spoofed in the runtime
 	SpoofedContainer = "io.kubernetes.cri-o.Spoofed"
 

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -163,6 +163,7 @@ type RuntimeHandler struct {
 	// "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod.
 	// "io.kubernetes.cri-o.Devices" for configuring devices for the pod.
 	// "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm.
+	// "io.kubernetes.cri-o.UnifiedCgroup.$CTR_NAME" for configuring the cgroup v2 unified block for a container.
 	AllowedAnnotations []string `toml:"allowed_annotations,omitempty"`
 }
 

diff --git a/pkg/config/template.go b/pkg/config/template.go
@@ -303,6 +303,7 @@ default_runtime = "{{ .DefaultRuntime }}"
 #   "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod.
 #   "io.kubernetes.cri-o.Devices" for configuring devices for the pod.
 #   "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm.
+#   "io.kubernetes.cri-o.UnifiedCgroup.$CTR_NAME" for configuring the cgroup v2 unified block for a container.
 
 {{ range $runtime_name, $runtime_handler := .Runtimes  }}
 [crio.runtime.runtimes.{{ $runtime_name }}]

diff --git a/pkg/container/container.go b/pkg/container/container.go
@@ -2,6 +2,7 @@ package container
 
 import (
 	"context"
+	b64 "encoding/base64"
 	"encoding/json"
 	"fmt"
 	"path/filepath"
@@ -16,13 +17,15 @@ import (
 	"github.com/cri-o/cri-o/internal/lib/sandbox"
 	oci "github.com/cri-o/cri-o/internal/oci"
 	"github.com/cri-o/cri-o/internal/storage"
+	crioann "github.com/cri-o/cri-o/pkg/annotations"
 	"github.com/cri-o/cri-o/server/cri/types"
 	"github.com/cri-o/cri-o/utils"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	kubeletTypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
 
 // Container is the main public container interface
@@ -89,6 +92,9 @@ type Container interface {
 
 	// SpecAddDevices adds devices from the server config, and container CRI config
 	SpecAddDevices([]device.Device, []device.Device, bool) error
+
+	// AddUnifiedResourcesFromAnnotations adds the cgroup-v2 resources specified in the io.kubernetes.cri-o.UnifiedCgroup annotation
+	AddUnifiedResourcesFromAnnotations(annotationsMap map[string]string) error
 }
 
 // container is the hidden default type behind the Container interface
@@ -429,3 +435,47 @@ func (c *container) SelinuxLabel(sboxLabel string) ([]string, error) {
 	}
 	return ret, nil
 }
+
+// AddUnifiedResourcesFromAnnotations adds the cgroup-v2 resources specified in the io.kubernetes.cri-o.UnifiedCgroup annotation
+func (c *container) AddUnifiedResourcesFromAnnotations(annotationsMap map[string]string) error {
+	if c.config == nil || c.config.Labels == nil {
+		return nil
+	}
+	containerName := c.config.Labels[kubeletTypes.KubernetesContainerNameLabel]
+	if containerName == "" {
+		return nil
+	}
+
+	annotationKey := fmt.Sprintf("%s.%s", crioann.UnifiedCgroupAnnotation, containerName)
+	annotation := annotationsMap[annotationKey]
+	if annotation == "" {
+		return nil
+	}
+
+	if c.spec.Config.Linux == nil {
+		c.spec.Config.Linux = &rspec.Linux{}
+	}
+	if c.spec.Config.Linux.Resources == nil {
+		c.spec.Config.Linux.Resources = &rspec.LinuxResources{}
+	}
+	if c.spec.Config.Linux.Resources.Unified == nil {
+		c.spec.Config.Linux.Resources.Unified = make(map[string]string)
+	}
+	for _, r := range strings.Split(annotation, ";") {
+		parts := strings.SplitN(r, "=", 2)
+		if len(parts) != 2 {
+			return fmt.Errorf("invalid annotation %q", crioann.UnifiedCgroupAnnotation)
+		}
+		d, err := b64.StdEncoding.DecodeString(parts[1])
+		// if the value is not specified in base64, then use its raw value.
+		v := ""
+		if err == nil {
+			v = string(d)
+		} else {
+			v = parts[1]
+		}
+		c.spec.Config.Linux.Resources.Unified[parts[0]] = v
+	}
+
+	return nil
+}
diff --git a/pkg/container/container_test.go b/pkg/container/container_test.go
@@ -2,6 +2,7 @@ package container_test
 
 import (
 	"encoding/json"
+	"fmt"
 	"strconv"
 	"time"
 
@@ -11,10 +12,12 @@ import (
 	"github.com/cri-o/cri-o/internal/lib/sandbox"
 	oci "github.com/cri-o/cri-o/internal/oci"
 	"github.com/cri-o/cri-o/internal/storage"
+	crioann "github.com/cri-o/cri-o/pkg/annotations"
 	"github.com/cri-o/cri-o/server/cri/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	kubeletTypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
 
 var _ = t.Describe("Container", func() {
@@ -292,4 +295,53 @@ var _ = t.Describe("Container", func() {
 			Expect(err).To(BeNil())
 		})
 	})
+	t.Describe("AddUnifiedResourcesFromAnnotations", func() {
+		It("should add the limits", func() {
+			// Given
+			containerName := "foo"
+			config.Labels = map[string]string{
+				kubeletTypes.KubernetesContainerNameLabel: containerName,
+			}
+			annotationKey := fmt.Sprintf("%s.%s", crioann.UnifiedCgroupAnnotation, containerName)
+			annotationsMap := map[string]string{
+				annotationKey: "memory.max=1000000;memory.min=MTAwMDA=;memory.low=20000",
+			}
+
+			// When
+			Expect(sut.SetConfig(config, sboxConfig)).To(BeNil())
+			Expect(sut.AddUnifiedResourcesFromAnnotations(annotationsMap)).To(BeNil())
+
+			// Then
+			spec := sut.Spec()
+			Expect(spec).To(Not(BeNil()))
+			Expect(spec.Config.Linux.Resources.Unified["memory.max"]).To(Equal("1000000"))
+			Expect(spec.Config.Linux.Resources.Unified["memory.min"]).To(Equal("10000"))
+			Expect(spec.Config.Linux.Resources.Unified["memory.low"]).To(Equal("20000"))
+		})
+
+		It("should not add the limits for a different container", func() {
+			// Given
+			containerName := "foo"
+			config.Labels = map[string]string{
+				kubeletTypes.KubernetesContainerNameLabel: containerName,
+			}
+
+			differentContainerName := "bar"
+			annotationKey := fmt.Sprintf("%s.%s", crioann.UnifiedCgroupAnnotation, differentContainerName)
+			annotationsMap := map[string]string{
+				annotationKey: "memory.max=1000000;memory.min=MTAwMDA=;memory.low=20000",
+			}
+
+			// When
+			Expect(sut.SetConfig(config, sboxConfig)).To(BeNil())
+			Expect(sut.AddUnifiedResourcesFromAnnotations(annotationsMap)).To(BeNil())
+
+			// Then
+			spec := sut.Spec()
+			Expect(spec).To(Not(BeNil()))
+			Expect(spec.Config.Linux.Resources.Unified["memory.max"]).To(Equal(""))
+			Expect(spec.Config.Linux.Resources.Unified["memory.min"]).To(Equal(""))
+			Expect(spec.Config.Linux.Resources.Unified["memory.low"]).To(Equal(""))
+		})
+	})
 })
diff --git a/server/container_create_linux.go b/server/container_create_linux.go
@@ -417,6 +417,16 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai
 		}
 	}
 
+	allowUnifiedResources, err := s.Runtime().AllowUnifiedCgroupAnnotation(sb.RuntimeHandler())
+	if err != nil {
+		return nil, err
+	}
+	if allowUnifiedResources {
+		if err := ctr.AddUnifiedResourcesFromAnnotations(sb.Annotations()); err != nil {
+			return nil, err
+		}
+	}
+
 	// Join the namespace paths for the pod sandbox container.
 	if err := configureGeneratorGivenNamespacePaths(sb.NamespacePaths(), *specgen); err != nil {
 		return nil, errors.Wrap(err, "failed to configure namespaces in container create")

diff --git a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go
diff --git a/vendor/github.com/opencontainers/runtime-spec/specs-go/state.go b/vendor/github.com/opencontainers/runtime-spec/specs-go/state.go
diff --git a/vendor/github.com/opencontainers/runtime-tools/generate/generate.go b/vendor/github.com/opencontainers/runtime-tools/generate/generate.go
diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -670,10 +670,10 @@ github.com/opencontainers/runc/libcontainer/devices
 github.com/opencontainers/runc/libcontainer/system
 github.com/opencontainers/runc/libcontainer/user
 github.com/opencontainers/runc/libcontainer/utils
-# github.com/opencontainers/runtime-spec v1.0.3-0.20200728170252-4d89ac9fbff6 => github.com/opencontainers/runtime-spec v1.0.3-0.20200710190001-3e4195d92445
+# github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1 => github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1
 ## explicit
 github.com/opencontainers/runtime-spec/specs-go
-# github.com/opencontainers/runtime-tools v0.9.1-0.20200714183735-07406c5828aa
+# github.com/opencontainers/runtime-tools v0.9.1-0.20200121211434-d1bf3e66ff0a
 ## explicit
 github.com/opencontainers/runtime-tools/error
 github.com/opencontainers/runtime-tools/filepath
@@ -1354,7 +1354,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
 sigs.k8s.io/yaml
 # github.com/golang/protobuf => github.com/golang/protobuf v1.3.5
 # github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc90
-# github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20200710190001-3e4195d92445
+# github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1
 # github.com/syndtr/gocapability => github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2
 # google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24
 # google.golang.org/grpc => google.golang.org/grpc v1.27.0