Merge pull request kubevirt#6182 from jordigilh/poc_cnv_rt_locked_field

Enhance support for real time workloads
lyarwood · Oct 6, 2021 · d64731b · d64731b
2 parents e7d5439 + 307cf35
commit d64731b
Show file tree

Hide file tree

Showing 48 changed files with 1,069 additions and 96 deletions.
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -197,6 +197,7 @@ container_bundle(
         "$(container_prefix)/$(image_prefix)alpine-ext-kernel-boot-demo:$(container_tag)": "//containerimages:alpine-ext-kernel-boot-demo-container",
         # Customized container-disk images
         "$(container_prefix)/$(image_prefix)fedora-with-test-tooling-container-disk:$(container_tag)": "//containerimages:fedora-with-test-tooling",
+        "$(container_prefix)/$(image_prefix)fedora-realtime-container-disk:$(container_tag)": "//containerimages:fedora-realtime",
         # testing images
         "$(container_prefix)/$(image_prefix)disks-images-provider:$(container_tag)": "//images/disks-images-provider:disks-images-provider-image",
         "$(container_prefix)/$(image_prefix)nfs-server:$(container_tag)": "//images/nfs-server:nfs-server-image",

diff --git a/Makefile b/Makefile
@@ -94,6 +94,9 @@ conformance:
 perftest: build-functests
 	hack/perftests.sh
 
+realtime-perftest: build-functests
+	hack/realtime-perftests.sh
+
 clean:
 	hack/dockerized "./hack/build-go.sh clean ${WHAT} && rm _out/* -rf"
 	hack/dockerized "bazel clean --expunge"
@@ -213,4 +216,5 @@ fossa:
 	coverage \
 	goveralls \
 	build-functests \
-	fossa
+	fossa \
+	realtime-perftest
diff --git a/WORKSPACE b/WORKSPACE
@@ -375,6 +375,13 @@ container_pull(
     repository = "kubevirt/alpine-ext-kernel-boot-demo",
 )
 
+container_pull(
+    name = "fedora_realtime",
+    digest = "sha256:437f4e02986daf0058239f4a282d32304dcac629d5d1b4c75a74025f1ce22811",
+    registry = "quay.io",
+    repository = "kubevirt/fedora-realtime-container-disk",
+)
+
 load(
     "@io_bazel_rules_docker//go:image.bzl",
     _go_image_repos = "repositories",

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -9766,6 +9766,10 @@
       "description": "NUMA allows specifying settings for the guest NUMA topology",
       "$ref": "#/definitions/v1.NUMA"
      },
+     "realtime": {
+      "description": "Realtime instructs the virt-launcher to tune the VMI for lower latency, optional for real time workloads",
+      "$ref": "#/definitions/v1.Realtime"
+     },
      "sockets": {
       "description": "Sockets specifies the number of sockets inside the vmi. Must be a value greater or equal 1.",
       "type": "integer",
@@ -11832,6 +11836,16 @@
      }
     }
    },
+   "v1.Realtime": {
+    "description": "Realtime holds the tuning knobs specific for realtime workloads.",
+    "type": "object",
+    "properties": {
+     "mask": {
+      "description": "Mask defines the vcpu mask expression that defines which vcpus are used for realtime. Format matches libvirt's expressions. Example: \"0-3,^1\",\"0,2,3\",\"2-3\"",
+      "type": "string"
+     }
+    }
+   },
    "v1.ReloadableComponentConfiguration": {
     "description": "ReloadableComponentConfiguration holds all generic k8s configuration options which can be reloaded by components without requiring a restart.",
     "type": "object",

diff --git a/automation/test.sh b/automation/test.sh
@@ -49,6 +49,10 @@ elif [[ $TARGET =~ sig-network ]]; then
 elif [[ $TARGET =~ sig-storage ]]; then
   export KUBEVIRT_PROVIDER=${TARGET/-sig-storage/}
   export KUBEVIRT_STORAGE="rook-ceph-default"
+elif [[ $TARGET =~ sig-compute-realtime ]]; then
+  export KUBEVIRT_PROVIDER=${TARGET/-sig-compute-realtime/}
+  export KUBEVIRT_HUGEPAGES_2M=512
+  export KUBEVIRT_REALTIME_SCHEDULER=true
 elif [[ $TARGET =~ sig-compute ]]; then
   export KUBEVIRT_PROVIDER=${TARGET/-sig-compute/}
 elif [[ $TARGET =~ sig-operator ]]; then
@@ -353,6 +357,8 @@ if [[ -z ${KUBEVIRT_E2E_FOCUS} && -z ${KUBEVIRT_E2E_SKIP} ]]; then
     export KUBEVIRT_E2E_FOCUS="\\[sig-storage\\]|\\[rook-ceph\\]"
   elif [[ $TARGET =~ vgpu.* ]]; then
     export KUBEVIRT_E2E_FOCUS=MediatedDevices
+  elif [[ $TARGET =~ sig-compute-realtime ]]; then
+    export KUBEVIRT_E2E_FOCUS="\\[sig-compute-realtime\\]"
   elif [[ $TARGET =~ sig-compute ]]; then
     export KUBEVIRT_E2E_FOCUS="\\[sig-compute\\]"
     export KUBEVIRT_E2E_SKIP="GPU|MediatedDevices"

diff --git a/containerimages/BUILD.bazel b/containerimages/BUILD.bazel
@@ -129,3 +129,17 @@ container_image(
     tars = [":alpine-image-tar"],
     visibility = ["//visibility:public"],
 )
+
+container_image(
+    name = "fedora-realtime",
+    architecture = select({
+        "@io_bazel_rules_go//go/platform:linux_arm64": "arm64",
+        "//conditions:default": "amd64",
+    }),
+    base = select({
+        "@io_bazel_rules_go//go/platform:linux_arm64": "@fedora_realtime_aarch64//image",
+        "//conditions:default": "@fedora_realtime//image",
+    }),
+    mode = "444",
+    visibility = ["//visibility:public"],
+)
diff --git a/hack/check-unassigned-tests.sh b/hack/check-unassigned-tests.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 main() {
-    skip="SRIOV|GPU|\\[sig-operator\\]|\\[sig-network\\]|\\[sig-storage\\]|\\[sig-compute\\]|\\[sig-performance\\]"
+    skip="SRIOV|GPU|\\[sig-operator\\]|\\[sig-network\\]|\\[sig-storage\\]|\\[sig-compute\\]|\\[sig-performance\\]|\\[sig-compute-realtime\\]"
     result=$(FUNC_TEST_ARGS="-dryRun -skip=${skip}" make functest)
     total_tests=$(echo "${result}" | grep "Ran[[:space:]].*of" | awk '{print $2}')
     if [ "${total_tests}" != "0" ]; then

diff --git a/hack/realtime-perftests.sh b/hack/realtime-perftests.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+#
+# This file is part of the KubeVirt project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright 2021 Red Hat, Inc.
+#
+# This script should be executed through the makefile via `make perftest`.
+
+set -e
+
+DOCKER_TAG=${DOCKER_TAG:-devel}
+DOCKER_TAG_ALT=${DOCKER_TAG_ALT:-devel_alt}
+
+source hack/common.sh
+source hack/config.sh
+
+_default_previous_release_registry="quay.io/kubevirt"
+
+previous_release_registry=${PREVIOUS_RELEASE_REGISTRY:-$_default_previous_release_registry}
+
+perftest_docker_prefix=${manifest_docker_prefix-${docker_prefix}}
+
+if [[ ${KUBEVIRT_PROVIDER} == os-* ]] || [[ ${KUBEVIRT_PROVIDER} =~ (okd|ocp)-* ]]; then
+    oc=${kubectl}
+fi
+
+if [ -z "$kubeconfig" ]; then
+    kubeconfig="$KUBECONFIG"
+fi
+
+echo 'Preparing directory for artifacts'
+export ARTIFACTS=${ARTIFACTS}/performance
+rm -rf $ARTIFACTS
+mkdir -p $ARTIFACTS
+
+export TESTS_OUT_DIR=${TESTS_OUT_DIR}
+mkdir -p ${TESTS_OUT_DIR}/performance
+
+echo 'ARTIFACTS ' ${ARTIFACTS}
+echo 'TESTS_OUT_DIR ' ${TESTS_OUT_DIR}
+
+function perftest() {
+    _out/tests/ginkgo -r --slowSpecThreshold 60 $@ _out/tests/tests.test -- ${extra_args} -kubeconfig=${kubeconfig} -container-tag=${docker_tag} -container-tag-alt=${docker_tag_alt} -container-prefix=${perftest_docker_prefix} -image-prefix-alt=${image_prefix_alt} -oc-path=${oc} -kubectl-path=${kubectl} -gocli-path=${gocli} -installed-namespace=${namespace} -previous-release-tag=${PREVIOUS_RELEASE_TAG} -previous-release-registry=${previous_release_registry} -deploy-testing-infra=${deploy_testing_infra} -config=${KUBEVIRT_DIR}/tests/default-config.json --artifacts=${ARTIFACTS}
+}
+
+if [ -n "$KUBEVIRT_E2E_FOCUS" ]; then
+    export KUBEVIRT_E2E_FOCUS="${KUBEVIRT_E2E_FOCUS}|\\[sig-performance\\]"
+else
+    export KUBEVIRT_E2E_FOCUS="\\[sig-performance\\]"
+fi
+
+additional_test_args=""
+if [ -n "$KUBEVIRT_E2E_SKIP" ]; then
+    additional_test_args="${additional_test_args} --skip=${KUBEVIRT_E2E_SKIP}"
+fi
+
+if [ -n "$KUBEVIRT_E2E_FOCUS" ]; then
+    additional_test_args="${additional_test_args} --focus=${KUBEVIRT_E2E_FOCUS}"
+fi
+
+additional_test_args="${additional_test_args} --skipPackage test/performance"
+
+perftest ${additional_test_args} ${FUNC_TEST_ARGS}
diff --git a/pkg/util/util.go b/pkg/util/util.go
@@ -8,15 +8,17 @@ import (
 	v1 "kubevirt.io/client-go/api/v1"
 )
 
-const ExtensionAPIServerAuthenticationConfigMap = "extension-apiserver-authentication"
-const RequestHeaderClientCAFileKey = "requestheader-client-ca-file"
-const VirtShareDir = "/var/run/kubevirt"
-const VirtPrivateDir = "/var/run/kubevirt-private"
-const VirtLibDir = "/var/lib/kubevirt"
-const KubeletPodsDir = "/var/lib/kubelet/pods"
-const HostRootMount = "/proc/1/root/"
-const CPUManagerOS3Path = HostRootMount + "var/lib/origin/openshift.local.volumes/cpu_manager_state"
-const CPUManagerPath = HostRootMount + "var/lib/kubelet/cpu_manager_state"
+const (
+	ExtensionAPIServerAuthenticationConfigMap = "extension-apiserver-authentication"
+	RequestHeaderClientCAFileKey              = "requestheader-client-ca-file"
+	VirtShareDir                              = "/var/run/kubevirt"
+	VirtPrivateDir                            = "/var/run/kubevirt-private"
+	VirtLibDir                                = "/var/lib/kubevirt"
+	KubeletPodsDir                            = "/var/lib/kubelet/pods"
+	HostRootMount                             = "/proc/1/root/"
+	CPUManagerOS3Path                         = HostRootMount + "var/lib/origin/openshift.local.volumes/cpu_manager_state"
+	CPUManagerPath                            = HostRootMount + "var/lib/kubelet/cpu_manager_state"
+)
 
 const NonRootUID = 107
 const NonRootUserString = "qemu"

diff --git a/pkg/virt-api/webhooks/mutating-webhook/mutators/vmi-mutator.go b/pkg/virt-api/webhooks/mutating-webhook/mutators/vmi-mutator.go
@@ -116,6 +116,11 @@ func (mutator *VMIsMutator) Mutate(ar *admissionv1.AdmissionReview) *admissionv1
 				log.Log.V(2).Infof("Failed to setting for Arm64: %s", err)
 			}
 		}
+		if newVMI.IsRealtimeEnabled() {
+			log.Log.V(4).Info("Add realtime node label selector")
+			addRealtimeNodeSelector(newVMI)
+		}
+
 		// Add foreground finalizer
 		newVMI.Finalizers = append(newVMI.Finalizers, v1.VirtualMachineInstanceFinalizer)
 
@@ -356,3 +361,11 @@ func canBeNonRoot(vmi *v1.VirtualMachineInstance) error {
 	}
 	return nil
 }
+
+// AddRealtimeNodeSelector adds the realtime node selector
+func addRealtimeNodeSelector(vmi *v1.VirtualMachineInstance) {
+	if vmi.Spec.NodeSelector == nil {
+		vmi.Spec.NodeSelector = map[string]string{}
+	}
+	vmi.Spec.NodeSelector[v1.RealtimeLabel] = ""
+}
diff --git a/pkg/virt-api/webhooks/mutating-webhook/mutators/vmi-mutator_test.go b/pkg/virt-api/webhooks/mutating-webhook/mutators/vmi-mutator_test.go
@@ -941,4 +941,22 @@ var _ = Describe("VirtualMachineInstance Mutator", func() {
 		})
 	})
 
+	It("should add realtime node label selector with realtime workload", func() {
+		vmi.Spec.Domain.CPU = &v1.CPU{Realtime: &v1.Realtime{}}
+		vmiSpec, _ := getVMISpecMetaFromResponse()
+		Expect(vmiSpec.NodeSelector).NotTo(BeNil())
+		Expect(vmiSpec.NodeSelector).To(BeEquivalentTo(map[string]string{v1.RealtimeLabel: ""}))
+	})
+	It("should not add realtime node label selector when no realtime workload", func() {
+		vmi.Spec.Domain.CPU = &v1.CPU{Realtime: nil}
+		vmi.Spec.NodeSelector = map[string]string{v1.NodeSchedulable: "true"}
+		vmiSpec, _ := getVMISpecMetaFromResponse()
+		Expect(vmiSpec.NodeSelector).To(BeEquivalentTo(map[string]string{v1.NodeSchedulable: "true"}))
+	})
+	It("should not overwrite existing node label selectors with realtime workload", func() {
+		vmi.Spec.Domain.CPU = &v1.CPU{Realtime: &v1.Realtime{}}
+		vmi.Spec.NodeSelector = map[string]string{v1.NodeSchedulable: "true"}
+		vmiSpec, _ := getVMISpecMetaFromResponse()
+		Expect(vmiSpec.NodeSelector).To(BeEquivalentTo(map[string]string{v1.NodeSchedulable: "true", v1.RealtimeLabel: ""}))
+	})
 })
diff --git a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-create-admitter.go b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-create-admitter.go
@@ -142,6 +142,7 @@ func ValidateVirtualMachineInstanceSpec(field *k8sfield.Path, spec *v1.VirtualMa
 	causes = append(causes, validateCPUIsolatorThread(field, spec)...)
 	causes = append(causes, validateCPUFeaturePolicies(field, spec)...)
 	causes = append(causes, validateStartStrategy(field, spec)...)
+	causes = append(causes, validateRealtime(field, spec)...)
 
 	maxNumberOfInterfacesExceeded := len(spec.Domain.Devices.Interfaces) > arrayLenMax
 	if maxNumberOfInterfacesExceeded {
@@ -996,7 +997,7 @@ func validateNUMA(field *k8sfield.Path, spec *v1.VirtualMachineInstanceSpec, con
 				Field: field.Child("domain", "cpu", "numa", "guestMappingPassthrough").String(),
 			})
 		}
-		if spec.Domain.CPU.DedicatedCPUPlacement == false {
+		if !spec.Domain.CPU.DedicatedCPUPlacement {
 			causes = append(causes, metav1.StatusCause{
 				Type: metav1.CauseTypeFieldValueInvalid,
 				Message: fmt.Sprintf("%s must be set to true when NUMA topology strategy is set in %s",
@@ -1371,6 +1372,42 @@ func validateHostNameNotConformingToDNSLabelRules(field *k8sfield.Path, spec *v1
 	return causes
 }
 
+func validateRealtime(field *k8sfield.Path, spec *v1.VirtualMachineInstanceSpec) (causes []metav1.StatusCause) {
+	if spec.Domain.CPU != nil && spec.Domain.CPU.Realtime != nil {
+		causes = append(causes, validateCPURealtime(field, spec)...)
+		causes = append(causes, validateMemoryRealtime(field, spec)...)
+	}
+	return causes
+}
+
+func validateCPURealtime(field *k8sfield.Path, spec *v1.VirtualMachineInstanceSpec) (causes []metav1.StatusCause) {
+	if !spec.Domain.CPU.DedicatedCPUPlacement {
+		causes = append(causes, metav1.StatusCause{
+			Type: metav1.CauseTypeFieldValueRequired,
+			Message: fmt.Sprintf("%s must be set to true when %s is used",
+				field.Child("domain", "cpu", "dedicatedCpuPlacement").String(),
+				field.Child("domain", "cpu", "realtime").String(),
+			),
+			Field: field.Child("domain", "cpu", "dedicatedCpuPlacement").String(),
+		})
+	}
+	return causes
+}
+
+func validateMemoryRealtime(field *k8sfield.Path, spec *v1.VirtualMachineInstanceSpec) (causes []metav1.StatusCause) {
+	if spec.Domain.CPU.NUMA == nil || spec.Domain.CPU.NUMA.GuestMappingPassthrough == nil {
+		causes = append(causes, metav1.StatusCause{
+			Type: metav1.CauseTypeFieldValueRequired,
+			Message: fmt.Sprintf("%s must be defined when %s is used",
+				field.Child("domain", "cpu", "numa", "guestMappingPassthrough").String(),
+				field.Child("domain", "cpu", "realtime").String(),
+			),
+			Field: field.Child("domain", "cpu", "numa", "guestMappingPassthrough").String(),
+		})
+	}
+	return causes
+}
+
 func appendNewStatusCauseForHostNameNotConformingToDNSLabelRules(field *k8sfield.Path, causes []metav1.StatusCause, errors []string) []metav1.StatusCause {
 	return append(causes, metav1.StatusCause{
 		Type: metav1.CauseTypeFieldValueInvalid,

diff --git a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-create-admitter_test.go b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-create-admitter_test.go
@@ -3467,6 +3467,36 @@ var _ = Describe("Validating VMICreate Admitter", func() {
 			Expect(len(causes)).To(Equal(1))
 		})
 	})
+
+	Context("with realtime", func() {
+		var vmi *v1.VirtualMachineInstance
+		BeforeEach(func() {
+			vmi = v1.NewMinimalVMI("testvmi")
+			vmi.Spec.Domain.CPU = &v1.CPU{Realtime: &v1.Realtime{}, Cores: 4}
+			enableFeatureGate(virtconfig.NUMAFeatureGate)
+		})
+		It("should reject the realtime knob without DedicatedCPUPlacement", func() {
+			vmi.Spec.Domain.Memory = &v1.Memory{Hugepages: &v1.Hugepages{PageSize: "2Mi"}}
+			vmi.Spec.Domain.CPU.NUMA = &v1.NUMA{GuestMappingPassthrough: &v1.NUMAGuestMappingPassthrough{}}
+			causes := ValidateVirtualMachineInstanceSpec(k8sfield.NewPath("fake"), &vmi.Spec, config)
+			Expect(len(causes)).To(BeNumerically(">=", 1))
+			Expect(causes).To(ContainElement(metav1.StatusCause{Type: metav1.CauseTypeFieldValueRequired, Field: "fake.domain.cpu.dedicatedCpuPlacement", Message: "fake.domain.cpu.dedicatedCpuPlacement must be set to true when fake.domain.cpu.realtime is used"}))
+		})
+		It("should reject the realtime knob when NUMA Guest Mapping Passthrough is not defined", func() {
+			vmi.Spec.Domain.CPU.DedicatedCPUPlacement = true
+			vmi.Spec.Domain.CPU.NUMA = &v1.NUMA{}
+			causes := ValidateVirtualMachineInstanceSpec(k8sfield.NewPath("fake"), &vmi.Spec, config)
+			Expect(causes).To(HaveLen(1))
+			Expect(causes).To(ContainElement(metav1.StatusCause{Type: metav1.CauseTypeFieldValueRequired, Field: "fake.domain.cpu.numa.guestMappingPassthrough", Message: "fake.domain.cpu.numa.guestMappingPassthrough must be defined when fake.domain.cpu.realtime is used"}))
+		})
+		It("should reject the realtime knob when NUMA is nil", func() {
+			vmi.Spec.Domain.CPU.DedicatedCPUPlacement = true
+			vmi.Spec.Domain.CPU.NUMA = nil
+			causes := ValidateVirtualMachineInstanceSpec(k8sfield.NewPath("fake"), &vmi.Spec, config)
+			Expect(causes).To(HaveLen(1))
+			Expect(causes).To(ContainElement(metav1.StatusCause{Type: metav1.CauseTypeFieldValueRequired, Field: "fake.domain.cpu.numa.guestMappingPassthrough", Message: "fake.domain.cpu.numa.guestMappingPassthrough must be defined when fake.domain.cpu.realtime is used"}))
+		})
+	})
 })
 
 var _ = Describe("Function getNumberOfPodInterfaces()", func() {
@@ -3885,5 +3915,4 @@ var _ = Describe("Function getNumberOfPodInterfaces()", func() {
 		causes := webhooks.ValidateVirtualMachineInstanceHypervFeatureDependencies(path, &vmi.Spec)
 		Expect(len(causes)).To(Equal(0))
 	})
-
 })