From 03e439493830abc0c27e1cef0c5b26ab927bf367 Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Sat, 5 Aug 2017 09:38:15 +0800 Subject: [PATCH] Add MemoryPressure/DiskPressure toleration for no BestEffort pod. --- .../admission/podtolerationrestriction/BUILD | 5 ++ .../podtolerationrestriction/admission.go | 12 +++ .../admission_test.go | 85 ++++++++++++++++++- 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/plugin/pkg/admission/podtolerationrestriction/BUILD b/plugin/pkg/admission/podtolerationrestriction/BUILD index 8b4f692b6b6f3..35cf74fb48420 100644 --- a/plugin/pkg/admission/podtolerationrestriction/BUILD +++ b/plugin/pkg/admission/podtolerationrestriction/BUILD @@ -21,8 +21,11 @@ go_test( "//pkg/kubeapiserver/admission:go_default_library", "//pkg/util/tolerations:go_default_library", "//plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction:go_default_library", + "//plugin/pkg/scheduler/algorithm:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], ) @@ -35,6 +38,7 @@ go_library( tags = ["automanaged"], deps = [ "//pkg/api:go_default_library", + "//pkg/api/helper/qos:go_default_library", "//pkg/api/v1:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/informers/informers_generated/internalversion:go_default_library", @@ -45,6 +49,7 @@ go_library( "//plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/install:go_default_library", "//plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/v1alpha1:go_default_library", "//plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/validation:go_default_library", + "//plugin/pkg/scheduler/algorithm:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/plugin/pkg/admission/podtolerationrestriction/admission.go b/plugin/pkg/admission/podtolerationrestriction/admission.go index ed589c97b6e31..a71f92ba4a121 100644 --- a/plugin/pkg/admission/podtolerationrestriction/admission.go +++ b/plugin/pkg/admission/podtolerationrestriction/admission.go @@ -28,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/admission" "k8s.io/kubernetes/pkg/api" + qoshelper "k8s.io/kubernetes/pkg/api/helper/qos" k8s_api_v1 "k8s.io/kubernetes/pkg/api/v1" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" @@ -35,6 +36,7 @@ import ( kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" "k8s.io/kubernetes/pkg/util/tolerations" pluginapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction" + "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" ) // Register registers a plugin @@ -162,6 +164,16 @@ func (p *podTolerationsPlugin) Admit(a admission.Attributes) error { } } + if qoshelper.GetPodQOS(pod) != api.PodQOSBestEffort { + finalTolerations = tolerations.MergeTolerations(finalTolerations, []api.Toleration{ + { + Key: algorithm.TaintNodeMemoryPressure, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoSchedule, + }, + }) + } + pod.Spec.Tolerations = finalTolerations return nil diff --git a/plugin/pkg/admission/podtolerationrestriction/admission_test.go b/plugin/pkg/admission/podtolerationrestriction/admission_test.go index 53717f613485b..76faaa718949e 100644 --- a/plugin/pkg/admission/podtolerationrestriction/admission_test.go +++ b/plugin/pkg/admission/podtolerationrestriction/admission_test.go @@ -21,8 +21,10 @@ import ( "testing" "time" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/admission" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" @@ -30,6 +32,7 @@ import ( kubeadmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" "k8s.io/kubernetes/pkg/util/tolerations" pluginapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction" + "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" ) // TestPodAdmission verifies various scenarios involving pod/namespace tolerations @@ -50,11 +53,56 @@ func TestPodAdmission(t *testing.T) { defer close(stopCh) informerFactory.Start(stopCh) - pod := &api.Pod{ + CPU1000m := resource.MustParse("1000m") + CPU500m := resource.MustParse("500m") + + burstablePod := &api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "testPod", Namespace: "testNamespace"}, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "test", + Resources: api.ResourceRequirements{ + Limits: api.ResourceList{api.ResourceCPU: CPU1000m}, + Requests: api.ResourceList{api.ResourceCPU: CPU500m}, + }, + }, + }, + }, + } + + guaranteedPod := &api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "testPod", Namespace: "testNamespace"}, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "test", + Resources: api.ResourceRequirements{ + Limits: api.ResourceList{api.ResourceCPU: CPU1000m}, + Requests: api.ResourceList{api.ResourceCPU: CPU1000m}, + }, + }, + }, + }, + } + + bestEffortPod := &api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "testPod", Namespace: "testNamespace"}, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "test", + }, + }, + }, + } + + if err := utilfeature.DefaultFeatureGate.Set("TaintNodesByCondition=true"); err != nil { + t.Errorf("Failed to enable TaintByCondition feature: %v.", err) } tests := []struct { + pod *api.Pod defaultClusterTolerations []api.Toleration namespaceTolerations []api.Toleration whitelist []api.Toleration @@ -65,6 +113,7 @@ func TestPodAdmission(t *testing.T) { testName string }{ { + pod: bestEffortPod, defaultClusterTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, namespaceTolerations: []api.Toleration{}, podTolerations: []api.Toleration{}, @@ -73,6 +122,7 @@ func TestPodAdmission(t *testing.T) { testName: "default cluster tolerations with empty pod tolerations", }, { + pod: bestEffortPod, defaultClusterTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, namespaceTolerations: []api.Toleration{}, podTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, @@ -81,6 +131,7 @@ func TestPodAdmission(t *testing.T) { testName: "default cluster tolerations with pod tolerations specified", }, { + pod: bestEffortPod, defaultClusterTolerations: []api.Toleration{}, namespaceTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, podTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, @@ -89,6 +140,7 @@ func TestPodAdmission(t *testing.T) { testName: "namespace tolerations", }, { + pod: bestEffortPod, defaultClusterTolerations: []api.Toleration{}, namespaceTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, podTolerations: []api.Toleration{}, @@ -97,6 +149,7 @@ func TestPodAdmission(t *testing.T) { testName: "no pod tolerations", }, { + pod: bestEffortPod, defaultClusterTolerations: []api.Toleration{}, namespaceTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, podTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue1", Effect: "NoSchedule", TolerationSeconds: nil}}, @@ -104,6 +157,7 @@ func TestPodAdmission(t *testing.T) { testName: "conflicting pod and namespace tolerations", }, { + pod: bestEffortPod, defaultClusterTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue2", Effect: "NoSchedule", TolerationSeconds: nil}}, namespaceTolerations: []api.Toleration{}, podTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue1", Effect: "NoSchedule", TolerationSeconds: nil}}, @@ -111,6 +165,7 @@ func TestPodAdmission(t *testing.T) { testName: "conflicting pod and default cluster tolerations", }, { + pod: bestEffortPod, defaultClusterTolerations: []api.Toleration{}, namespaceTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, whitelist: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, @@ -120,6 +175,7 @@ func TestPodAdmission(t *testing.T) { testName: "merged pod tolerations satisfy whitelist", }, { + pod: bestEffortPod, defaultClusterTolerations: []api.Toleration{}, namespaceTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, whitelist: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue1", Effect: "NoSchedule", TolerationSeconds: nil}}, @@ -127,6 +183,32 @@ func TestPodAdmission(t *testing.T) { admit: false, testName: "merged pod tolerations conflict with the whitelist", }, + { + pod: burstablePod, + defaultClusterTolerations: []api.Toleration{}, + namespaceTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, + whitelist: []api.Toleration{}, + podTolerations: []api.Toleration{}, + mergedTolerations: []api.Toleration{ + {Key: algorithm.TaintNodeMemoryPressure, Operator: api.TolerationOpExists, Effect: api.TaintEffectNoSchedule, TolerationSeconds: nil}, + {Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}, + }, + admit: true, + testName: "added memoryPressure/DiskPressure for Burstable pod", + }, + { + pod: guaranteedPod, + defaultClusterTolerations: []api.Toleration{}, + namespaceTolerations: []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}}, + whitelist: []api.Toleration{}, + podTolerations: []api.Toleration{}, + mergedTolerations: []api.Toleration{ + {Key: algorithm.TaintNodeMemoryPressure, Operator: api.TolerationOpExists, Effect: api.TaintEffectNoSchedule, TolerationSeconds: nil}, + {Key: "testKey", Operator: "Equal", Value: "testValue", Effect: "NoSchedule", TolerationSeconds: nil}, + }, + admit: true, + testName: "added memoryPressure/DiskPressure for Guaranteed pod", + }, } for _, test := range tests { if len(test.namespaceTolerations) > 0 { @@ -148,6 +230,7 @@ func TestPodAdmission(t *testing.T) { informerFactory.Core().InternalVersion().Namespaces().Informer().GetStore().Update(namespace) handler.pluginConfig = &pluginapi.Configuration{Default: test.defaultClusterTolerations, Whitelist: test.clusterWhitelist} + pod := test.pod pod.Spec.Tolerations = test.podTolerations err := handler.Admit(admission.NewAttributesRecord(pod, nil, api.Kind("Pod").WithVersion("version"), "testNamespace", namespace.ObjectMeta.Name, api.Resource("pods").WithVersion("version"), "", admission.Create, nil))