Skip to content

Commit

Permalink
Merge pull request kubernetes#83601 from ahg-g/ahg-migration-priority
Browse files Browse the repository at this point in the history 
Implemented taints and tolerations priority function as a Score plugin
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Oct 11, 2019
2 parents 1185012 + 7429b0f commit c1a735c
Show file tree
Hide file tree
Showing 17 changed files with 509 additions and 87 deletions.
1 change: 1 addition & 0 deletions pkg/scheduler/api/compatibility/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/client-go/informers:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/github.com/google/go-cmp/cmp:go_default_library",
],
)

Expand Down
111 changes: 59 additions & 52 deletions pkg/scheduler/api/compatibility/compatibility_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package compatibility
import (
"testing"

"github.com/google/go-cmp/cmp"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
Expand All @@ -37,11 +39,11 @@ import (
func TestCompatibility_v1_Scheduler(t *testing.T) {
// Add serialized versions of scheduler config that exercise available options to ensure compatibility between releases
schedulerFiles := map[string]struct {
JSON string
wantPredicates sets.String
wantPrioritizers sets.String
wantFilterPlugins sets.String
wantExtenders []schedulerapi.ExtenderConfig
JSON string
wantPredicates sets.String
wantPrioritizers sets.String
wantPlugins map[string][]kubeschedulerconfig.Plugin
wantExtenders []schedulerapi.ExtenderConfig
}{
// Do not change this JSON after the corresponding release has been tagged.
// A failure indicates backwards compatibility with the specified release was broken.
Expand Down Expand Up @@ -231,12 +233,12 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
"BalancedResourceAllocation",
"SelectorSpreadPriority",
"NodeAffinityPriority",
"TaintTolerationPriority",
"InterPodAffinityPriority",
),
wantFilterPlugins: sets.NewString(
"TaintToleration",
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {{Name: "TaintToleration"}},
"ScorePlugin": {{Name: "TaintToleration", Weight: 2}},
},
},

// Do not change this JSON after the corresponding release has been tagged.
Expand Down Expand Up @@ -300,13 +302,13 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
"SelectorSpreadPriority",
"NodePreferAvoidPodsPriority",
"NodeAffinityPriority",
"TaintTolerationPriority",
"InterPodAffinityPriority",
"MostRequestedPriority",
),
wantFilterPlugins: sets.NewString(
"TaintToleration",
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {{Name: "TaintToleration"}},
"ScorePlugin": {{Name: "TaintToleration", Weight: 2}},
},
},
// Do not change this JSON after the corresponding release has been tagged.
// A failure indicates backwards compatibility with the specified release was broken.
Expand Down Expand Up @@ -379,13 +381,13 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
"SelectorSpreadPriority",
"NodePreferAvoidPodsPriority",
"NodeAffinityPriority",
"TaintTolerationPriority",
"InterPodAffinityPriority",
"MostRequestedPriority",
),
wantFilterPlugins: sets.NewString(
"TaintToleration",
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {{Name: "TaintToleration"}},
"ScorePlugin": {{Name: "TaintToleration", Weight: 2}},
},
wantExtenders: []schedulerapi.ExtenderConfig{{
URLPrefix: "/prefix",
FilterVerb: "filter",
Expand Down Expand Up @@ -471,13 +473,13 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
"SelectorSpreadPriority",
"NodePreferAvoidPodsPriority",
"NodeAffinityPriority",
"TaintTolerationPriority",
"InterPodAffinityPriority",
"MostRequestedPriority",
),
wantFilterPlugins: sets.NewString(
"TaintToleration",
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {{Name: "TaintToleration"}},
"ScorePlugin": {{Name: "TaintToleration", Weight: 2}},
},
wantExtenders: []schedulerapi.ExtenderConfig{{
URLPrefix: "/prefix",
FilterVerb: "filter",
Expand Down Expand Up @@ -565,13 +567,13 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
"SelectorSpreadPriority",
"NodePreferAvoidPodsPriority",
"NodeAffinityPriority",
"TaintTolerationPriority",
"InterPodAffinityPriority",
"MostRequestedPriority",
),
wantFilterPlugins: sets.NewString(
"TaintToleration",
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {{Name: "TaintToleration"}},
"ScorePlugin": {{Name: "TaintToleration", Weight: 2}},
},
wantExtenders: []schedulerapi.ExtenderConfig{{
URLPrefix: "/prefix",
FilterVerb: "filter",
Expand Down Expand Up @@ -664,13 +666,13 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
"SelectorSpreadPriority",
"NodePreferAvoidPodsPriority",
"NodeAffinityPriority",
"TaintTolerationPriority",
"InterPodAffinityPriority",
"MostRequestedPriority",
),
wantFilterPlugins: sets.NewString(
"TaintToleration",
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {{Name: "TaintToleration"}},
"ScorePlugin": {{Name: "TaintToleration", Weight: 2}},
},
wantExtenders: []schedulerapi.ExtenderConfig{{
URLPrefix: "/prefix",
FilterVerb: "filter",
Expand Down Expand Up @@ -775,14 +777,14 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
"SelectorSpreadPriority",
"NodePreferAvoidPodsPriority",
"NodeAffinityPriority",
"TaintTolerationPriority",
"InterPodAffinityPriority",
"MostRequestedPriority",
"RequestedToCapacityRatioPriority",
),
wantFilterPlugins: sets.NewString(
"TaintToleration",
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {{Name: "TaintToleration"}},
"ScorePlugin": {{Name: "TaintToleration", Weight: 2}},
},
wantExtenders: []schedulerapi.ExtenderConfig{{
URLPrefix: "/prefix",
FilterVerb: "filter",
Expand Down Expand Up @@ -889,14 +891,14 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
"SelectorSpreadPriority",
"NodePreferAvoidPodsPriority",
"NodeAffinityPriority",
"TaintTolerationPriority",
"InterPodAffinityPriority",
"MostRequestedPriority",
"RequestedToCapacityRatioPriority",
),
wantFilterPlugins: sets.NewString(
"TaintToleration",
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {{Name: "TaintToleration"}},
"ScorePlugin": {{Name: "TaintToleration", Weight: 2}},
},
wantExtenders: []schedulerapi.ExtenderConfig{{
URLPrefix: "/prefix",
FilterVerb: "filter",
Expand Down Expand Up @@ -1003,14 +1005,14 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
"SelectorSpreadPriority",
"NodePreferAvoidPodsPriority",
"NodeAffinityPriority",
"TaintTolerationPriority",
"InterPodAffinityPriority",
"MostRequestedPriority",
"RequestedToCapacityRatioPriority",
),
wantFilterPlugins: sets.NewString(
"TaintToleration",
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {{Name: "TaintToleration"}},
"ScorePlugin": {{Name: "TaintToleration", Weight: 2}},
},
wantExtenders: []schedulerapi.ExtenderConfig{{
URLPrefix: "/prefix",
FilterVerb: "filter",
Expand Down Expand Up @@ -1121,14 +1123,14 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
"SelectorSpreadPriority",
"NodePreferAvoidPodsPriority",
"NodeAffinityPriority",
"TaintTolerationPriority",
"InterPodAffinityPriority",
"MostRequestedPriority",
"RequestedToCapacityRatioPriority",
),
wantFilterPlugins: sets.NewString(
"TaintToleration",
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {{Name: "TaintToleration"}},
"ScorePlugin": {{Name: "TaintToleration", Weight: 2}},
},
wantExtenders: []schedulerapi.ExtenderConfig{{
URLPrefix: "/prefix",
FilterVerb: "filter",
Expand All @@ -1152,6 +1154,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
filterToPredicateMap := map[string]string{
"TaintToleration": "PodToleratesNodeTaints",
}
scoreToPriorityMap := map[string]string{
"TaintToleration": "TaintTolerationPriority",
}

for v, tc := range schedulerFiles {
t.Run(v, func(t *testing.T) {
Expand Down Expand Up @@ -1208,15 +1213,17 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
t.Errorf("Got prioritizers %v, want %v", gotPrioritizers, tc.wantPrioritizers)
}

gotFilterPlugins := sets.NewString()
plugins := sched.Framework.ListPlugins()
for _, p := range plugins["FilterPlugin"] {
gotFilterPlugins.Insert(p)
seenPredicates.Insert(filterToPredicateMap[p])
gotPlugins := sched.Framework.ListPlugins()
for _, p := range gotPlugins["FilterPlugin"] {
seenPredicates.Insert(filterToPredicateMap[p.Name])

}
for _, p := range gotPlugins["FilterPlugin"] {
seenPriorities.Insert(scoreToPriorityMap[p.Name])

}
if !gotFilterPlugins.Equal(tc.wantFilterPlugins) {
t.Errorf("Got filter plugins %v, want %v", gotFilterPlugins, tc.wantFilterPlugins)
if diff := cmp.Diff(tc.wantPlugins, gotPlugins); diff != "" {
t.Errorf("unexpected plugins diff (-want, +got): %s", diff)
}

gotExtenders := sched.Algorithm.Extenders()
Expand Down
2 changes: 1 addition & 1 deletion pkg/scheduler/factory/factory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -636,7 +636,7 @@ func (t *TestPlugin) Name() string {
return t.name
}

func (t *TestPlugin) Score(state *framework.CycleState, p *v1.Pod, nodeName string) (int, *framework.Status) {
func (t *TestPlugin) Score(state *framework.CycleState, p *v1.Pod, nodeName string) (int64, *framework.Status) {
return 1, nil
}

Expand Down
1 change: 1 addition & 0 deletions pkg/scheduler/framework/plugins/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ go_library(
deps = [
"//pkg/scheduler/algorithm:go_default_library",
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/algorithm/priorities:go_default_library",
"//pkg/scheduler/apis/config:go_default_library",
"//pkg/scheduler/framework/plugins/tainttoleration:go_default_library",
"//pkg/scheduler/framework/v1alpha1:go_default_library",
Expand Down
7 changes: 7 additions & 0 deletions pkg/scheduler/framework/plugins/default_registry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
storagelistersv1 "k8s.io/client-go/listers/storage/v1"
"k8s.io/kubernetes/pkg/scheduler/algorithm"
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
"k8s.io/kubernetes/pkg/scheduler/algorithm/priorities"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
Expand Down Expand Up @@ -83,6 +84,12 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry {
return
})

registry.RegisterPriority(priorities.TaintTolerationPriority,
func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
plugins.Score = appendToPluginSet(plugins.Score, tainttoleration.Name, &args.Weight)
return
})

return registry
}

Expand Down
1 change: 1 addition & 0 deletions pkg/scheduler/framework/plugins/migration/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ go_library(
deps = [
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/framework/v1alpha1:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)

Expand Down
16 changes: 16 additions & 0 deletions pkg/scheduler/framework/plugins/migration/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package migration

import (
"k8s.io/klog"
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
)
Expand Down Expand Up @@ -88,3 +89,18 @@ func (p *PrioritiesStateData) Clone() framework.StateData {
Reference: p.Reference,
}
}

// PriorityMetadata returns priority metadata stored in CycleState.
func PriorityMetadata(state *framework.CycleState) interface{} {
if state == nil {
return nil
}

var meta interface{}
if s, err := state.Read(PrioritiesStateKey); err == nil {
meta = s.(*PrioritiesStateData).Reference
} else {
klog.Errorf("reading key %q from CycleState, continuing without metadata: %v", PrioritiesStateKey, err)
}
return meta
}
16 changes: 15 additions & 1 deletion pkg/scheduler/framework/plugins/tainttoleration/BUILD
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")

go_library(
name = "go_default_library",
Expand All @@ -7,6 +7,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/algorithm/priorities:go_default_library",
"//pkg/scheduler/framework/plugins/migration:go_default_library",
"//pkg/scheduler/framework/v1alpha1:go_default_library",
"//pkg/scheduler/nodeinfo:go_default_library",
Expand All @@ -28,3 +29,16 @@ filegroup(
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

go_test(
name = "go_default_test",
srcs = ["taint_toleration_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/framework/v1alpha1:go_default_library",
"//pkg/scheduler/nodeinfo:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
],
)
Loading

0 comments on commit c1a735c

Please sign in to comment.