Skip to content

Commit

Permalink
Graduate HugePages feature to GA
Browse files Browse the repository at this point in the history
  • Loading branch information
@derekwaynecarr
derekwaynecarr committed Feb 2, 2019
1 parent 30566b9 commit deae071
Show file tree
Hide file tree
Showing 9 changed files with 41 additions and 111 deletions.
23 changes: 9 additions & 14 deletions pkg/apis/core/validation/validation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,9 +415,6 @@ func validateVolumeSource(source *core.VolumeSource, fldPath *field.Path, volNam
if source.EmptyDir.SizeLimit != nil && source.EmptyDir.SizeLimit.Cmp(resource.Quantity{}) < 0 {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("emptyDir").Child("sizeLimit"), "SizeLimit field must be a valid resource quantity"))
}
if !utilfeature.DefaultFeatureGate.Enabled(features.HugePages) && source.EmptyDir.Medium == core.StorageMediumHugePages {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("emptyDir").Child("medium"), "HugePages medium is disabled by feature-gate for EmptyDir volumes"))
}
}
if source.HostPath != nil {
if numVolumes > 0 {
Expand Down Expand Up @@ -2925,19 +2922,17 @@ func ValidatePod(pod *core.Pod) field.ErrorList {
allErrs = append(allErrs, validateContainersOnlyForPod(pod.Spec.Containers, specPath.Child("containers"))...)
allErrs = append(allErrs, validateContainersOnlyForPod(pod.Spec.InitContainers, specPath.Child("initContainers"))...)

if utilfeature.DefaultFeatureGate.Enabled(features.HugePages) {
hugePageResources := sets.NewString()
for i := range pod.Spec.Containers {
resourceSet := toContainerResourcesSet(&pod.Spec.Containers[i])
for resourceStr := range resourceSet {
if v1helper.IsHugePageResourceName(v1.ResourceName(resourceStr)) {
hugePageResources.Insert(resourceStr)
}
hugePageResources := sets.NewString()
for i := range pod.Spec.Containers {
resourceSet := toContainerResourcesSet(&pod.Spec.Containers[i])
for resourceStr := range resourceSet {
if v1helper.IsHugePageResourceName(v1.ResourceName(resourceStr)) {
hugePageResources.Insert(resourceStr)
}
}
if len(hugePageResources) > 1 {
allErrs = append(allErrs, field.Invalid(specPath, hugePageResources, "must use a single hugepage size in a pod spec"))
}
}
if len(hugePageResources) > 1 {
allErrs = append(allErrs, field.Invalid(specPath, hugePageResources, "must use a single hugepage size in a pod spec"))
}

return allErrs
Expand Down
22 changes: 2 additions & 20 deletions pkg/apis/core/validation/validation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3659,24 +3659,17 @@ func TestValidateVolumes(t *testing.T) {
t.Errorf("expected error type %v, got %v", field.ErrorTypeDuplicate, errs[0].Type)
}

// Validate HugePages medium type for EmptyDir when HugePages feature is enabled/disabled
// Validate HugePages medium type for EmptyDir
hugePagesCase := core.VolumeSource{EmptyDir: &core.EmptyDirVolumeSource{Medium: core.StorageMediumHugePages}}

// Enable HugePages
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)()
if errs := validateVolumeSource(&hugePagesCase, field.NewPath("field").Index(0), "working"); len(errs) != 0 {
t.Errorf("Unexpected error when HugePages feature is enabled.")
}

// Disable feature HugePages
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)()
if errs := validateVolumeSource(&hugePagesCase, field.NewPath("field").Index(0), "failing"); len(errs) == 0 {
t.Errorf("Expected error when HugePages feature is disabled got nothing.")
}

}

func TestAlphaHugePagesIsolation(t *testing.T) {
func TestHugePagesIsolation(t *testing.T) {
successCases := []core.Pod{
{ // Basic fields.
ObjectMeta: metav1.ObjectMeta{Name: "123", Namespace: "ns"},
Expand Down Expand Up @@ -3774,8 +3767,6 @@ func TestAlphaHugePagesIsolation(t *testing.T) {
},
},
}
// Enable feature HugePages
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)()
for i := range successCases {
pod := &successCases[i]
if errs := ValidatePod(pod); len(errs) != 0 {
Expand All @@ -3788,15 +3779,6 @@ func TestAlphaHugePagesIsolation(t *testing.T) {
t.Errorf("Expected error for case[%d], pod: %v", i, pod.Name)
}
}
// Disable feature HugePages
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)()
// Disable feature HugePages and ensure all success cases fail
for i := range successCases {
pod := &successCases[i]
if errs := ValidatePod(pod); len(errs) == 0 {
t.Errorf("Expected error for case[%d], pod: %v", i, pod.Name)
}
}
}

func TestPVCVolumeMode(t *testing.T) {
Expand Down
3 changes: 2 additions & 1 deletion pkg/features/kube_features.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ const (

// owner: @derekwaynecarr
// beta: v1.10
// GA: v1.14
//
// Enable pods to consume pre-allocated huge pages of varying page sizes
HugePages utilfeature.Feature = "HugePages"
Expand Down Expand Up @@ -424,7 +425,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
RotateKubeletClientCertificate: {Default: true, PreRelease: utilfeature.Beta},
PersistentLocalVolumes: {Default: true, PreRelease: utilfeature.Beta},
LocalStorageCapacityIsolation: {Default: true, PreRelease: utilfeature.Beta},
HugePages: {Default: true, PreRelease: utilfeature.Beta},
HugePages: {Default: true, PreRelease: utilfeature.GA, LockToDefault: true}, // remove in 1.16
Sysctls: {Default: true, PreRelease: utilfeature.Beta},
DebugContainers: {Default: false, PreRelease: utilfeature.Alpha},
PodShareProcessNamespace: {Default: true, PreRelease: utilfeature.Beta},
Expand Down
5 changes: 0 additions & 5 deletions pkg/kubelet/cadvisor/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,9 @@ go_library(
importpath = "k8s.io/kubernetes/pkg/kubelet/cadvisor",
deps = [
"//pkg/apis/core/v1/helper:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet/types:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/github.com/google/cadvisor/events:go_default_library",
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
Expand All @@ -51,11 +49,8 @@ go_test(
embed = [":go_default_library"],
deps = select({
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/features:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
"//vendor/github.com/google/cadvisor/container/crio:go_default_library",
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
Expand Down
14 changes: 5 additions & 9 deletions pkg/kubelet/cadvisor/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,7 @@ import (
cadvisorapi2 "github.com/google/cadvisor/info/v2"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
utilfeature "k8s.io/apiserver/pkg/util/feature"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
"k8s.io/kubernetes/pkg/features"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
)

Expand All @@ -46,13 +44,11 @@ func CapacityFromMachineInfo(info *cadvisorapi.MachineInfo) v1.ResourceList {
}

// if huge pages are enabled, we report them as a schedulable resource on the node
if utilfeature.DefaultFeatureGate.Enabled(features.HugePages) {
for _, hugepagesInfo := range info.HugePages {
pageSizeBytes := int64(hugepagesInfo.PageSize * 1024)
hugePagesBytes := pageSizeBytes * int64(hugepagesInfo.NumPages)
pageSizeQuantity := resource.NewQuantity(pageSizeBytes, resource.BinarySI)
c[v1helper.HugePageResourceName(*pageSizeQuantity)] = *resource.NewQuantity(hugePagesBytes, resource.BinarySI)
}
for _, hugepagesInfo := range info.HugePages {
pageSizeBytes := int64(hugepagesInfo.PageSize * 1024)
hugePagesBytes := pageSizeBytes * int64(hugepagesInfo.NumPages)
pageSizeQuantity := resource.NewQuantity(pageSizeBytes, resource.BinarySI)
c[v1helper.HugePageResourceName(*pageSizeQuantity)] = *resource.NewQuantity(hugePagesBytes, resource.BinarySI)
}

return c
Expand Down
27 changes: 0 additions & 27 deletions pkg/kubelet/cadvisor/util_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ import (
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
"k8s.io/kubernetes/pkg/features"
)

func TestCapacityFromMachineInfoWithHugePagesEnable(t *testing.T) {
Expand All @@ -49,36 +46,12 @@ func TestCapacityFromMachineInfoWithHugePagesEnable(t *testing.T) {
v1.ResourceMemory: *resource.NewQuantity(int64(2048), resource.BinarySI),
"hugepages-5Ki": *resource.NewQuantity(int64(51200), resource.BinarySI),
}
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)()
actual := CapacityFromMachineInfo(machineInfo)
if !reflect.DeepEqual(actual, expected) {
t.Errorf("when set hugepages true, got resource list %v, want %v", actual, expected)
}
}

func TestCapacityFromMachineInfoWithHugePagesDisable(t *testing.T) {
machineInfo := &info.MachineInfo{
NumCores: 2,
MemoryCapacity: 2048,
HugePages: []info.HugePagesInfo{
{
PageSize: 5,
NumPages: 10,
},
},
}

expected := v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(int64(2000), resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(int64(2048), resource.BinarySI),
}
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)()
actual := CapacityFromMachineInfo(machineInfo)
if !reflect.DeepEqual(actual, expected) {
t.Errorf("when set hugepages false, got resource list %v, want %v", actual, expected)
}
}

func TestCrioSocket(t *testing.T) {
assert.EqualValues(t, CrioSocket, crio.CrioSocket, "CrioSocket in this package must equal the one in github.com/google/cadvisor/container/crio/client.go")
}
42 changes: 19 additions & 23 deletions pkg/kubelet/cm/cgroup_manager_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -327,9 +327,7 @@ func getSupportedSubsystems() map[subsystem]bool {
&cgroupfs.CpuGroup{}: true,
}
// not all hosts support hugetlb cgroup, and in the absent of hugetlb, we will fail silently by reporting no capacity.
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) {
supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false
}
supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportPodPidsLimit) {
supportedSubsystems[&cgroupfs.PidsGroup{}] = true
}
Expand Down Expand Up @@ -385,27 +383,25 @@ func (m *cgroupManagerImpl) toResources(resourceConfig *ResourceConfig) *libcont
}
}
// if huge pages are enabled, we set them in libcontainer
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) {
// for each page size enumerated, set that value
pageSizes := sets.NewString()
for pageSize, limit := range resourceConfig.HugePageLimit {
sizeString := units.CustomSize("%g%s", float64(pageSize), 1024.0, hugePageSizeList)
resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{
Pagesize: sizeString,
Limit: uint64(limit),
})
pageSizes.Insert(sizeString)
}
// for each page size omitted, limit to 0
for _, pageSize := range cgroupfs.HugePageSizes {
if pageSizes.Has(pageSize) {
continue
}
resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{
Pagesize: pageSize,
Limit: uint64(0),
})
// for each page size enumerated, set that value
pageSizes := sets.NewString()
for pageSize, limit := range resourceConfig.HugePageLimit {
sizeString := units.CustomSize("%g%s", float64(pageSize), 1024.0, hugePageSizeList)
resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{
Pagesize: sizeString,
Limit: uint64(limit),
})
pageSizes.Insert(sizeString)
}
// for each page size omitted, limit to 0
for _, pageSize := range cgroupfs.HugePageSizes {
if pageSizes.Has(pageSize) {
continue
}
resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{
Pagesize: pageSize,
Limit: uint64(0),
})
}
return resources
}
Expand Down
6 changes: 1 addition & 5 deletions pkg/kubelet/cm/node_container_manager_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,7 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog"
kubefeatures "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/events"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
)
Expand Down Expand Up @@ -159,9 +157,7 @@ func getCgroupConfig(rl v1.ResourceList) *ResourceConfig {
val := MilliCPUToShares(q.MilliValue())
rc.CpuShares = &val
}
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) {
rc.HugePageLimit = HugePageLimits(rl)
}
rc.HugePageLimit = HugePageLimits(rl)

return &rc
}
Expand Down
10 changes: 3 additions & 7 deletions pkg/kubelet/cm/qos_container_manager_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,7 @@ func (m *qosContainerManagerImpl) Start(getNodeAllocatable func() v1.ResourceLis
}

// for each enumerated huge page size, the qos tiers are unbounded
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) {
m.setHugePagesUnbounded(containerConfig)
}
m.setHugePagesUnbounded(containerConfig)

// check if it exists
if !cm.Exists(containerName) {
Expand Down Expand Up @@ -290,10 +288,8 @@ func (m *qosContainerManagerImpl) UpdateCgroups() error {
}

// update the qos level cgroup settings for huge pages (ensure they remain unbounded)
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) {
if err := m.setHugePagesConfig(qosConfigs); err != nil {
return err
}
if err := m.setHugePagesConfig(qosConfigs); err != nil {
return err
}

if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.QOSReserved) {
Expand Down

0 comments on commit deae071

Please sign in to comment.