Skip to content

Commit

Permalink
Merge pull request kubernetes#115813 from atiratree/controller-aliases
Browse files Browse the repository at this point in the history
Introduce controller aliases for KCM and CCM
  • Loading branch information
k8s-ci-robot authored Jun 22, 2023
2 parents 3e28404 + 85d9339 commit 4f6a750
Show file tree
Hide file tree
Showing 27 changed files with 1,035 additions and 285 deletions.
1 change: 1 addition & 0 deletions cmd/cloud-controller-manager/.import-restrictions
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ rules:
allowedPrefixes:
- k8s.io/kubernetes/cmd/kube-controller-manager/app/options
- k8s.io/kubernetes/cmd/kube-controller-manager/app/config
- k8s.io/kubernetes/cmd/kube-controller-manager/names
- k8s.io/kubernetes/pkg/api/legacyscheme
- k8s.io/kubernetes/pkg/api/service
- k8s.io/kubernetes/pkg/api/v1/pod
Expand Down
10 changes: 7 additions & 3 deletions cmd/cloud-controller-manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,14 @@ import (
cloudprovider "k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/names"
"k8s.io/cloud-provider/options"
"k8s.io/component-base/cli"
cliflag "k8s.io/component-base/cli/flag"
_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
"k8s.io/klog/v2"
kcmnames "k8s.io/kubernetes/cmd/kube-controller-manager/names"
// For existing cloud providers, the option to import legacy providers is still available.
// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
)
Expand All @@ -48,6 +50,7 @@ func main() {
}

controllerInitializers := app.DefaultInitFuncConstructors
controllerAliases := names.CCMControllerAliases()
// Here is an example to remove the controller which is not needed.
// e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need.
//delete(controllerInitializers, "cloud-node-lifecycle")
Expand All @@ -59,18 +62,19 @@ func main() {
nodeIpamController := nodeIPAMController{}
nodeIpamController.nodeIPAMControllerOptions.NodeIPAMControllerConfiguration = &nodeIpamController.nodeIPAMControllerConfiguration
fss := cliflag.NamedFlagSets{}
nodeIpamController.nodeIPAMControllerOptions.AddFlags(fss.FlagSet("nodeipam controller"))
nodeIpamController.nodeIPAMControllerOptions.AddFlags(fss.FlagSet(kcmnames.NodeIpamController))

controllerInitializers["nodeipam"] = app.ControllerInitFuncConstructor{
controllerInitializers[kcmnames.NodeIpamController] = app.ControllerInitFuncConstructor{
// "node-controller" is the shared identity of all node controllers, including node, node lifecycle, and node ipam.
// See https://github.com/kubernetes/kubernetes/pull/72764#issuecomment-453300990 for more context.
InitContext: app.ControllerInitContext{
ClientName: "node-controller",
},
Constructor: nodeIpamController.StartNodeIpamControllerWrapper,
}
controllerAliases["nodeipam"] = kcmnames.NodeIpamController

command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop)
command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, controllerAliases, fss, wait.NeverStop)
code := cli.Run(command)
os.Exit(code)
}
Expand Down
106 changes: 52 additions & 54 deletions cmd/kube-controller-manager/app/controllermanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ import (
certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/util/keyutil"
cloudprovider "k8s.io/cloud-provider"
cpnames "k8s.io/cloud-provider/names"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/cli/globalflag"
"k8s.io/component-base/configz"
Expand All @@ -73,6 +74,7 @@ import (

"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
"k8s.io/kubernetes/cmd/kube-controller-manager/names"
kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config"
serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
"k8s.io/kubernetes/pkg/serviceaccount"
Expand Down Expand Up @@ -135,7 +137,7 @@ controller, and serviceaccounts controller.`,
}
cliflag.PrintFlags(cmd.Flags())

c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List(), names.KCMControllerAliases())
if err != nil {
return err
}
Expand All @@ -154,7 +156,7 @@ controller, and serviceaccounts controller.`,
}

fs := cmd.Flags()
namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())
namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List(), names.KCMControllerAliases())
verflag.AddFlags(namedFlagSets.FlagSet("global"))
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
registerLegacyGlobalFlags(namedFlagSets)
Expand Down Expand Up @@ -407,20 +409,16 @@ func KnownControllers() []string {
// first to ensure that the SA tokens for future controllers will exist. Think very carefully before adding
// to this list.
ret.Insert(
saTokenControllerName,
names.ServiceAccountTokenController,
)

return ret.List()
}

// ControllersDisabledByDefault is the set of controllers which is disabled by default
var ControllersDisabledByDefault = sets.NewString(
"bootstrapsigner",
"tokencleaner",
)

const (
saTokenControllerName = "serviceaccount-token"
names.BootstrapSignerController,
names.TokenCleanerController,
)

// NewControllerInitializers is a public map of named controller groups (you can start more than one in an init func)
Expand All @@ -436,55 +434,55 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc
controllers[name] = fn
}

register("endpoint", startEndpointController)
register("endpointslice", startEndpointSliceController)
register("endpointslicemirroring", startEndpointSliceMirroringController)
register("replicationcontroller", startReplicationController)
register("podgc", startPodGCController)
register("resourcequota", startResourceQuotaController)
register("namespace", startNamespaceController)
register("serviceaccount", startServiceAccountController)
register("garbagecollector", startGarbageCollectorController)
register("daemonset", startDaemonSetController)
register("job", startJobController)
register("deployment", startDeploymentController)
register("replicaset", startReplicaSetController)
register("horizontalpodautoscaling", startHPAController)
register("disruption", startDisruptionController)
register("statefulset", startStatefulSetController)
register("cronjob", startCronJobController)
register("csrsigning", startCSRSigningController)
register("csrapproving", startCSRApprovingController)
register("csrcleaner", startCSRCleanerController)
register("ttl", startTTLController)
register("bootstrapsigner", startBootstrapSignerController)
register("tokencleaner", startTokenCleanerController)
register("nodeipam", startNodeIpamController)
register("nodelifecycle", startNodeLifecycleController)
register(names.EndpointsController, startEndpointController)
register(names.EndpointSliceController, startEndpointSliceController)
register(names.EndpointSliceMirroringController, startEndpointSliceMirroringController)
register(names.ReplicationControllerController, startReplicationController)
register(names.PodGarbageCollectorController, startPodGCController)
register(names.ResourceQuotaController, startResourceQuotaController)
register(names.NamespaceController, startNamespaceController)
register(names.ServiceAccountController, startServiceAccountController)
register(names.GarbageCollectorController, startGarbageCollectorController)
register(names.DaemonSetController, startDaemonSetController)
register(names.JobController, startJobController)
register(names.DeploymentController, startDeploymentController)
register(names.ReplicaSetController, startReplicaSetController)
register(names.HorizontalPodAutoscalerController, startHPAController)
register(names.DisruptionController, startDisruptionController)
register(names.StatefulSetController, startStatefulSetController)
register(names.CronJobController, startCronJobController)
register(names.CertificateSigningRequestSigningController, startCSRSigningController)
register(names.CertificateSigningRequestApprovingController, startCSRApprovingController)
register(names.CertificateSigningRequestCleanerController, startCSRCleanerController)
register(names.TTLController, startTTLController)
register(names.BootstrapSignerController, startBootstrapSignerController)
register(names.TokenCleanerController, startTokenCleanerController)
register(names.NodeIpamController, startNodeIpamController)
register(names.NodeLifecycleController, startNodeLifecycleController)
if loopMode == IncludeCloudLoops {
register("service", startServiceController)
register("route", startRouteController)
register("cloud-node-lifecycle", startCloudNodeLifecycleController)
// TODO: volume controller into the IncludeCloudLoops only set.
}
register("persistentvolume-binder", startPersistentVolumeBinderController)
register("attachdetach", startAttachDetachController)
register("persistentvolume-expander", startVolumeExpandController)
register("clusterrole-aggregation", startClusterRoleAggregrationController)
register("pvc-protection", startPVCProtectionController)
register("pv-protection", startPVProtectionController)
register("ttl-after-finished", startTTLAfterFinishedController)
register("root-ca-cert-publisher", startRootCACertPublisher)
register("ephemeral-volume", startEphemeralVolumeController)
register(cpnames.ServiceLBController, startServiceController)
register(cpnames.NodeRouteController, startRouteController)
register(cpnames.CloudNodeLifecycleController, startCloudNodeLifecycleController)
// TODO: persistent volume controllers into the IncludeCloudLoops only set.
}
register(names.PersistentVolumeBinderController, startPersistentVolumeBinderController)
register(names.PersistentVolumeAttachDetachController, startAttachDetachController)
register(names.PersistentVolumeExpanderController, startVolumeExpandController)
register(names.ClusterRoleAggregationController, startClusterRoleAggregrationController)
register(names.PersistentVolumeClaimProtectionController, startPVCProtectionController)
register(names.PersistentVolumeProtectionController, startPVProtectionController)
register(names.TTLAfterFinishedController, startTTLAfterFinishedController)
register(names.RootCACertificatePublisherController, startRootCACertPublisher)
register(names.EphemeralVolumeController, startEphemeralVolumeController)
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) &&
utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) {
register("storage-version-gc", startStorageVersionGCController)
register(names.StorageVersionGarbageCollectorController, startStorageVersionGCController)
}
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DynamicResourceAllocation) {
register("resource-claim-controller", startResourceClaimController)
register(names.ResourceClaimController, startResourceClaimController)
}
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LegacyServiceAccountTokenCleanUp) {
register("legacy-service-account-token-cleaner", startLegacySATokenCleaner)
register(names.LegacyServiceAccountTokenCleanerController, startLegacySATokenCleaner)
}

return controllers
Expand Down Expand Up @@ -655,13 +653,13 @@ type serviceAccountTokenControllerStarter struct {

func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController(ctx context.Context, controllerContext ControllerContext) (controller.Interface, bool, error) {
logger := klog.FromContext(ctx)
if !controllerContext.IsControllerEnabled(saTokenControllerName) {
logger.Info("Warning: controller is disabled", "controller", saTokenControllerName)
if !controllerContext.IsControllerEnabled(names.ServiceAccountTokenController) {
logger.Info("Warning: controller is disabled", "controller", names.ServiceAccountTokenController)
return nil, false, nil
}

if len(controllerContext.ComponentConfig.SAController.ServiceAccountKeyFile) == 0 {
logger.Info("Controller is disabled because there is no private key", "controller", saTokenControllerName)
logger.Info("Controller is disabled because there is no private key", "controller", names.ServiceAccountTokenController)
return nil, false, nil
}
privateKey, err := keyutil.PrivateKeyFromFile(controllerContext.ComponentConfig.SAController.ServiceAccountKeyFile)
Expand Down
93 changes: 93 additions & 0 deletions cmd/kube-controller-manager/app/controllermanager_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
"regexp"
"strings"
"testing"

"k8s.io/apimachinery/pkg/util/sets"
cpnames "k8s.io/cloud-provider/names"

"k8s.io/kubernetes/cmd/kube-controller-manager/names"
)

func TestControllerNamesConsistency(t *testing.T) {
controllerNameRegexp := regexp.MustCompile("^[a-z]([-a-z]*[a-z])?$")

for _, name := range KnownControllers() {
if !controllerNameRegexp.MatchString(name) {
t.Errorf("name consistency check failed: controller %q must consist of lower case alphabetic characters or '-', and must start and end with an alphabetic character", name)
}
if !strings.HasSuffix(name, "-controller") {
t.Errorf("name consistency check failed: controller %q must have \"-controller\" suffix", name)
}
}
}

func TestControllerNamesDeclaration(t *testing.T) {
declaredControllers := sets.New(
names.ServiceAccountTokenController,
names.EndpointsController,
names.EndpointSliceController,
names.EndpointSliceMirroringController,
names.ReplicationControllerController,
names.PodGarbageCollectorController,
names.ResourceQuotaController,
names.NamespaceController,
names.ServiceAccountController,
names.GarbageCollectorController,
names.DaemonSetController,
names.JobController,
names.DeploymentController,
names.ReplicaSetController,
names.HorizontalPodAutoscalerController,
names.DisruptionController,
names.StatefulSetController,
names.CronJobController,
names.CertificateSigningRequestSigningController,
names.CertificateSigningRequestApprovingController,
names.CertificateSigningRequestCleanerController,
names.TTLController,
names.BootstrapSignerController,
names.TokenCleanerController,
names.NodeIpamController,
names.NodeLifecycleController,
cpnames.ServiceLBController,
cpnames.NodeRouteController,
cpnames.CloudNodeLifecycleController,
names.PersistentVolumeBinderController,
names.PersistentVolumeAttachDetachController,
names.PersistentVolumeExpanderController,
names.ClusterRoleAggregationController,
names.PersistentVolumeClaimProtectionController,
names.PersistentVolumeProtectionController,
names.TTLAfterFinishedController,
names.RootCACertificatePublisherController,
names.EphemeralVolumeController,
names.StorageVersionGarbageCollectorController,
names.ResourceClaimController,
names.LegacyServiceAccountTokenCleanerController,
)

for _, name := range KnownControllers() {
if !declaredControllers.Has(name) {
t.Errorf("name declaration check failed: controller name %q should be declared in \"controller_names.go\" and added to this test", name)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/spf13/pflag"

"k8s.io/kubernetes/cmd/kube-controller-manager/names"
endpointslicemirroringconfig "k8s.io/kubernetes/pkg/controller/endpointslicemirroring/config"
)

Expand All @@ -44,9 +45,9 @@ func (o *EndpointSliceMirroringControllerOptions) AddFlags(fs *pflag.FlagSet) {
return
}

fs.Int32Var(&o.MirroringConcurrentServiceEndpointSyncs, "mirroring-concurrent-service-endpoint-syncs", o.MirroringConcurrentServiceEndpointSyncs, "The number of service endpoint syncing operations that will be done concurrently by the EndpointSliceMirroring controller. Larger number = faster endpoint slice updating, but more CPU (and network) load. Defaults to 5.")
fs.Int32Var(&o.MirroringMaxEndpointsPerSubset, "mirroring-max-endpoints-per-subset", o.MirroringMaxEndpointsPerSubset, "The maximum number of endpoints that will be added to an EndpointSlice by the EndpointSliceMirroring controller. More endpoints per slice will result in less endpoint slices, but larger resources. Defaults to 100.")
fs.DurationVar(&o.MirroringEndpointUpdatesBatchPeriod.Duration, "mirroring-endpointslice-updates-batch-period", o.MirroringEndpointUpdatesBatchPeriod.Duration, "The length of EndpointSlice updates batching period for EndpointSliceMirroring controller. Processing of EndpointSlice changes will be delayed by this duration to join them with potential upcoming updates and reduce the overall number of EndpointSlice updates. Larger number = higher endpoint programming latency, but lower number of endpoints revision generated")
fs.Int32Var(&o.MirroringConcurrentServiceEndpointSyncs, "mirroring-concurrent-service-endpoint-syncs", o.MirroringConcurrentServiceEndpointSyncs, fmt.Sprintf("The number of service endpoint syncing operations that will be done concurrently by the %s. Larger number = faster endpoint slice updating, but more CPU (and network) load. Defaults to 5.", names.EndpointSliceMirroringController))
fs.Int32Var(&o.MirroringMaxEndpointsPerSubset, "mirroring-max-endpoints-per-subset", o.MirroringMaxEndpointsPerSubset, fmt.Sprintf("The maximum number of endpoints that will be added to an EndpointSlice by the %s. More endpoints per slice will result in less endpoint slices, but larger resources. Defaults to 100.", names.EndpointSliceMirroringController))
fs.DurationVar(&o.MirroringEndpointUpdatesBatchPeriod.Duration, "mirroring-endpointslice-updates-batch-period", o.MirroringEndpointUpdatesBatchPeriod.Duration, fmt.Sprintf("The length of EndpointSlice updates batching period for %s. Processing of EndpointSlice changes will be delayed by this duration to join them with potential upcoming updates and reduce the overall number of EndpointSlice updates. Larger number = higher endpoint programming latency, but lower number of endpoints revision generated", names.EndpointSliceMirroringController))
}

// ApplyTo fills up EndpointSliceMirroringController config with options.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@ limitations under the License.
package options

import (
"fmt"

"github.com/spf13/pflag"

"k8s.io/kubernetes/cmd/kube-controller-manager/names"
nodelifecycleconfig "k8s.io/kubernetes/pkg/controller/nodelifecycle/config"
)

Expand All @@ -41,7 +44,7 @@ func (o *NodeLifecycleControllerOptions) AddFlags(fs *pflag.FlagSet) {
"where N means number of retries allowed for kubelet to post node status.")
fs.Float32Var(&o.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.")
fs.Float32Var(&o.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.")
fs.Int32Var(&o.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, "Number of nodes from which NodeController treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller.")
fs.Int32Var(&o.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, fmt.Sprintf("Number of nodes from which %s treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller.", names.NodeLifecycleController))
fs.Float32Var(&o.UnhealthyZoneThreshold, "unhealthy-zone-threshold", 0.55, "Fraction of Nodes in a zone which needs to be not Ready (minimum 3) for zone to be treated as unhealthy. ")
}

Expand Down
Loading

0 comments on commit 4f6a750

Please sign in to comment.