Merge pull request openshift#1592 from enxebre/cp-infra-nodes

Introduce node selector support for HostedClusters
gnufied · Aug 2, 2022 · aadcf32 · aadcf32
2 parents 48e2402 + 2434cdf
commit aadcf32
Show file tree

Hide file tree

Showing 32 changed files with 423 additions and 307 deletions.
diff --git a/api/v1alpha1/hosted_controlplane.go b/api/v1alpha1/hosted_controlplane.go
@@ -177,6 +177,11 @@ type HostedControlPlaneSpec struct {
 	//
 	// +optional
 	Autoscaling ClusterAutoscaling `json:"autoscaling,omitempty"`
+
+	// NodeSelector when specified, must be true for the pods managed by the HostedCluster to be scheduled.
+	//
+	// +optional
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
 }
 
 // AvailabilityPolicy specifies a high level availability policy for components.

diff --git a/api/v1alpha1/hostedcluster_types.go b/api/v1alpha1/hostedcluster_types.go
@@ -290,6 +290,11 @@ type HostedClusterSpec struct {
 	// +optional
 	// +immutable
 	OLMCatalogPlacement OLMCatalogPlacement `json:"olmCatalogPlacement,omitempty"`
+
+	// NodeSelector when specified, must be true for the pods managed by the HostedCluster to be scheduled.
+	//
+	// +optional
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
 }
 
 // OLMCatalogPlacement is an enum specifying the placement of OLM catalog components.

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml b/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml
@@ -2130,6 +2130,12 @@ spec:
                 required:
                 - networkType
                 type: object
+              nodeSelector:
+                additionalProperties:
+                  type: string
+                description: NodeSelector when specified, must be true for the pods
+                  managed by the HostedCluster to be scheduled.
+                type: object
               olmCatalogPlacement:
                 default: management
                 description: OLMCatalogPlacement specifies the placement of OLM catalog

diff --git a/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml b/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml
@@ -2125,6 +2125,12 @@ spec:
                 required:
                 - networkType
                 type: object
+              nodeSelector:
+                additionalProperties:
+                  type: string
+                description: NodeSelector when specified, must be true for the pods
+                  managed by the HostedCluster to be scheduled.
+                type: object
               olmCatalogPlacement:
                 default: management
                 description: OLMCatalogPlacement specifies the placement of OLM catalog

diff --git a/control-plane-operator/controllers/hostedcontrolplane/autoscaler/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/autoscaler/reconcile.go
@@ -169,17 +169,14 @@ func ReconcileAutoscalerDeployment(deployment *appsv1.Deployment, hcp *hyperv1.H
 	util.AvailabilityProber(kas.InClusterKASReadyURL(deployment.Namespace, util.APIPort(hcp)), availabilityProberImage, &deployment.Spec.Template.Spec)
 
 	deploymentConfig := config.DeploymentConfig{
-		Replicas: 1,
 		Scheduling: config.Scheduling{
 			PriorityClass: config.DefaultPriorityClass,
 		},
 		SetDefaultSecurityContext: setDefaultSecurityContext,
 	}
 
-	deploymentConfig.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
-	deploymentConfig.SetColocation(hcp)
+	deploymentConfig.SetDefaults(hcp, nil, k8sutilspointer.Int(1))
 	deploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
-	deploymentConfig.SetControlPlaneIsolation(hcp)
 	deploymentConfig.ApplyTo(deployment)
 
 	return nil

diff --git a/control-plane-operator/controllers/hostedcontrolplane/clusterpolicy/params.go b/control-plane-operator/controllers/hostedcontrolplane/clusterpolicy/params.go
@@ -40,21 +40,13 @@ func NewClusterPolicyControllerParams(hcp *hyperv1.HostedControlPlane, images ma
 			},
 		},
 	}
-	params.DeploymentConfig.SetColocation(hcp)
-	params.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
-	params.DeploymentConfig.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
-	params.DeploymentConfig.SetControlPlaneIsolation(hcp)
-	switch hcp.Spec.ControllerAvailabilityPolicy {
-	case hyperv1.HighlyAvailable:
-		params.DeploymentConfig.Replicas = 3
-		params.DeploymentConfig.SetMultizoneSpread(clusterPolicyControllerLabels)
-	default:
-		params.DeploymentConfig.Replicas = 1
-	}
-	params.OwnerRef = config.OwnerRefFrom(hcp)
 
+	params.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
+	params.DeploymentConfig.SetDefaults(hcp, clusterPolicyControllerLabels, nil)
 	params.DeploymentConfig.SetDefaultSecurityContext = setDefaultSecurityContext
 
+	params.OwnerRef = config.OwnerRefFrom(hcp)
+
 	return params
 }
 

diff --git a/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go b/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go
@@ -90,11 +90,8 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, images map[strin
 	}
 
 	p.DeploymentConfig.Scheduling.PriorityClass = config.DefaultPriorityClass
-	p.DeploymentConfig.SetColocation(hcp)
 	p.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
-	p.DeploymentConfig.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
-	p.DeploymentConfig.SetControlPlaneIsolation(hcp)
-	p.DeploymentConfig.Replicas = 1
+	p.DeploymentConfig.SetDefaults(hcp, nil, utilpointer.IntPtr(1))
 	p.DeploymentConfig.SetDefaultSecurityContext = setDefaultSecurityContext
 	if util.IsPrivateHCP(hcp) {
 		p.APIServerAddress = fmt.Sprintf("api.%s.hypershift.local", hcp.Name)

diff --git a/control-plane-operator/controllers/hostedcontrolplane/configoperator/params.go b/control-plane-operator/controllers/hostedcontrolplane/configoperator/params.go
@@ -3,6 +3,8 @@ package configoperator
 import (
 	"context"
 
+	utilpointer "k8s.io/utils/pointer"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -31,7 +33,6 @@ func NewHostedClusterConfigOperatorParams(ctx context.Context, hcp *hyperv1.Host
 		KubernetesVersion:       kubernetesVersion,
 		AvailabilityProberImage: images[util.AvailabilityProberImageName],
 	}
-	params.Replicas = 1
 	params.Scheduling = config.Scheduling{
 		PriorityClass: config.DefaultPriorityClass,
 	}
@@ -76,11 +77,10 @@ func NewHostedClusterConfigOperatorParams(ctx context.Context, hcp *hyperv1.Host
 			TimeoutSeconds:      5,
 		},
 	}
-	params.DeploymentConfig.SetColocation(hcp)
+
 	params.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
-	params.DeploymentConfig.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
-	params.DeploymentConfig.SetControlPlaneIsolation(hcp)
-	params.SetDefaultSecurityContext = setDefaultSecurityContext
+	params.DeploymentConfig.SetDefaults(hcp, nil, utilpointer.IntPtr(1))
+	params.DeploymentConfig.SetDefaultSecurityContext = setDefaultSecurityContext
 
 	return params
 }
diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/params.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/params.go
@@ -3,6 +3,7 @@ package cvo
 import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	utilpointer "k8s.io/utils/pointer"
 
 	hyperv1 "github.com/openshift/hypershift/api/v1alpha1"
 
@@ -42,11 +43,9 @@ func NewCVOParams(hcp *hyperv1.HostedControlPlane, images map[string]string, set
 		},
 	}
 	p.DeploymentConfig.Scheduling.PriorityClass = config.DefaultPriorityClass
-	p.DeploymentConfig.SetColocation(hcp)
+
 	p.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
-	p.DeploymentConfig.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
-	p.DeploymentConfig.SetControlPlaneIsolation(hcp)
-	p.DeploymentConfig.Replicas = 1
+	p.DeploymentConfig.SetDefaults(hcp, nil, utilpointer.IntPtr(1))
 	p.DeploymentConfig.SetDefaultSecurityContext = setDefaultSecurityContext
 
 	return p

diff --git a/control-plane-operator/controllers/hostedcontrolplane/etcd/params.go b/control-plane-operator/controllers/hostedcontrolplane/etcd/params.go
@@ -46,16 +46,7 @@ func NewEtcdParams(hcp *hyperv1.HostedControlPlane, images map[string]string) *E
 	}
 	p.DeploymentConfig.AdditionalLabels[hyperv1.ControlPlaneComponent] = "etcd"
 	p.DeploymentConfig.Scheduling.PriorityClass = config.EtcdPriorityClass
-	p.DeploymentConfig.SetMultizoneSpread(etcdPodSelector())
-	p.DeploymentConfig.SetControlPlaneIsolation(hcp)
-	p.DeploymentConfig.SetColocationAnchor(hcp)
-
-	switch hcp.Spec.ControllerAvailabilityPolicy {
-	case hyperv1.HighlyAvailable:
-		p.DeploymentConfig.Replicas = 3
-	default:
-		p.DeploymentConfig.Replicas = 1
-	}
+	p.DeploymentConfig.SetDefaults(hcp, etcdPodSelector(), nil)
 
 	if hcp.Spec.Etcd.Managed == nil {
 		hcp.Spec.Etcd.Managed = &hyperv1.ManagedEtcdSpec{

diff --git a/control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go b/control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go
@@ -383,23 +383,10 @@ func ReconcileIgnitionServer(ctx context.Context,
 			}
 		}
 		hyperutil.SetRestartAnnotation(hcp.ObjectMeta, ignitionServerDeployment)
-		hyperutil.SetColocation(hcp.ObjectMeta, ignitionServerDeployment)
-		hyperutil.SetControlPlaneIsolation(hcp.ObjectMeta, ignitionServerDeployment)
 		hyperutil.SetDefaultPriorityClass(ignitionServerDeployment)
-		switch hcp.Spec.ControllerAvailabilityPolicy {
-		case hyperv1.HighlyAvailable:
-			maxSurge := intstr.FromInt(1)
-			maxUnavailable := intstr.FromInt(1)
-			ignitionServerDeployment.Spec.Strategy.Type = appsv1.RollingUpdateDeploymentStrategyType
-			ignitionServerDeployment.Spec.Strategy.RollingUpdate = &appsv1.RollingUpdateDeployment{
-				MaxSurge:       &maxSurge,
-				MaxUnavailable: &maxUnavailable,
-			}
-			ignitionServerDeployment.Spec.Replicas = utilpointer.Int32Ptr(3)
-			hyperutil.SetMultizoneSpread(ignitionServerLabels, ignitionServerDeployment)
-		default:
-			ignitionServerDeployment.Spec.Replicas = utilpointer.Int32(1)
-		}
+		deploymentConfig := config.DeploymentConfig{}
+		deploymentConfig.SetDefaults(hcp, ignitionServerLabels, nil)
+		deploymentConfig.ApplyTo(ignitionServerDeployment)
 
 		return nil
 	}); err != nil {

diff --git a/control-plane-operator/controllers/hostedcontrolplane/ingress/router.go b/control-plane-operator/controllers/hostedcontrolplane/ingress/router.go
@@ -73,17 +73,8 @@ func PrivateRouterConfig(hcp *hyperv1.HostedControlPlane, setDefaultSecurityCont
 		},
 	}
 	cfg.Scheduling.PriorityClass = config.APICriticalPriorityClass
-	cfg.SetColocation(hcp)
+	cfg.SetDefaults(hcp, privateRouterLabels(), nil)
 	cfg.SetRestartAnnotation(hcp.ObjectMeta)
-	cfg.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
-	cfg.SetControlPlaneIsolation(hcp)
-	cfg.SetMultizoneSpread(privateRouterLabels())
-	switch hcp.Spec.ControllerAvailabilityPolicy {
-	case hyperv1.HighlyAvailable:
-		cfg.Replicas = 3
-	default:
-		cfg.Replicas = 1
-	}
 	cfg.SetDefaultSecurityContext = setDefaultSecurityContext
 	return cfg
 }

diff --git a/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go b/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go
@@ -48,11 +48,8 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, images map[strin
 		Platform:                platform,
 	}
 	p.DeploymentConfig.Scheduling.PriorityClass = config.DefaultPriorityClass
-	p.DeploymentConfig.SetColocation(hcp)
 	p.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
-	p.DeploymentConfig.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
-	p.DeploymentConfig.SetControlPlaneIsolation(hcp)
-	p.DeploymentConfig.Replicas = 1
+	p.DeploymentConfig.SetDefaults(hcp, nil, utilpointer.IntPtr(1))
 	p.DeploymentConfig.SetDefaultSecurityContext = setDefaultSecurityContext
 	p.DeploymentConfig.ReadinessProbes = config.ReadinessProbes{
 		ingressOperatorContainerName: {

diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go
@@ -10,7 +10,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/pointer"
 
 	configv1 "github.com/openshift/api/config/v1"
@@ -112,10 +111,6 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment,
 	}
 	configHash := util.ComputeHash(configBytes)
 
-	ownerRef.ApplyTo(deployment)
-	maxSurge := intstr.FromInt(3)
-	maxUnavailable := intstr.FromInt(0)
-
 	// preserve existing resource requirements for main KAS container
 	mainContainer := util.FindContainer(kasContainerMain().Name, deployment.Spec.Template.Spec.Containers)
 	if mainContainer != nil {
@@ -126,13 +121,7 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment,
 			MatchLabels: kasLabels(),
 		}
 	}
-	deployment.Spec.Strategy = appsv1.DeploymentStrategy{
-		Type: appsv1.RollingUpdateDeploymentStrategyType,
-		RollingUpdate: &appsv1.RollingUpdateDeployment{
-			MaxSurge:       &maxSurge,
-			MaxUnavailable: &maxUnavailable,
-		},
-	}
+
 	deployment.Spec.Template = corev1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: kasLabels(),
@@ -285,6 +274,7 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment,
 			// nothing needed to be done
 		}
 	}
+	ownerRef.ApplyTo(deployment)
 	deploymentConfig.ApplyTo(deployment)
 	return nil
 }

diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/params.go b/control-plane-operator/controllers/hostedcontrolplane/kas/params.go
@@ -273,10 +273,6 @@ func NewKubeAPIServerParams(ctx context.Context, hcp *hyperv1.HostedControlPlane
 			},
 		},
 	}
-	params.DeploymentConfig.SetColocation(hcp)
-	params.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
-	params.DeploymentConfig.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
-	params.DeploymentConfig.SetControlPlaneIsolation(hcp)
 
 	switch hcp.Spec.Platform.Type {
 	case hyperv1.AWSPlatform:
@@ -297,17 +293,12 @@ func NewKubeAPIServerParams(ctx context.Context, hcp *hyperv1.HostedControlPlane
 		params.Images.IBMCloudKMS = hcp.Annotations[hyperv1.IBMCloudKMSProviderImage]
 	}
 
-	switch hcp.Spec.ControllerAvailabilityPolicy {
-	case hyperv1.HighlyAvailable:
-		params.Replicas = 3
-		params.DeploymentConfig.SetMultizoneSpread(kasLabels())
-	default:
-		params.Replicas = 1
-	}
 	params.KubeConfigRef = hcp.Spec.KubeConfig
 	params.OwnerRef = config.OwnerRefFrom(hcp)
 
-	params.SetDefaultSecurityContext = setDefaultSecurityContext
+	params.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
+	params.DeploymentConfig.SetDefaults(hcp, kasLabels(), nil)
+	params.DeploymentConfig.SetDefaultSecurityContext = setDefaultSecurityContext
 
 	return params
 }

diff --git a/control-plane-operator/controllers/hostedcontrolplane/kcm/params.go b/control-plane-operator/controllers/hostedcontrolplane/kcm/params.go
@@ -99,10 +99,9 @@ func NewKubeControllerManagerParams(ctx context.Context, hcp *hyperv1.HostedCont
 			},
 		},
 	}
-	params.DeploymentConfig.SetColocation(hcp)
+	params.DeploymentConfig.SetDefaults(hcp, kcmLabels(), nil)
 	params.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
-	params.DeploymentConfig.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
-	params.DeploymentConfig.SetControlPlaneIsolation(hcp)
+
 	switch hcp.Spec.Platform.Type {
 	case hyperv1.AWSPlatform:
 		params.CloudProvider = aws.Provider
@@ -113,14 +112,6 @@ func NewKubeControllerManagerParams(ctx context.Context, hcp *hyperv1.HostedCont
 		params.CloudProviderConfig = &corev1.LocalObjectReference{Name: manifests.AzureProviderConfigWithCredentials("").Name}
 	}
 
-	switch hcp.Spec.ControllerAvailabilityPolicy {
-	case hyperv1.HighlyAvailable:
-		params.Replicas = 3
-		params.DeploymentConfig.SetMultizoneSpread(kcmLabels())
-	default:
-		params.Replicas = 1
-	}
-
 	params.SetDefaultSecurityContext = setDefaultSecurityContext
 
 	params.OwnerRef = config.OwnerRefFrom(hcp)