OCPBUGS-34816: Block data plane HC configuration requests

Added ValidatingAdmissionPolicies to the KAS's HostedCluster to block all configuration requests managed in the management cluster via the HostedCluster object. Enabling the ValidatingAdmissionPolicy feature gate in the KAS allows us to block certain requests from the Data Plane side without using a webhook. To achieve this, we created the necessary resources (Certificate, RBAC, etc.) that allow HCCO to be identified with a specific user. This designated user will be the only one authorized to execute the required requests. All other users, including those using the admin-kubeconfig, will be blocked by the VAPs validation Pull Request Content: - Enabled ValidatingAdmissionPolicies (VAP) FeatureGate on the KAS - Added a new user (via certificates) to HCCO - Implemented a new reconciler on HCCO for VAPs managed by Hypershift - Updated Kubernetes dependency to v0.30.2 Signed-off-by: Juan Manuel Parrilla Madrid <jparrill@redhat.com>
openshift · Jul 3, 2024 · 253cfbf · 253cfbf
1 parent 049c72e
commit 253cfbf
Show file tree

Hide file tree

Showing 24 changed files with 437 additions and 30 deletions.
diff --git a/Makefile b/Makefile
@@ -45,7 +45,7 @@ build: hypershift-operator control-plane-operator control-plane-pki-operator hyp
 
 .PHONY: sync
 sync:
-	$(GO) work sync 
+	$(GO) work sync
 
 .PHONY: update
 update: sync api-deps api api-docs deps clients app-sre-saas-template

diff --git a/cmd/infra/aws/delegating_client.go b/cmd/infra/aws/delegating_client.go
@@ -21,7 +21,7 @@ import (
 )
 
 // NewDelegatingClient creates a new set of AWS service clients that delegate individual calls to the right credentials.
-func NewDelegatingClient(
+func NewDelegatingClient (
 	awsEbsCsiDriverControllerCredentialsFile string,
 	cloudControllerCredentialsFile string,
 	cloudNetworkConfigControllerCredentialsFile string,
@@ -50,8 +50,8 @@ func NewDelegatingClient(
 		Fn:   request.MakeAddToUserAgentHandler("openshift.io hypershift", "cloud-controller"),
 	})
 	cloudController := &cloudControllerClientDelegate{
-		ec2Client:   ec2.New(cloudControllerSession, awsConfig),
-		elbClient:   elb.New(cloudControllerSession, awsConfig),
+		ec2Client: ec2.New(cloudControllerSession, awsConfig),
+		elbClient: elb.New(cloudControllerSession, awsConfig),
 		elbv2Client: elbv2.New(cloudControllerSession, awsConfig),
 	}
 	cloudNetworkConfigControllerSession, err := session.NewSessionWithOptions(session.Options{SharedConfigFiles: []string{cloudNetworkConfigControllerCredentialsFile}})
@@ -74,7 +74,7 @@ func NewDelegatingClient(
 		Fn:   request.MakeAddToUserAgentHandler("openshift.io hypershift", "control-plane-operator"),
 	})
 	controlPlaneOperator := &controlPlaneOperatorClientDelegate{
-		ec2Client:     ec2.New(controlPlaneOperatorSession, awsConfig),
+		ec2Client: ec2.New(controlPlaneOperatorSession, awsConfig),
 		route53Client: route53.New(controlPlaneOperatorSession, awsConfig),
 	}
 	nodePoolSession, err := session.NewSessionWithOptions(session.Options{SharedConfigFiles: []string{nodePoolCredentialsFile}})
@@ -101,39 +101,38 @@ func NewDelegatingClient(
 	}
 	return &DelegatingClient{
 		EC2API: &ec2Client{
-			EC2API:                       nil,
-			awsEbsCsiDriverController:    awsEbsCsiDriverController,
-			cloudController:              cloudController,
+			EC2API: nil,
+			awsEbsCsiDriverController: awsEbsCsiDriverController,
+			cloudController: cloudController,
 			cloudNetworkConfigController: cloudNetworkConfigController,
-			controlPlaneOperator:         controlPlaneOperator,
-			nodePool:                     nodePool,
+			controlPlaneOperator: controlPlaneOperator,
+			nodePool: nodePool,
 		},
 		ELBAPI: &elbClient{
-			ELBAPI:          nil,
+			ELBAPI: nil,
 			cloudController: cloudController,
 		},
 		ELBV2API: &elbv2Client{
-			ELBV2API:        nil,
+			ELBV2API: nil,
 			cloudController: cloudController,
 		},
 		Route53API: &route53Client{
-			Route53API:           nil,
+			Route53API: nil,
 			controlPlaneOperator: controlPlaneOperator,
 		},
 		S3API: &s3Client{
-			S3API:                  nil,
+			S3API: nil,
 			openshiftImageRegistry: openshiftImageRegistry,
 		},
 	}, nil
 }
-
 type awsEbsCsiDriverControllerClientDelegate struct {
 	ec2Client ec2iface.EC2API
 }
 
 type cloudControllerClientDelegate struct {
-	ec2Client   ec2iface.EC2API
-	elbClient   elbiface.ELBAPI
+	ec2Client ec2iface.EC2API
+	elbClient elbiface.ELBAPI
 	elbv2Client elbv2iface.ELBV2API
 }
 
@@ -142,7 +141,7 @@ type cloudNetworkConfigControllerClientDelegate struct {
 }
 
 type controlPlaneOperatorClientDelegate struct {
-	ec2Client     ec2iface.EC2API
+	ec2Client ec2iface.EC2API
 	route53Client route53iface.Route53API
 }
 
@@ -154,6 +153,7 @@ type openshiftImageRegistryClientDelegate struct {
 	s3Client s3iface.S3API
 }
 
+
 // DelegatingClient embeds clients for AWS services we have privileges to use with guest cluster component roles.
 type DelegatingClient struct {
 	ec2iface.EC2API
@@ -163,16 +163,17 @@ type DelegatingClient struct {
 	s3iface.S3API
 }
 
+
 // ec2Client delegates to individual component clients for API calls we know those components will have privileges to make.
 type ec2Client struct {
 	// embedding this fulfills the interface and falls back to a panic for APIs we don't have privileges for
 	ec2iface.EC2API
 
-	awsEbsCsiDriverController    *awsEbsCsiDriverControllerClientDelegate
-	cloudController              *cloudControllerClientDelegate
+	awsEbsCsiDriverController *awsEbsCsiDriverControllerClientDelegate
+	cloudController *cloudControllerClientDelegate
 	cloudNetworkConfigController *cloudNetworkConfigControllerClientDelegate
-	controlPlaneOperator         *controlPlaneOperatorClientDelegate
-	nodePool                     *nodePoolClientDelegate
+	controlPlaneOperator *controlPlaneOperatorClientDelegate
+	nodePool *nodePoolClientDelegate
 }
 
 func (c *ec2Client) AttachVolumeWithContext(ctx aws.Context, input *ec2.AttachVolumeInput, opts ...request.Option) (*ec2.VolumeAttachment, error) {
@@ -593,3 +594,5 @@ func (c *s3Client) PutObjectWithContext(ctx aws.Context, input *s3.PutObjectInpu
 func (c *s3Client) PutPublicAccessBlockWithContext(ctx aws.Context, input *s3.PutPublicAccessBlockInput, opts ...request.Option) (*s3.PutPublicAccessBlockOutput, error) {
 	return c.openshiftImageRegistry.s3Client.PutPublicAccessBlockWithContext(ctx, input, opts...)
 }
+
+
diff --git a/control-plane-operator/controllers/hostedcontrolplane/configoperator/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/configoperator/reconcile.go
@@ -232,6 +232,7 @@ func ReconcileRole(role *rbacv1.Role, ownerRef config.OwnerRef, platform hyperv1
 			},
 		}...)
 	}
+	// TODO (jparrill): Add RBAC specific needs for Agent platform
 	return nil
 }
 
@@ -443,7 +444,7 @@ func buildHCCContainerMain(image, hcpName, openShiftVersion, kubeVersion string,
 
 func buildHCCVolumeKubeconfig(v *corev1.Volume) {
 	v.Secret = &corev1.SecretVolumeSource{
-		SecretName:  manifests.KASServiceKubeconfigSecret("").Name,
+		SecretName:  manifests.HCCOKubeconfigSecret("").Name,
 		DefaultMode: pointer.Int32(0640),
 	}
 }

diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go
@@ -2776,6 +2776,10 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex
 	if err := r.Get(ctx, client.ObjectKeyFromObject(bootstrapClientCertSecret), bootstrapClientCertSecret); err != nil {
 		return fmt.Errorf("failed to get bootstrap client cert secret: %w", err)
 	}
+	hccoClientCertSecret := manifests.HCCOClientCertSecret(hcp.Namespace)
+	if err := r.Get(ctx, client.ObjectKeyFromObject(hccoClientCertSecret), hccoClientCertSecret); err != nil {
+		return fmt.Errorf("failed to get HCCO client cert secret: %w", err)
+	}
 
 	serviceKubeconfigSecret := manifests.KASServiceKubeconfigSecret(hcp.Namespace)
 	if _, err := createOrUpdate(ctx, r, serviceKubeconfigSecret, func() error {
@@ -2795,6 +2799,13 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex
 		return fmt.Errorf("failed to reconcile CAPI service admin kubeconfig secret: %w", err)
 	}
 
+	hccoKubeconfigSecret := manifests.HCCOKubeconfigSecret(hcp.Namespace)
+	if _, err := createOrUpdate(ctx, r, hccoKubeconfigSecret, func() error {
+		return kas.ReconcileHCCOKubeconfigSecret(hccoKubeconfigSecret, hccoClientCertSecret, rootCA, p.OwnerRef, hcp.Spec.Platform.Type)
+	}); err != nil {
+		return fmt.Errorf("failed to reconcile HCCO kubeconfig secret: %w", err)
+	}
+
 	localhostKubeconfigSecret := manifests.KASLocalhostKubeconfigSecret(hcp.Namespace)
 	if _, err := createOrUpdate(ctx, r, localhostKubeconfigSecret, func() error {
 		return kas.ReconcileLocalhostKubeconfigSecret(localhostKubeconfigSecret, clientCertSecret, rootCA, p.OwnerRef, p.KASPodPort)

diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/config.go b/control-plane-operator/controllers/hostedcontrolplane/kas/config.go
@@ -183,6 +183,7 @@ func generateConfig(p KubeAPIServerConfigParams) *kcpv1.KubeAPIServerConfig {
 	// TODO remove in 4.16 once we're able to have different featuregates for hypershift
 	featureGates := append([]string{}, p.FeatureGates...)
 	featureGates = append(featureGates, "StructuredAuthenticationConfiguration=true")
+	featureGates = append(featureGates, "ValidatingAdmissionPolicy=true")
 	args.Set("feature-gates", featureGates...)
 	args.Set("goaway-chance", "0")
 	args.Set("http2-max-streams-per-connection", "2000")

diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go
@@ -756,6 +756,22 @@ cat <<EOF >/tmp/manifests/99_feature-gate.yaml
 %[3]s
 EOF
 
+touch /tmp/manifests/hcco-rolebinding.yaml
+cat <<EOF >/tmp/manifests/hcco-rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: hcco-cluster-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:hosted-cluster-config-operator
+EOF
+
 /usr/bin/render \
    --asset-output-dir /tmp/output \
    --rendered-manifest-dir=/tmp/manifests \

diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/kubeconfig.go b/control-plane-operator/controllers/hostedcontrolplane/kas/kubeconfig.go
@@ -58,3 +58,8 @@ func ReconcileExternalKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.C
 func ReconcileBootstrapKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.ConfigMap, ownerRef config.OwnerRef, externalURL string) error {
 	return pki.ReconcileKubeConfig(secret, cert, ca, externalURL, "", manifests.KubeconfigScopeBootstrap, ownerRef)
 }
+
+func ReconcileHCCOKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.ConfigMap, ownerRef config.OwnerRef, platformType hyperv1.PlatformType) error {
+	svcURL := InClusterKASURL(platformType)
+	return pki.ReconcileKubeConfig(secret, cert, ca, svcURL, "", "service", ownerRef)
+}
diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go b/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go
@@ -159,6 +159,29 @@ func (r *HostedControlPlaneReconciler) setupKASClientSigners(
 		return err
 	}
 
+	// ----------
+	//	HCCO signer
+	// ----------
+
+	hccoKubeconfigSigner, err := reconcileSigner(
+		manifests.HCCOSigner(hcp.Namespace),
+		pki.ReconcileHCCOSigner,
+	)
+
+	if err != nil {
+		return err
+	}
+	totalClientCABundle = append(totalClientCABundle, hccoKubeconfigSigner)
+
+	// system:hosted-cluster-config-operator client cert
+	if _, err := reconcileSub(
+		manifests.HCCOClientCertSecret(hcp.Namespace),
+		hccoKubeconfigSigner,
+		pki.ReconcileHCCOClientCertSecret,
+	); err != nil {
+		return err
+	}
+
 	// ----------
 	//	CSR signer
 	// ----------

diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go
@@ -63,6 +63,15 @@ func KASServiceCAPIKubeconfigSecret(controlPlaneNamespace, infraID string) *core
 	}
 }
 
+func HCCOKubeconfigSecret(controlPlaneNamespace string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "hcco-kubeconfig",
+			Namespace: controlPlaneNamespace,
+		},
+	}
+}
+
 func KASExternalKubeconfigSecret(controlPlaneNamespace string, ref *hyperv1.KubeconfigSecretRef) *corev1.Secret {
 	s := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{

diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go
@@ -224,6 +224,14 @@ func SystemAdminClientCertSecret(ns string) *corev1.Secret {
 	return secretFor(ns, "system-admin-client")
 }
 
+func HCCOSigner(ns string) *corev1.Secret {
+	return secretFor(ns, "hcco-signer")
+}
+
+func HCCOClientCertSecret(ns string) *corev1.Secret {
+	return secretFor(ns, "hcco-client")
+}
+
 func KASMachineBootstrapClientCertSecret(ns string) *corev1.Secret {
 	return secretFor(ns, "kas-bootstrap-client")
 }

diff --git a/control-plane-operator/controllers/hostedcontrolplane/pki/ca.go b/control-plane-operator/controllers/hostedcontrolplane/pki/ca.go
@@ -49,6 +49,10 @@ func ReconcileAdminKubeconfigSigner(secret *corev1.Secret, ownerRef config.Owner
 	return reconcileSelfSignedCA(secret, ownerRef, "admin-kubeconfig-signer", "openshift")
 }
 
+func ReconcileHCCOSigner(secret *corev1.Secret, ownerRef config.OwnerRef) error {
+	return reconcileSelfSignedCA(secret, ownerRef, "hcco-signer", "openshift")
+}
+
 func ReconcileKubeCSRSigner(secret *corev1.Secret, ownerRef config.OwnerRef) error {
 	return reconcileSelfSignedCA(secret, ownerRef, "kube-csr-signer", "openshift")
 }

diff --git a/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go b/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go
@@ -90,6 +90,10 @@ func ReconcileSystemAdminClientCertSecret(secret, ca *corev1.Secret, ownerRef co
 	return reconcileSignedCert(secret, ca, ownerRef, "system:admin", []string{"system:masters"}, X509UsageClientAuth)
 }
 
+func ReconcileHCCOClientCertSecret(secret, ca *corev1.Secret, ownerRef config.OwnerRef) error {
+	return reconcileSignedCert(secret, ca, ownerRef, fmt.Sprintf("system:%s", config.HCCOUser), []string{"kubernetes"}, X509UsageClientAuth)
+}
+
 func ReconcileServiceAccountKubeconfig(secret, csrSigner *corev1.Secret, ca *corev1.ConfigMap, hcp *hyperv1.HostedControlPlane, serviceAccountNamespace, serviceAccountName string) error {
 	cn := serviceaccount.MakeUsername(serviceAccountNamespace, serviceAccountName)
 	if err := reconcileSignedCert(secret, csrSigner, config.OwnerRef{}, cn, serviceaccount.MakeGroupNames(serviceAccountNamespace), X509UsageClientAuth); err != nil {