OCPBUGS-34816: Configure user for HCCO

Until now, all the operators has been using the system:admin user (based on certificate). For security and feature reasons I've create and managed separatedly the HCCO authentication configuring a new user with concrete permissions for this operator.
openshift · Jul 30, 2024 · 64045a3 · 64045a3
1 parent c65722a
commit 64045a3
Show file tree

Hide file tree

Showing 13 changed files with 114 additions and 24 deletions.
diff --git a/Makefile b/Makefile
@@ -49,7 +49,7 @@ build: hypershift-operator control-plane-operator control-plane-pki-operator hyp
 
 .PHONY: sync
 sync:
-	$(GO) work sync 
+	$(GO) work sync
 
 .PHONY: update
 update: sync api-deps api api-docs deps clients app-sre-saas-template

diff --git a/cmd/infra/aws/delegating_client.go b/cmd/infra/aws/delegating_client.go
@@ -21,7 +21,7 @@ import (
 )
 
 // NewDelegatingClient creates a new set of AWS service clients that delegate individual calls to the right credentials.
-func NewDelegatingClient(
+func NewDelegatingClient (
 	awsEbsCsiDriverControllerCredentialsFile string,
 	cloudControllerCredentialsFile string,
 	cloudNetworkConfigControllerCredentialsFile string,
@@ -50,8 +50,8 @@ func NewDelegatingClient(
 		Fn:   request.MakeAddToUserAgentHandler("openshift.io hypershift", "cloud-controller"),
 	})
 	cloudController := &cloudControllerClientDelegate{
-		ec2Client:   ec2.New(cloudControllerSession, awsConfig),
-		elbClient:   elb.New(cloudControllerSession, awsConfig),
+		ec2Client: ec2.New(cloudControllerSession, awsConfig),
+		elbClient: elb.New(cloudControllerSession, awsConfig),
 		elbv2Client: elbv2.New(cloudControllerSession, awsConfig),
 	}
 	cloudNetworkConfigControllerSession, err := session.NewSessionWithOptions(session.Options{SharedConfigFiles: []string{cloudNetworkConfigControllerCredentialsFile}})
@@ -74,7 +74,7 @@ func NewDelegatingClient(
 		Fn:   request.MakeAddToUserAgentHandler("openshift.io hypershift", "control-plane-operator"),
 	})
 	controlPlaneOperator := &controlPlaneOperatorClientDelegate{
-		ec2Client:     ec2.New(controlPlaneOperatorSession, awsConfig),
+		ec2Client: ec2.New(controlPlaneOperatorSession, awsConfig),
 		route53Client: route53.New(controlPlaneOperatorSession, awsConfig),
 	}
 	nodePoolSession, err := session.NewSessionWithOptions(session.Options{SharedConfigFiles: []string{nodePoolCredentialsFile}})
@@ -101,39 +101,38 @@ func NewDelegatingClient(
 	}
 	return &DelegatingClient{
 		EC2API: &ec2Client{
-			EC2API:                       nil,
-			awsEbsCsiDriverController:    awsEbsCsiDriverController,
-			cloudController:              cloudController,
+			EC2API: nil,
+			awsEbsCsiDriverController: awsEbsCsiDriverController,
+			cloudController: cloudController,
 			cloudNetworkConfigController: cloudNetworkConfigController,
-			controlPlaneOperator:         controlPlaneOperator,
-			nodePool:                     nodePool,
+			controlPlaneOperator: controlPlaneOperator,
+			nodePool: nodePool,
 		},
 		ELBAPI: &elbClient{
-			ELBAPI:          nil,
+			ELBAPI: nil,
 			cloudController: cloudController,
 		},
 		ELBV2API: &elbv2Client{
-			ELBV2API:        nil,
+			ELBV2API: nil,
 			cloudController: cloudController,
 		},
 		Route53API: &route53Client{
-			Route53API:           nil,
+			Route53API: nil,
 			controlPlaneOperator: controlPlaneOperator,
 		},
 		S3API: &s3Client{
-			S3API:                  nil,
+			S3API: nil,
 			openshiftImageRegistry: openshiftImageRegistry,
 		},
 	}, nil
 }
-
 type awsEbsCsiDriverControllerClientDelegate struct {
 	ec2Client ec2iface.EC2API
 }
 
 type cloudControllerClientDelegate struct {
-	ec2Client   ec2iface.EC2API
-	elbClient   elbiface.ELBAPI
+	ec2Client ec2iface.EC2API
+	elbClient elbiface.ELBAPI
 	elbv2Client elbv2iface.ELBV2API
 }
 
@@ -142,7 +141,7 @@ type cloudNetworkConfigControllerClientDelegate struct {
 }
 
 type controlPlaneOperatorClientDelegate struct {
-	ec2Client     ec2iface.EC2API
+	ec2Client ec2iface.EC2API
 	route53Client route53iface.Route53API
 }
 
@@ -154,6 +153,7 @@ type openshiftImageRegistryClientDelegate struct {
 	s3Client s3iface.S3API
 }
 
+
 // DelegatingClient embeds clients for AWS services we have privileges to use with guest cluster component roles.
 type DelegatingClient struct {
 	ec2iface.EC2API
@@ -163,16 +163,17 @@ type DelegatingClient struct {
 	s3iface.S3API
 }
 
+
 // ec2Client delegates to individual component clients for API calls we know those components will have privileges to make.
 type ec2Client struct {
 	// embedding this fulfills the interface and falls back to a panic for APIs we don't have privileges for
 	ec2iface.EC2API
 
-	awsEbsCsiDriverController    *awsEbsCsiDriverControllerClientDelegate
-	cloudController              *cloudControllerClientDelegate
+	awsEbsCsiDriverController *awsEbsCsiDriverControllerClientDelegate
+	cloudController *cloudControllerClientDelegate
 	cloudNetworkConfigController *cloudNetworkConfigControllerClientDelegate
-	controlPlaneOperator         *controlPlaneOperatorClientDelegate
-	nodePool                     *nodePoolClientDelegate
+	controlPlaneOperator *controlPlaneOperatorClientDelegate
+	nodePool *nodePoolClientDelegate
 }
 
 func (c *ec2Client) AttachVolumeWithContext(ctx aws.Context, input *ec2.AttachVolumeInput, opts ...request.Option) (*ec2.VolumeAttachment, error) {
@@ -593,3 +594,5 @@ func (c *s3Client) PutObjectWithContext(ctx aws.Context, input *s3.PutObjectInpu
 func (c *s3Client) PutPublicAccessBlockWithContext(ctx aws.Context, input *s3.PutPublicAccessBlockInput, opts ...request.Option) (*s3.PutPublicAccessBlockOutput, error) {
 	return c.openshiftImageRegistry.s3Client.PutPublicAccessBlockWithContext(ctx, input, opts...)
 }
+
+
diff --git a/control-plane-operator/controllers/hostedcontrolplane/configoperator/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/configoperator/reconcile.go
@@ -232,6 +232,7 @@ func ReconcileRole(role *rbacv1.Role, ownerRef config.OwnerRef, platform hyperv1
 			},
 		}...)
 	}
+	// TODO (jparrill): Add RBAC specific needs for Agent platform
 	return nil
 }
 
@@ -443,7 +444,7 @@ func buildHCCContainerMain(image, hcpName, openShiftVersion, kubeVersion string,
 
 func buildHCCVolumeKubeconfig(v *corev1.Volume) {
 	v.Secret = &corev1.SecretVolumeSource{
-		SecretName:  manifests.KASServiceKubeconfigSecret("").Name,
+		SecretName:  manifests.HCCOKubeconfigSecret("").Name,
 		DefaultMode: pointer.Int32(0640),
 	}
 }

diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go
@@ -2832,6 +2832,10 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex
 	if err := r.Get(ctx, client.ObjectKeyFromObject(bootstrapClientCertSecret), bootstrapClientCertSecret); err != nil {
 		return fmt.Errorf("failed to get bootstrap client cert secret: %w", err)
 	}
+	hccoClientCertSecret := manifests.HCCOClientCertSecret(hcp.Namespace)
+	if err := r.Get(ctx, client.ObjectKeyFromObject(hccoClientCertSecret), hccoClientCertSecret); err != nil {
+		return fmt.Errorf("failed to get HCCO client cert secret: %w", err)
+	}
 
 	serviceKubeconfigSecret := manifests.KASServiceKubeconfigSecret(hcp.Namespace)
 	if _, err := createOrUpdate(ctx, r, serviceKubeconfigSecret, func() error {
@@ -2851,6 +2855,13 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex
 		return fmt.Errorf("failed to reconcile CAPI service admin kubeconfig secret: %w", err)
 	}
 
+	hccoKubeconfigSecret := manifests.HCCOKubeconfigSecret(hcp.Namespace)
+	if _, err := createOrUpdate(ctx, r, hccoKubeconfigSecret, func() error {
+		return kas.ReconcileHCCOKubeconfigSecret(hccoKubeconfigSecret, hccoClientCertSecret, rootCA, p.OwnerRef, hcp.Spec.Platform.Type)
+	}); err != nil {
+		return fmt.Errorf("failed to reconcile HCCO kubeconfig secret: %w", err)
+	}
+
 	localhostKubeconfigSecret := manifests.KASLocalhostKubeconfigSecret(hcp.Namespace)
 	if _, err := createOrUpdate(ctx, r, localhostKubeconfigSecret, func() error {
 		return kas.ReconcileLocalhostKubeconfigSecret(localhostKubeconfigSecret, clientCertSecret, rootCA, p.OwnerRef, p.KASPodPort)

diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go
@@ -756,6 +756,22 @@ cat <<EOF >/tmp/manifests/99_feature-gate.yaml
 %[3]s
 EOF
 
+touch /tmp/manifests/hcco-rolebinding.yaml
+cat <<EOF >/tmp/manifests/hcco-rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: hcco-cluster-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:hosted-cluster-config-operator
+EOF
+
 /usr/bin/render \
    --asset-output-dir /tmp/output \
    --rendered-manifest-dir=/tmp/manifests \

diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/kubeconfig.go b/control-plane-operator/controllers/hostedcontrolplane/kas/kubeconfig.go
@@ -58,3 +58,8 @@ func ReconcileExternalKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.C
 func ReconcileBootstrapKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.ConfigMap, ownerRef config.OwnerRef, externalURL string) error {
 	return pki.ReconcileKubeConfig(secret, cert, ca, externalURL, "", manifests.KubeconfigScopeBootstrap, ownerRef)
 }
+
+func ReconcileHCCOKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.ConfigMap, ownerRef config.OwnerRef, platformType hyperv1.PlatformType) error {
+	svcURL := InClusterKASURL(platformType)
+	return pki.ReconcileKubeConfig(secret, cert, ca, svcURL, "", "service", ownerRef)
+}
diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go b/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go
@@ -159,6 +159,29 @@ func (r *HostedControlPlaneReconciler) setupKASClientSigners(
 		return err
 	}
 
+	// ----------
+	//	HCCO signer
+	// ----------
+
+	hccoKubeconfigSigner, err := reconcileSigner(
+		manifests.HCCOSigner(hcp.Namespace),
+		pki.ReconcileHCCOSigner,
+	)
+
+	if err != nil {
+		return err
+	}
+	totalClientCABundle = append(totalClientCABundle, hccoKubeconfigSigner)
+
+	// system:hosted-cluster-config-operator client cert
+	if _, err := reconcileSub(
+		manifests.HCCOClientCertSecret(hcp.Namespace),
+		hccoKubeconfigSigner,
+		pki.ReconcileHCCOClientCertSecret,
+	); err != nil {
+		return err
+	}
+
 	// ----------
 	//	CSR signer
 	// ----------

diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go
@@ -63,6 +63,15 @@ func KASServiceCAPIKubeconfigSecret(controlPlaneNamespace, infraID string) *core
 	}
 }
 
+func HCCOKubeconfigSecret(controlPlaneNamespace string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "hcco-kubeconfig",
+			Namespace: controlPlaneNamespace,
+		},
+	}
+}
+
 func KASExternalKubeconfigSecret(controlPlaneNamespace string, ref *hyperv1.KubeconfigSecretRef) *corev1.Secret {
 	s := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{

diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go
@@ -224,6 +224,14 @@ func SystemAdminClientCertSecret(ns string) *corev1.Secret {
 	return secretFor(ns, "system-admin-client")
 }
 
+func HCCOSigner(ns string) *corev1.Secret {
+	return secretFor(ns, "hcco-signer")
+}
+
+func HCCOClientCertSecret(ns string) *corev1.Secret {
+	return secretFor(ns, "hcco-client")
+}
+
 func KASMachineBootstrapClientCertSecret(ns string) *corev1.Secret {
 	return secretFor(ns, "kas-bootstrap-client")
 }

diff --git a/control-plane-operator/controllers/hostedcontrolplane/pki/ca.go b/control-plane-operator/controllers/hostedcontrolplane/pki/ca.go
@@ -49,6 +49,10 @@ func ReconcileAdminKubeconfigSigner(secret *corev1.Secret, ownerRef config.Owner
 	return reconcileSelfSignedCA(secret, ownerRef, "admin-kubeconfig-signer", "openshift")
 }
 
+func ReconcileHCCOSigner(secret *corev1.Secret, ownerRef config.OwnerRef) error {
+	return reconcileSelfSignedCA(secret, ownerRef, "hcco-signer", "openshift")
+}
+
 func ReconcileKubeCSRSigner(secret *corev1.Secret, ownerRef config.OwnerRef) error {
 	return reconcileSelfSignedCA(secret, ownerRef, "kube-csr-signer", "openshift")
 }

diff --git a/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go b/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go
@@ -90,6 +90,10 @@ func ReconcileSystemAdminClientCertSecret(secret, ca *corev1.Secret, ownerRef co
 	return reconcileSignedCert(secret, ca, ownerRef, "system:admin", []string{"system:masters"}, X509UsageClientAuth)
 }
 
+func ReconcileHCCOClientCertSecret(secret, ca *corev1.Secret, ownerRef config.OwnerRef) error {
+	return reconcileSignedCert(secret, ca, ownerRef, fmt.Sprintf("system:%s", config.HCCOUser), []string{"kubernetes"}, X509UsageClientAuth)
+}
+
 func ReconcileServiceAccountKubeconfig(secret, csrSigner *corev1.Secret, ca *corev1.ConfigMap, hcp *hyperv1.HostedControlPlane, serviceAccountNamespace, serviceAccountName string) error {
 	cn := serviceaccount.MakeUsername(serviceAccountNamespace, serviceAccountName)
 	if err := reconcileSignedCert(secret, csrSigner, config.OwnerRef{}, cn, serviceaccount.MakeGroupNames(serviceAccountNamespace), X509UsageClientAuth); err != nil {

diff --git a/control-plane-operator/hostedclusterconfigoperator/operator/config.go b/control-plane-operator/hostedclusterconfigoperator/operator/config.go
@@ -25,6 +25,7 @@ import (
 	operatorv1 "github.com/openshift/api/operator/v1"
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/api"
+	"github.com/openshift/hypershift/support/config"
 	"github.com/openshift/hypershift/support/labelenforcingclient"
 	"github.com/openshift/hypershift/support/releaseinfo"
 	"github.com/openshift/hypershift/support/upsert"
@@ -75,7 +76,7 @@ type HostedClusterConfigOperatorConfig struct {
 }
 
 func Mgr(cfg, cpConfig *rest.Config, namespace string) ctrl.Manager {
-	cfg.UserAgent = "hosted-cluster-config-operator-manager"
+	cfg.UserAgent = config.HCCOUserAgent
 	allSelector := cache.ByObject{
 		Label: labels.Everything(),
 	}

diff --git a/support/config/types.go b/support/config/types.go
@@ -4,4 +4,9 @@ const (
 	// PodSafeToEvictLocalVolumesKey is an annotation used by the CA operator which makes sure
 	// all the pods annotated with it and the picking the desired local volumes that are safe to evict, could be drained properly.
 	PodSafeToEvictLocalVolumesKey = "cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes"
+
+	// HCCOUser references the user used by the HostedClusterConfigOperator
+	HCCOUser = "hosted-cluster-config-operator"
+	// HCCOUserAgent references the userAgent used by the HostedClusterConfigOperator
+	HCCOUserAgent = "hosted-cluster-config-operator-manager"
 )