Merge pull request #4888 from bryan-cox/aro-hcp-image-reg-cert-deploy…

…ment HOSTEDCP-2022: Reconcile SecretProviderClass for Image Registry on ARO HCP
openshift · Nov 22, 2024 · 018a5e6 · 018a5e6
2 parents cf790ae + d9f5d0d
commit 018a5e6
Show file tree

Hide file tree

Showing 11 changed files with 211 additions and 68 deletions.
diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go
@@ -5,6 +5,7 @@ import (
 	crand "crypto/rand"
 	"errors"
 	"fmt"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/secretproviderclass"
 	"math/big"
 	"net/http"
 	"os"
@@ -85,6 +86,7 @@ import (
 	pkimanifests "github.com/openshift/hypershift/control-plane-pki-operator/manifests"
 	sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress"
 	supportawsutil "github.com/openshift/hypershift/support/awsutil"
+	hyperazureutil "github.com/openshift/hypershift/support/azureutil"
 	"github.com/openshift/hypershift/support/capabilities"
 	"github.com/openshift/hypershift/support/certs"
 	"github.com/openshift/hypershift/support/conditions"
@@ -2689,7 +2691,7 @@ func (r *HostedControlPlaneReconciler) reconcileCloudProviderConfig(ctx context.
 			return fmt.Errorf("failed to reconcile aws provider config: %w", err)
 		}
 	case hyperv1.AzurePlatform:
-		credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}}
+		credentialsSecret := manifests.AzureCredentialInformation(hcp.Namespace)
 		if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil {
 			return fmt.Errorf("failed to get Azure credentials secret: %w", err)
 		}
@@ -4107,6 +4109,24 @@ func checkCatalogImageOverides(images ...string) (bool, error) {
 func (r *HostedControlPlaneReconciler) reconcileImageRegistryOperator(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImageProvider, userReleaseImageProvider *imageprovider.SimpleReleaseImageProvider, createOrUpdate upsert.CreateOrUpdateFN) error {
 	params := registryoperator.NewParams(hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext)
 
+	// Create SecretProviderClass when deploying on managed Azure
+	if hyperazureutil.IsAroHCP() {
+		imageRegistrySecretProviderClass := manifests.ManagedAzureSecretProviderClass(config.ManagedAzureImageRegistrySecretStoreProviderClassName, hcp.Namespace)
+		if _, err := createOrUpdate(ctx, r, imageRegistrySecretProviderClass, func() error {
+			secretproviderclass.ReconcileManagedAzureSecretProviderClass(imageRegistrySecretProviderClass, hcp, hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ImageRegistry.CertificateName)
+			return nil
+		}); err != nil {
+			return fmt.Errorf("failed to reconcile image registry operator secret provider class: %w", err)
+		}
+
+		credentialsSecret := manifests.AzureCredentialInformation(hcp.Namespace)
+		if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil {
+			return fmt.Errorf("failed to get Azure credentials secret: %w", err)
+		}
+
+		params.AzureTenantID = string(credentialsSecret.Data["AZURE_TENANT_ID"])
+	}
+
 	deployment := manifests.ImageRegistryOperatorDeployment(hcp.Namespace)
 	if _, err := createOrUpdate(ctx, r, deployment, func() error {
 		return registryoperator.ReconcileDeployment(deployment, params)
@@ -4910,7 +4930,7 @@ func (r *HostedControlPlaneReconciler) reconcileClusterStorageOperator(ctx conte
 	params := storage.NewParams(hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext)
 
 	if hcp.Spec.Platform.Type == hyperv1.AzurePlatform {
-		credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}}
+		credentialsSecret := manifests.AzureCredentialInformation(hcp.Namespace)
 		if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil {
 			return fmt.Errorf("failed to get Azure credentials secret: %w", err)
 		}
@@ -5384,7 +5404,7 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex
 	}
 	azureKmsSpec := hcp.Spec.SecretEncryption.KMS.Azure
 
-	credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}}
+	credentialsSecret := manifests.AzureCredentialInformation(hcp.Namespace)
 	if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil {
 		condition := metav1.Condition{
 			Type:               string(hyperv1.ValidAzureKMSConfig),

diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go
@@ -1630,6 +1630,13 @@ func TestControlPlaneComponents(t *testing.T) {
 		},
 	}
 
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "azure-credential-information",
+			Namespace: "hcp-namespace",
+		},
+	}
+
 	cpContext := controlplanecomponent.ControlPlaneContext{
 		Context:                  context.Background(),
 		CreateOrUpdateProviderV2: upsert.NewV2(false),
@@ -1642,6 +1649,7 @@ func TestControlPlaneComponents(t *testing.T) {
 		fakeClient := fake.NewClientBuilder().WithScheme(api.Scheme).
 			WithObjects(componentsFakeObjects(hcp.Namespace)...).
 			WithObjects(componentsFakeDependencies(component.Name(), hcp.Namespace)...).
+			WithObjects(secret).
 			Build()
 		cpContext.Client = fakeClient
 

diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/azure.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/azure.go
@@ -43,3 +43,12 @@ func AzureFileConfigWithCredentials(ns string) *corev1.Secret {
 		},
 	}
 }
+
+func AzureCredentialInformation(ns string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "azure-credential-information",
+			Namespace: ns,
+		},
+	}
+}
diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/secretproviderclass.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/secretproviderclass.go
@@ -1,50 +1,15 @@
 package manifests
 
 import (
-	"fmt"
-	"github.com/openshift/hypershift/support/azureutil"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	secretsstorev1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1"
 )
 
-const (
-	objectFormat = `
-array:
-  - |
-    objectName: %s
-    objectType: secret
-`
-)
-
-// ManagedAzureKeyVaultSecretProviderClass returns an instance of a SecretProviderClass completed with its name, Azure Key Vault set
-// up, and the certificate name it needs to pull from the Key Vault.
-//
-// https://learn.microsoft.com/en-us/azure/aks/csi-secrets-store-identity-access?tabs=azure-portal&pivots=access-with-a-user-assigned-managed-identity
-func ManagedAzureKeyVaultSecretProviderClass(secretProviderClassName, namespace, keyVaultName, KeyVaultTenantID, certificateName string) *secretsstorev1.SecretProviderClass {
+func ManagedAzureSecretProviderClass(name, namespace string) *secretsstorev1.SecretProviderClass {
 	return &secretsstorev1.SecretProviderClass{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      secretProviderClassName,
+			Name:      name,
 			Namespace: namespace,
 		},
-		Spec: secretsstorev1.SecretProviderClassSpec{
-			Provider: "azure",
-			Parameters: map[string]string{
-				"usePodIdentity":         "false",
-				"useVMManagedIdentity":   "true",
-				"userAssignedIdentityID": azureutil.GetKeyVaultAuthorizedUser(),
-				"keyvaultName":           keyVaultName,
-				"tenantId":               KeyVaultTenantID,
-				"objects":                formatSecretProviderClassObject(certificateName),
-			},
-		},
 	}
 }
-
-// formatSecretProviderClassObject places the certificate name in the appropriate string structure the
-// SecretProviderClass expects for an object. More details here:
-// - https://learn.microsoft.com/en-us/azure/aks/csi-secrets-store-identity-access?tabs=azure-portal&pivots=access-with-a-user-assigned-managed-identity#configure-managed-identity
-// - https://secrets-store-csi-driver.sigs.k8s.io/concepts.html?highlight=object#custom-resource-definitions-crds
-func formatSecretProviderClassObject(certName string) string {
-	return fmt.Sprintf(objectFormat, certName)
-}
diff --git a/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile.go
@@ -16,6 +16,7 @@ import (
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
+	"github.com/openshift/hypershift/support/azureutil"
 	"github.com/openshift/hypershift/support/certs"
 	"github.com/openshift/hypershift/support/config"
 	"github.com/openshift/hypershift/support/metrics"
@@ -97,14 +98,17 @@ var (
 )
 
 type Params struct {
-	operatorImage    string
-	tokenMinterImage string
-	platform         hyperv1.PlatformType
-	issuerURL        string
-	releaseVersion   string
-	registryImage    string
-	prunerImage      string
-	deploymentConfig config.DeploymentConfig
+	operatorImage        string
+	tokenMinterImage     string
+	platform             hyperv1.PlatformType
+	issuerURL            string
+	releaseVersion       string
+	registryImage        string
+	prunerImage          string
+	deploymentConfig     config.DeploymentConfig
+	AzureClientID        string
+	AzureTenantID        string
+	AzureCertificateName string
 }
 
 func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProvider imageprovider.ReleaseImageProvider, userReleaseImageProvider imageprovider.ReleaseImageProvider, setDefaultSecurityContext bool) Params {
@@ -143,6 +147,12 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProv
 			},
 		},
 	}
+
+	if azureutil.IsAroHCP() {
+		params.AzureClientID = hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ImageRegistry.ClientID
+		params.AzureCertificateName = hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ImageRegistry.CertificateName
+	}
+
 	params.deploymentConfig.SetRestartAnnotation(hcp.ObjectMeta)
 	if hcp.Annotations[hyperv1.ControlPlanePriorityClass] != "" {
 		params.deploymentConfig.Scheduling.PriorityClass = hcp.Annotations[hyperv1.ControlPlanePriorityClass]
@@ -193,6 +203,28 @@ func ReconcileDeployment(deployment *appsv1.Deployment, params Params) error {
 			},
 		)
 	}
+	// For managed azure deployments, we pass environment variables so we authenticate with Azure API through certificate
+	// authentication. We also mount the SecretProviderClass for the Secrets Store CSI driver to use; it will grab the
+	// certificate related to the ARO_HCP_MI_CLIENT_ID and mount it as a volume in the ingress pod in the path,
+	// ARO_HCP_CLIENT_CERTIFICATE_PATH.
+	if azureutil.IsAroHCP() {
+		deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env,
+			azureutil.CreateEnvVarsForAzureManagedIdentity(params.AzureClientID, params.AzureTenantID, params.AzureCertificateName)...)
+
+		if deployment.Spec.Template.Spec.Containers[0].VolumeMounts == nil {
+			deployment.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{}
+		}
+		deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts,
+			azureutil.CreateVolumeMountForAzureSecretStoreProviderClass(config.ManagedAzureImageRegistrySecretStoreVolumeName),
+		)
+
+		if deployment.Spec.Template.Spec.Volumes == nil {
+			deployment.Spec.Template.Spec.Volumes = []corev1.Volume{}
+		}
+		deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes,
+			azureutil.CreateVolumeForAzureSecretStoreProviderClass(config.ManagedAzureImageRegistrySecretStoreVolumeName, config.ManagedAzureImageRegistrySecretStoreProviderClassName),
+		)
+	}
 
 	params.deploymentConfig.ApplyTo(deployment)
 	return nil

diff --git a/...-plane-operator/controllers/hostedcontrolplane/secretproviderclass/secretproviderclass.go b/...-plane-operator/controllers/hostedcontrolplane/secretproviderclass/secretproviderclass.go
@@ -0,0 +1,44 @@
+package secretproviderclass
+
+import (
+	"fmt"
+	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
+	"github.com/openshift/hypershift/support/azureutil"
+
+	secretsstorev1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1"
+)
+
+const (
+	objectFormat = `
+array:
+  - |
+    objectName: %s
+    objectType: secret
+`
+)
+
+// ReconcileManagedAzureSecretProviderClass reconciles the Spec of a SecretProviderClass completed with its name, Azure
+// Key Vault setup, and the certificate name it needs to pull from the Key Vault.
+//
+// https://learn.microsoft.com/en-us/azure/aks/csi-secrets-store-identity-access?tabs=azure-portal&pivots=access-with-a-user-assigned-managed-identity
+func ReconcileManagedAzureSecretProviderClass(secretProviderClass *secretsstorev1.SecretProviderClass, hcp *hyperv1.HostedControlPlane, certName string) {
+	secretProviderClass.Spec = secretsstorev1.SecretProviderClassSpec{
+		Provider: "azure",
+		Parameters: map[string]string{
+			"usePodIdentity":         "false",
+			"useVMManagedIdentity":   "true",
+			"userAssignedIdentityID": azureutil.GetKeyVaultAuthorizedUser(),
+			"keyvaultName":           hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.Name,
+			"tenantId":               hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.TenantID,
+			"objects":                formatSecretProviderClassObject(certName),
+		},
+	}
+}
+
+// formatSecretProviderClassObject places the certificate name in the appropriate string structure the
+// SecretProviderClass expects for an object. More details here:
+// - https://learn.microsoft.com/en-us/azure/aks/csi-secrets-store-identity-access?tabs=azure-portal&pivots=access-with-a-user-assigned-managed-identity#configure-managed-identity
+// - https://secrets-store-csi-driver.sigs.k8s.io/concepts.html?highlight=object#custom-resource-definitions-crds
+func formatSecretProviderClassObject(certName string) string {
+	return fmt.Sprintf(objectFormat, certName)
+}
diff --git a/...plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config.go b/...plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config.go
@@ -3,13 +3,12 @@ package azure
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
 
 	"github.com/openshift/hypershift/support/azureutil"
 	component "github.com/openshift/hypershift/support/controlplane-component"
 
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -53,10 +52,7 @@ func azureConfig(cpContext component.ControlPlaneContext, withCredentials bool)
 	hcp := cpContext.HCP
 	azureplatform := hcp.Spec.Platform.Azure
 
-	credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{
-		Namespace: cpContext.HCP.Namespace,
-		Name:      azureplatform.Credentials.Name,
-	}}
+	credentialsSecret := manifests.AzureCredentialInformation(hcp.Namespace)
 	if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil {
 		return AzureConfig{}, fmt.Errorf("failed to get Azure credentials secret: %w", err)
 	}

diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go
@@ -1438,7 +1438,7 @@ func (r *reconciler) reconcileCloudCredentialSecrets(ctx context.Context, hcp *h
 			}
 		}
 	case hyperv1.AzurePlatform:
-		referenceCredentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}}
+		referenceCredentialsSecret := cpomanifests.AzureCredentialInformation(hcp.Namespace)
 		if err := r.cpClient.Get(ctx, client.ObjectKeyFromObject(referenceCredentialsSecret), referenceCredentialsSecret); err != nil {
 			return []error{fmt.Errorf("failed to get cloud credentials secret in hcp namespace: %w", err)}
 		}

diff --git a/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go b/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go
@@ -6,6 +6,7 @@ import (
 	"os"
 
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
 	"github.com/openshift/hypershift/support/azureutil"
 	"github.com/openshift/hypershift/support/images"
 	"github.com/openshift/hypershift/support/upsert"
@@ -151,13 +152,14 @@ func (a Azure) ReconcileCredentials(ctx context.Context, c client.Client, create
 		return fmt.Errorf("failed to get secret %s: %w", name, err)
 	}
 
-	userCloudCreds := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: controlPlaneNamespace, Name: name.Name}}
-	if _, err := createOrUpdate(ctx, c, userCloudCreds, func() error {
-		if userCloudCreds.Data == nil {
-			userCloudCreds.Data = map[string][]byte{}
+	// Reconcile the user cloud-credentials secret contents into the control plane version of the same secret, azure-credential-information
+	azureCredsInfo := manifests.AzureCredentialInformation(controlPlaneNamespace)
+	if _, err := createOrUpdate(ctx, c, azureCredsInfo, func() error {
+		if azureCredsInfo.Data == nil {
+			azureCredsInfo.Data = map[string][]byte{}
 		}
 		for k, v := range source.Data {
-			userCloudCreds.Data[k] = v
+			azureCredsInfo.Data[k] = v
 		}
 		return nil
 	}); err != nil {
@@ -184,13 +186,13 @@ func (a Azure) ReconcileCredentials(ctx context.Context, c client.Client, create
 	// Sync CNCC secret
 	cloudNetworkConfigCreds := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: controlPlaneNamespace, Name: "cloud-network-config-controller-creds"}}
 	secretData := map[string][]byte{
-		"azure_client_id":       userCloudCreds.Data["AZURE_CLIENT_ID"],
-		"azure_client_secret":   userCloudCreds.Data["AZURE_CLIENT_SECRET"],
+		"azure_client_id":       azureCredsInfo.Data["AZURE_CLIENT_ID"],
+		"azure_client_secret":   azureCredsInfo.Data["AZURE_CLIENT_SECRET"],
 		"azure_region":          []byte(hcluster.Spec.Platform.Azure.Location),
 		"azure_resource_prefix": []byte(hcluster.Name + "-" + hcluster.Spec.InfraID),
 		"azure_resourcegroup":   []byte(hcluster.Spec.Platform.Azure.ResourceGroupName),
-		"azure_subscription_id": userCloudCreds.Data["AZURE_SUBSCRIPTION_ID"],
-		"azure_tenant_id":       userCloudCreds.Data["AZURE_TENANT_ID"],
+		"azure_subscription_id": azureCredsInfo.Data["AZURE_SUBSCRIPTION_ID"],
+		"azure_tenant_id":       azureCredsInfo.Data["AZURE_TENANT_ID"],
 	}
 	if _, err := createOrUpdate(ctx, c, cloudNetworkConfigCreds, func() error {
 		cloudNetworkConfigCreds.Data = secretData
@@ -248,9 +250,9 @@ func reconcileAzureCluster(azureCluster *capiazure.AzureCluster, hcluster *hyper
 }
 
 func reconcileAzureClusterIdentity(ctx context.Context, c client.Client, hcluster *hyperv1.HostedCluster, azureClusterIdentity *capiazure.AzureClusterIdentity, controlPlaneNamespace string) error {
-	credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: hcluster.Spec.Platform.Azure.Credentials.Name, Namespace: controlPlaneNamespace}}
+	credentialsSecret := manifests.AzureCredentialInformation(controlPlaneNamespace)
 	if err := c.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil {
-		return fmt.Errorf("failed to get secret %s: %w", credentialsSecret, err)
+		return fmt.Errorf("failed to get Azure credentials secret: %w", err)
 	}
 
 	azureClusterIdentity.Spec = capiazure.AzureClusterIdentitySpec{