Move image registry operator to control plane

Stops CVO from reconciling the cluster image registry operator into the guest cluster and enables the control plane operator to reconcile it on the management cluster side. This change is needed to make it possible to clean up resources created by the guest cluster. Having the image registry operator on the control plane side allows it to keep running after worker nodes have been removed from the cluster. On the guest cluster, changing the operator config state to "Removed" tells the operator to remove the S3 bucket (or other cloud storage) it created when starting up.
gnufied · Aug 4, 2022 · d529f6d · d529f6d
1 parent dfff72a
commit d529f6d
Show file tree

Hide file tree

Showing 9 changed files with 858 additions and 1 deletion.
diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go
@@ -73,6 +73,9 @@ var (
 		"0000_80_machine-config-operator_01_machineconfigpool.crd.yaml",
 		"0000_50_cluster-node-tuning-operator_50-operator-ibm-cloud-managed.yaml",
 		"0000_50_cluster-node-tuning-operator_60-clusteroperator.yaml",
+		"0000_50_cluster-image-registry-operator_07-operator-ibm-cloud-managed.yaml",
+		"0000_50_cluster-image-registry-operator_07-operator-service.yaml",
+		"0000_90_cluster-image-registry-operator_02_operator-servicemonitor.yaml",
 
 		// TODO: Remove these when cluster profiles annotations are fixed
 		// for cco and auth  operators
@@ -233,6 +236,12 @@ func resourcesToRemove() []resourceDesc {
 			name:       "cluster-node-tuning-operator",
 			namespace:  "openshift-cluster-node-tuning-operator",
 		},
+		{
+			apiVersion: "apps/v1",
+			kind:       "Deployment",
+			name:       "cluster-image-registry-operator",
+			namespace:  "openshift-image-registry",
+		},
 	}
 }
 
@@ -251,7 +260,10 @@ func preparePayloadScript() string {
 	}
 	toRemove := resourcesToRemove()
 	if len(toRemove) > 0 {
-		stmts = append(stmts, fmt.Sprintf("cat > %s/release-manifests/cleanup.yaml <<EOF", payloadDir))
+		// NOTE: the name of the cleanup file indicates the CVO runlevel for the cleanup.
+		// A level of 0000_01 forces the cleanup to happen first without waiting for any cluster operators to
+		// become available.
+		stmts = append(stmts, fmt.Sprintf("cat > %s/release-manifests/0000_01_cleanup.yaml <<EOF", payloadDir))
 	}
 	for _, desc := range resourcesToRemove() {
 		stmts = append(stmts,

diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go
@@ -39,6 +39,7 @@ import (
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/ocm"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/olm"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/pki"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/registryoperator"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/scheduler"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator"
 	"github.com/openshift/hypershift/support/capabilities"
@@ -768,6 +769,12 @@ func (r *HostedControlPlaneReconciler) update(ctx context.Context, hostedControl
 		return fmt.Errorf("failed to reconcile olm: %w", err)
 	}
 
+	// Reconcile image registry operator
+	r.Log.Info("Reconciling Image Registry Operator")
+	if err = r.reconcileImageRegistryOperator(ctx, hostedControlPlane, releaseImage, createOrUpdate); err != nil {
+		return fmt.Errorf("failed to reconcile image registry operator: %w", err)
+	}
+
 	// Reconcile Ignition
 	r.Log.Info("Reconciling core machine configs")
 	if err = r.reconcileCoreIgnitionConfig(ctx, hostedControlPlane, createOrUpdate); err != nil {
@@ -1366,6 +1373,14 @@ func (r *HostedControlPlaneReconciler) reconcilePKI(ctx context.Context, hcp *hy
 		return fmt.Errorf("failed to reconcile olm operator serving cert: %w", err)
 	}
 
+	// Image Registry Operator Serving Cert
+	imageRegistryOperatorServingCert := manifests.ImageRegistryOperatorServingCert(hcp.Namespace)
+	if _, err := createOrUpdate(ctx, r, imageRegistryOperatorServingCert, func() error {
+		return pki.ReconcileRegistryOperatorServingCert(imageRegistryOperatorServingCert, rootCASecret, p.OwnerRef)
+	}); err != nil {
+		return fmt.Errorf("failed to reconcile image registry operator serving cert: %w", err)
+	}
+
 	kcmServerSecret := manifests.KCMServerCertSecret(hcp.Namespace)
 	if _, err := createOrUpdate(ctx, r, kcmServerSecret, func() error {
 		return pki.ReconcileKCMServerSecret(kcmServerSecret, rootCASecret, p.OwnerRef)
@@ -2285,6 +2300,26 @@ func (r *HostedControlPlaneReconciler) reconcileOperatorLifecycleManager(ctx con
 	return nil
 }
 
+func (r *HostedControlPlaneReconciler) reconcileImageRegistryOperator(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImage *releaseinfo.ReleaseImage, createOrUpdate upsert.CreateOrUpdateFN) error {
+	params := registryoperator.NewParams(hcp, releaseImage.Version(), releaseImage.ComponentImages(), r.SetDefaultSecurityContext)
+	deployment := manifests.ImageRegistryOperatorDeployment(hcp.Namespace)
+	if _, err := createOrUpdate(ctx, r, deployment, func() error {
+		return registryoperator.ReconcileDeployment(deployment, params)
+	}); err != nil {
+		return fmt.Errorf("failed to reconcile image registry operator deployment: %w", err)
+	}
+
+	pm := manifests.ImageRegistryOperatorPodMonitor(hcp.Namespace)
+	if _, err := createOrUpdate(ctx, r, pm, func() error {
+		registryoperator.ReconcilePodMonitor(pm, hcp.Spec.ClusterID, r.MetricsSet)
+		return nil
+	}); err != nil {
+		return fmt.Errorf("failed to reconcile image registry operator pod monitor: %w", err)
+	}
+
+	return nil
+}
+
 func (r *HostedControlPlaneReconciler) reconcileMachineConfigServerConfig(ctx context.Context, hcp *hyperv1.HostedControlPlane, createOrUpdate upsert.CreateOrUpdateFN) error {
 	rootCA := manifests.RootCASecret(hcp.Namespace)
 	if err := r.Get(ctx, client.ObjectKeyFromObject(rootCA), rootCA); err != nil {

diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/registryoperator.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/registryoperator.go
@@ -0,0 +1,35 @@
+package manifests
+
+import (
+	prometheusoperatorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func ImageRegistryOperatorDeployment(ns string) *appsv1.Deployment {
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "cluster-image-registry-operator",
+			Namespace: ns,
+		},
+	}
+}
+
+func ImageRegistryOperatorServingCert(ns string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "cluster-image-registry-operator",
+			Namespace: ns,
+		},
+	}
+}
+
+func ImageRegistryOperatorPodMonitor(ns string) *prometheusoperatorv1.PodMonitor {
+	return &prometheusoperatorv1.PodMonitor{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "cluster-image-registry-operator",
+			Namespace: ns,
+		},
+	}
+}
diff --git a/control-plane-operator/controllers/hostedcontrolplane/pki/registryoperator.go b/control-plane-operator/controllers/hostedcontrolplane/pki/registryoperator.go
@@ -0,0 +1,17 @@
+package pki
+
+import (
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/openshift/hypershift/support/config"
+)
+
+const metricsHostname = "cluster-image-registry-operator"
+
+func ReconcileRegistryOperatorServingCert(secret, ca *corev1.Secret, ownerRef config.OwnerRef) error {
+	dnsNames := []string{
+		metricsHostname,
+		"localhost",
+	}
+	return reconcileSignedCertWithAddresses(secret, ca, ownerRef, metricsHostname, []string{"openshift"}, X509UsageClientServerAuth, dnsNames, nil)
+}