From 0f2ad4318cca7b069d82e8f37cf42216ab889717 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Wed, 18 Dec 2024 16:07:24 +0100 Subject: [PATCH 1/9] NO-JIRA: disable image registry for testing --- .../hostedcontrolplane/hostedcontrolplane_controller.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index df5f9c23d4..4aac886558 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -1289,10 +1289,10 @@ func (r *HostedControlPlaneReconciler) reconcile(ctx context.Context, hostedCont } // Reconcile image registry operator - r.Log.Info("Reconciling Image Registry Operator") - if err := r.reconcileImageRegistryOperator(ctx, hostedControlPlane, releaseImageProvider, userReleaseImageProvider, createOrUpdate); err != nil { + /*r.Log.Info("Reconciling Image Registry Operator") + if err := r.reconcileImageRegistryOperator(ctx, hostedControlPlane, releaseImageProvider, userReleaseImageProvider, createOrUpdate); err != nil { return fmt.Errorf("failed to reconcile image registry operator: %w", err) - } + }*/ if IsStorageAndCSIManaged(hostedControlPlane) { // Reconcile cluster storage operator From 514699630f8f5552d9cbffca242fec34ccae17a8 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Thu, 19 Dec 2024 14:14:16 +0100 Subject: [PATCH 2/9] set baseline caps --- .../controllers/hostedcontrolplane/cvo/reconcile.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go index af0f4f41ad..ccaaa33a54 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go @@ -304,6 +304,14 @@ func preparePayloadScript(platformType hyperv1.PlatformType, oauthEnabled bool) } func cvoBootrapScript(clusterID string) string { + filtered := make([]string, 0) + for _, capability := range configv1.KnownClusterVersionCapabilities { + if capability != configv1.ClusterVersionCapabilityImageRegistry { + filtered = append(filtered, string(capability)) + } + } + + capabilities := strings.Join(filtered, ",") payloadDir := volumeMounts.Path(cvoContainerBootstrap().Name, cvoVolumePayload().Name) var scriptTemplate = `#!/bin/bash set -euo pipefail @@ -314,6 +322,8 @@ metadata: name: version spec: clusterID: %s + capabilities: + baselineCapabilitySet: [%s] EOF oc get ns openshift-config &> /dev/null || oc create ns openshift-config oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed @@ -327,7 +337,7 @@ while true; do done oc get clusterversion/version &> /dev/null || oc create -f /tmp/clusterversion.yaml ` - return fmt.Sprintf(scriptTemplate, clusterID, payloadDir) + return fmt.Sprintf(scriptTemplate, clusterID, capabilities, payloadDir) } func buildCVOContainerMain(image, releaseImage, namespace string, updateService configv1.URL, enableCVOManagementClusterMetricsAccess bool) func(c *corev1.Container) { From 225a9021b42e724ae0145810b0d326fc71d03490 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Thu, 19 Dec 2024 16:54:21 +0100 Subject: [PATCH 3/9] use additional caps --- .../controllers/hostedcontrolplane/cvo/reconcile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go index ccaaa33a54..7ea1207dfb 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go @@ -323,7 +323,8 @@ metadata: spec: clusterID: %s capabilities: - baselineCapabilitySet: [%s] + baselineCapabilitySet: 'None' + additionalEnabledCapabilities: [%s] EOF oc get ns openshift-config &> /dev/null || oc create ns openshift-config oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed From ef12c12f793ae5c69fa129b1d6ef78442504c615 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Fri, 20 Dec 2024 11:57:31 +0100 Subject: [PATCH 4/9] test yaml ordering --- .../controllers/hostedcontrolplane/cvo/reconcile.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go index 7ea1207dfb..e080fcd3ba 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go @@ -206,7 +206,7 @@ func buildCVOContainerBootstrap(image, clusterID string) func(*corev1.Container) c.Command = []string{"/bin/bash"} c.Args = []string{ "-c", - cvoBootrapScript(clusterID), + cvoBootstrapScript(clusterID), } c.Resources.Requests = corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("10m"), @@ -303,7 +303,7 @@ func preparePayloadScript(platformType hyperv1.PlatformType, oauthEnabled bool) return strings.Join(stmts, "\n") } -func cvoBootrapScript(clusterID string) string { +func cvoBootstrapScript(clusterID string) string { filtered := make([]string, 0) for _, capability := range configv1.KnownClusterVersionCapabilities { if capability != configv1.ClusterVersionCapabilityImageRegistry { @@ -311,7 +311,7 @@ func cvoBootrapScript(clusterID string) string { } } - capabilities := strings.Join(filtered, ",") + capabilities := strings.Join(filtered, ", ") payloadDir := volumeMounts.Path(cvoContainerBootstrap().Name, cvoVolumePayload().Name) var scriptTemplate = `#!/bin/bash set -euo pipefail @@ -321,10 +321,10 @@ kind: ClusterVersion metadata: name: version spec: - clusterID: %s capabilities: baselineCapabilitySet: 'None' additionalEnabledCapabilities: [%s] + clusterID: %s EOF oc get ns openshift-config &> /dev/null || oc create ns openshift-config oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed From f62cfc96ce35c7c5982768350bb0ede3aee8fe9f Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Fri, 20 Dec 2024 12:44:26 +0100 Subject: [PATCH 5/9] test yaml format ordering Signed-off-by: Thomas Jungblut --- .../hostedcontrolplane/cvo/reconcile.go | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go index e080fcd3ba..db144d9376 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go @@ -2,6 +2,7 @@ package cvo import ( "fmt" + "k8s.io/apimachinery/pkg/util/sets" "path" "strings" @@ -304,18 +305,23 @@ func preparePayloadScript(platformType hyperv1.PlatformType, oauthEnabled bool) } func cvoBootstrapScript(clusterID string) string { - filtered := make([]string, 0) - for _, capability := range configv1.KnownClusterVersionCapabilities { - if capability != configv1.ClusterVersionCapabilityImageRegistry { - filtered = append(filtered, string(capability)) + enabledCaps := sets.New[configv1.ClusterVersionCapability]( + configv1.ClusterVersionCapabilitySets[configv1.ClusterVersionCapabilitySetCurrent]...) + enabledCaps = enabledCaps.Delete(configv1.ClusterVersionCapabilityImageRegistry) + + capabilities := strings.Builder{} + capList := enabledCaps.UnsortedList() + for i, capability := range capList { + capabilities.WriteString(string(capability)) + if i != len(capList)-1 { + capabilities.WriteString(",") } } - capabilities := strings.Join(filtered, ", ") payloadDir := volumeMounts.Path(cvoContainerBootstrap().Name, cvoVolumePayload().Name) var scriptTemplate = `#!/bin/bash set -euo pipefail -cat > /tmp/clusterversion.yaml < /tmp/clusterversion.yaml <<-EOF apiVersion: config.openshift.io/v1 kind: ClusterVersion metadata: @@ -325,6 +331,7 @@ spec: baselineCapabilitySet: 'None' additionalEnabledCapabilities: [%s] clusterID: %s + EOF oc get ns openshift-config &> /dev/null || oc create ns openshift-config oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed @@ -338,7 +345,7 @@ while true; do done oc get clusterversion/version &> /dev/null || oc create -f /tmp/clusterversion.yaml ` - return fmt.Sprintf(scriptTemplate, clusterID, capabilities, payloadDir) + return fmt.Sprintf(scriptTemplate, capabilities.String(), clusterID, payloadDir) } func buildCVOContainerMain(image, releaseImage, namespace string, updateService configv1.URL, enableCVOManagementClusterMetricsAccess bool) func(c *corev1.Container) { From fa19699514c6ef9642fd9d488f0425d42d0e17c8 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Mon, 23 Dec 2024 11:15:22 +0100 Subject: [PATCH 6/9] try to create the CVO resources --- .../controllers/hostedcontrolplane/cvo/reconcile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go index db144d9376..2560242ec0 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go @@ -335,6 +335,7 @@ spec: EOF oc get ns openshift-config &> /dev/null || oc create ns openshift-config oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed +oc create -f /tmp/clusterversion.yaml while true; do echo "Applying CVO bootstrap manifests" if oc apply -f %s/manifests; then @@ -343,7 +344,6 @@ while true; do fi sleep 1 done -oc get clusterversion/version &> /dev/null || oc create -f /tmp/clusterversion.yaml ` return fmt.Sprintf(scriptTemplate, capabilities.String(), clusterID, payloadDir) } From 77bd3db16b47d9913913af800c83e67c656f8033 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Mon, 23 Dec 2024 15:07:16 +0100 Subject: [PATCH 7/9] ensure the crd is pushed first --- .../controllers/hostedcontrolplane/cvo/reconcile.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go index 2560242ec0..dba58d80d4 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go @@ -321,6 +321,7 @@ func cvoBootstrapScript(clusterID string) string { payloadDir := volumeMounts.Path(cvoContainerBootstrap().Name, cvoVolumePayload().Name) var scriptTemplate = `#!/bin/bash set -euo pipefail +MANIFEST_DIR=%s/manifests cat > /tmp/clusterversion.yaml <<-EOF apiVersion: config.openshift.io/v1 kind: ClusterVersion @@ -335,17 +336,20 @@ spec: EOF oc get ns openshift-config &> /dev/null || oc create ns openshift-config oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed + +oc apply -f ${MANIFEST_DIR}/0000_00_cluster-version-operator_01_clusterversions-Default.crd.yaml oc create -f /tmp/clusterversion.yaml + while true; do echo "Applying CVO bootstrap manifests" - if oc apply -f %s/manifests; then + if oc apply -f ${MANIFEST_DIR}; then echo "Bootstrap manifests applied successfully." break fi sleep 1 done ` - return fmt.Sprintf(scriptTemplate, capabilities.String(), clusterID, payloadDir) + return fmt.Sprintf(scriptTemplate, payloadDir, capabilities.String(), clusterID) } func buildCVOContainerMain(image, releaseImage, namespace string, updateService configv1.URL, enableCVOManagementClusterMetricsAccess bool) func(c *corev1.Container) { From 1cbcc53dd539940ad2a3f040bacb1ee2ce1e4a20 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Mon, 23 Dec 2024 16:49:05 +0100 Subject: [PATCH 8/9] list all manifests --- .../controllers/hostedcontrolplane/cvo/reconcile.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go index dba58d80d4..a8914aa00f 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go @@ -334,9 +334,12 @@ spec: clusterID: %s EOF + oc get ns openshift-config &> /dev/null || oc create ns openshift-config oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed +ls -la ${MANIFEST_DIR} + oc apply -f ${MANIFEST_DIR}/0000_00_cluster-version-operator_01_clusterversions-Default.crd.yaml oc create -f /tmp/clusterversion.yaml From 92f271fd06e036aa1bb98cfa3321735f5f4af33d Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Tue, 24 Dec 2024 11:40:52 +0100 Subject: [PATCH 9/9] retrieve the crd --- .../controllers/hostedcontrolplane/cvo/reconcile.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go index a8914aa00f..d801c2e918 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go @@ -338,10 +338,9 @@ EOF oc get ns openshift-config &> /dev/null || oc create ns openshift-config oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed -ls -la ${MANIFEST_DIR} - oc apply -f ${MANIFEST_DIR}/0000_00_cluster-version-operator_01_clusterversions-Default.crd.yaml oc create -f /tmp/clusterversion.yaml +oc get config/version -oyaml while true; do echo "Applying CVO bootstrap manifests"