From 095082570c82a0d8babe5286c6b80601eb0b8288 Mon Sep 17 00:00:00 2001 From: Mulham Raee Date: Thu, 19 Dec 2024 13:55:20 +0100 Subject: [PATCH] Refactor cluster-storage-operator --- .../hostedcontrolplane_controller.go | 14 +- ...zz_fixture_TestControlPlaneComponents.yaml | 212 ++++++++++++++++++ ...lPlaneComponents_TechPreviewNoUpgrade.yaml | 212 ++++++++++++++++++ ..._TestControlPlaneComponents_component.yaml | 31 +++ ...onents_component_TechPreviewNoUpgrade.yaml | 31 +++ .../assets/cluster-storage-operator/README.md | 11 + .../azure-disk-csi-config.yaml | 6 + .../azure-disk-csi-secretprovider.yaml | 4 + .../azure-file-csi-config.yaml | 6 + .../azure-file-csi-secretprovider.yaml | 4 + .../cluster-storage-operator/deployment.yaml | 140 ++++++++++++ .../assets/cluster-storage-operator/role.yaml | 66 ++++++ .../cluster-storage-operator/rolebinding.yaml | 12 + .../serviceaccount.yaml | 5 + .../cloud_controller_manager/azure/config.go | 6 +- .../hostedcontrolplane/v2/storage/azure.go | 70 ++++++ .../v2/storage/component.go | 80 +++++++ .../v2/storage/deployment.go | 35 +++ .../v2/storage/envreplace.go | 115 ++++++++++ 19 files changed, 1052 insertions(+), 8 deletions(-) create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_TechPreviewNoUpgrade.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_component.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_component_TechPreviewNoUpgrade.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/README.md create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-disk-csi-config.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-disk-csi-secretprovider.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-file-csi-config.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-file-csi-secretprovider.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/deployment.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/role.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/rolebinding.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/serviceaccount.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/storage/azure.go create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/storage/component.go create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/storage/deployment.go create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/storage/envreplace.go diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index dd76b70691..65c149b62c 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -84,6 +84,7 @@ import ( ocmv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/ocm" routecmv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/routecm" routerv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/router" + storagev2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/storage" pkimanifests "github.com/openshift/hypershift/control-plane-pki-operator/manifests" sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" supportawsutil "github.com/openshift/hypershift/support/awsutil" @@ -230,6 +231,7 @@ func (r *HostedControlPlaneReconciler) registerComponents() { kubevirtccmv2.NewComponent(), openstackccmv2.NewComponent(), powervsccmv2.NewComponent(), + storagev2.NewComponent(), ) } @@ -1301,11 +1303,13 @@ func (r *HostedControlPlaneReconciler) reconcile(ctx context.Context, hostedCont return fmt.Errorf("failed to reconcile image registry operator: %w", err) } - if IsStorageAndCSIManaged(hostedControlPlane) { - // Reconcile cluster storage operator - r.Log.Info("Reconciling cluster storage operator") - if err := r.reconcileClusterStorageOperator(ctx, hostedControlPlane, releaseImageProvider, userReleaseImageProvider, createOrUpdate); err != nil { - return fmt.Errorf("failed to reconcile cluster storage operator: %w", err) + if !r.IsCPOV2 { + if IsStorageAndCSIManaged(hostedControlPlane) { + // Reconcile cluster storage operator + r.Log.Info("Reconciling cluster storage operator") + if err := r.reconcileClusterStorageOperator(ctx, hostedControlPlane, releaseImageProvider, userReleaseImageProvider, createOrUpdate); err != nil { + return fmt.Errorf("failed to reconcile cluster storage operator: %w", err) + } } } diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents.yaml new file mode 100644 index 0000000000..60fd3faa7c --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents.yaml @@ -0,0 +1,212 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + hypershift.openshift.io/managed-by: control-plane-operator + name: cluster-storage-operator + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + name: cluster-storage-operator + strategy: {} + template: + metadata: + annotations: + hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64 + openshift.io/required-scc: restricted-v2 + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + creationTimestamp: null + labels: + hypershift.openshift.io/control-plane-component: cluster-storage-operator + hypershift.openshift.io/hosted-control-plane: hcp-namespace + hypershift.openshift.io/need-management-kas-access: "true" + name: cluster-storage-operator + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: hypershift.openshift.io/control-plane + operator: In + values: + - "true" + weight: 50 + - preference: + matchExpressions: + - key: hypershift.openshift.io/cluster + operator: In + values: + - hcp-namespace + weight: 100 + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + hypershift.openshift.io/hosted-control-plane: hcp-namespace + topologyKey: kubernetes.io/hostname + weight: 100 + automountServiceAccountToken: true + containers: + - args: + - start + - -v=2 + - --terminate-on-files=/var/run/secrets/serving-cert/tls.crt + - --terminate-on-files=/var/run/secrets/serving-cert/tls.key + - --guest-kubeconfig=/etc/guest-kubeconfig/kubeconfig + command: + - cluster-storage-operator + - start + env: + - name: OPERATOR_IMAGE_VERSION + value: "4.18" + - name: OPERAND_IMAGE_VERSION + value: "4.18" + - name: AWS_EBS_DRIVER_OPERATOR_IMAGE + value: aws-ebs-csi-driver-operator + - name: AWS_EBS_DRIVER_IMAGE + value: aws-ebs-csi-driver + - name: GCP_PD_DRIVER_OPERATOR_IMAGE + value: gcp-pd-csi-driver-operator + - name: GCP_PD_DRIVER_IMAGE + value: gcp-pd-csi-driver + - name: OPENSTACK_CINDER_DRIVER_OPERATOR_IMAGE + value: openstack-cinder-csi-driver-operator + - name: OPENSTACK_CINDER_DRIVER_IMAGE + value: openstack-cinder-csi-driver + - name: OVIRT_DRIVER_OPERATOR_IMAGE + value: ovirt-csi-driver-operator + - name: OVIRT_DRIVER_IMAGE + value: ovirt-csi-driver + - name: MANILA_DRIVER_OPERATOR_IMAGE + value: csi-driver-manila-operator + - name: MANILA_DRIVER_IMAGE + value: csi-driver-manila + - name: MANILA_NFS_DRIVER_IMAGE + value: csi-driver-nfs + - name: PROVISIONER_IMAGE + value: csi-external-provisioner + - name: ATTACHER_IMAGE + value: csi-external-attacher + - name: RESIZER_IMAGE + value: csi-external-resizer + - name: SNAPSHOTTER_IMAGE + value: csi-external-snapshotter + - name: NODE_DRIVER_REGISTRAR_IMAGE + value: csi-node-driver-registrar + - name: LIVENESS_PROBE_IMAGE + value: csi-livenessprobe + - name: VSPHERE_PROBLEM_DETECTOR_OPERATOR_IMAGE + value: vsphere-problem-detector + - name: AZURE_DISK_DRIVER_OPERATOR_IMAGE + value: azure-disk-csi-driver-operator + - name: AZURE_DISK_DRIVER_IMAGE + value: azure-disk-csi-driver + - name: AZURE_FILE_DRIVER_OPERATOR_IMAGE + value: azure-file-csi-driver-operator + - name: AZURE_FILE_DRIVER_IMAGE + value: azure-file-csi-driver + - name: KUBE_RBAC_PROXY_IMAGE + value: kube-rbac-proxy + - name: VMWARE_VSPHERE_DRIVER_OPERATOR_IMAGE + value: vsphere-csi-driver-operator + - name: VMWARE_VSPHERE_DRIVER_IMAGE + value: vsphere-csi-driver + - name: VMWARE_VSPHERE_SYNCER_IMAGE + value: vsphere-csi-driver-syncer + - name: CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE + value: cluster-cloud-controller-manager-operator + - name: IBM_VPC_BLOCK_DRIVER_OPERATOR_IMAGE + value: ibm-vpc-block-csi-driver-operator + - name: IBM_VPC_BLOCK_DRIVER_IMAGE + value: ibm-vpc-block-csi-driver + - name: POWERVS_BLOCK_CSI_DRIVER_OPERATOR_IMAGE + value: powervs-block-csi-driver-operator + - name: POWERVS_BLOCK_CSI_DRIVER_IMAGE + value: powervs-block-csi-driver + - name: HYPERSHIFT_IMAGE + value: token-minter + - name: AWS_EBS_DRIVER_CONTROL_PLANE_IMAGE + value: aws-ebs-csi-driver + - name: AZURE_DISK_DRIVER_CONTROL_PLANE_IMAGE + value: azure-disk-csi-driver + - name: LIVENESS_PROBE_CONTROL_PLANE_IMAGE + value: csi-livenessprobe + - name: AZURE_FILE_DRIVER_CONTROL_PLANE_IMAGE + value: azure-file-csi-driver + - name: OPENSTACK_CINDER_DRIVER_CONTROL_PLANE_IMAGE + value: openstack-cinder-csi-driver + - name: MANILA_DRIVER_CONTROL_PLANE_IMAGE + value: csi-driver-manila + - name: KUBE_RBAC_PROXY_CONTROL_PLANE_IMAGE + value: kube-rbac-proxy + - name: TOOLS_IMAGE + value: tools + image: cluster-storage-operator + imagePullPolicy: IfNotPresent + name: cluster-storage-operator + ports: + - containerPort: 8443 + name: metrics + resources: + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /etc/guest-kubeconfig + name: guest-kubeconfig + initContainers: + - command: + - /usr/bin/control-plane-operator + - availability-prober + - --target + - https://kube-apiserver:6443/readyz + - --kubeconfig=/var/kubeconfig/kubeconfig + - --required-api=operator.openshift.io,v1,Storage + image: availability-prober + imagePullPolicy: IfNotPresent + name: availability-prober + resources: {} + volumeMounts: + - mountPath: /var/kubeconfig + name: guest-kubeconfig + priorityClassName: hypershift-control-plane + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + serviceAccountName: cluster-storage-operator + tolerations: + - effect: NoSchedule + key: hypershift.openshift.io/control-plane + operator: Equal + value: "true" + - effect: NoSchedule + key: hypershift.openshift.io/cluster + operator: Equal + value: hcp-namespace + volumes: + - name: guest-kubeconfig + secret: + defaultMode: 416 + secretName: service-network-admin-kubeconfig +status: {} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_TechPreviewNoUpgrade.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_TechPreviewNoUpgrade.yaml new file mode 100644 index 0000000000..60fd3faa7c --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_TechPreviewNoUpgrade.yaml @@ -0,0 +1,212 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + hypershift.openshift.io/managed-by: control-plane-operator + name: cluster-storage-operator + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + name: cluster-storage-operator + strategy: {} + template: + metadata: + annotations: + hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64 + openshift.io/required-scc: restricted-v2 + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + creationTimestamp: null + labels: + hypershift.openshift.io/control-plane-component: cluster-storage-operator + hypershift.openshift.io/hosted-control-plane: hcp-namespace + hypershift.openshift.io/need-management-kas-access: "true" + name: cluster-storage-operator + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: hypershift.openshift.io/control-plane + operator: In + values: + - "true" + weight: 50 + - preference: + matchExpressions: + - key: hypershift.openshift.io/cluster + operator: In + values: + - hcp-namespace + weight: 100 + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + hypershift.openshift.io/hosted-control-plane: hcp-namespace + topologyKey: kubernetes.io/hostname + weight: 100 + automountServiceAccountToken: true + containers: + - args: + - start + - -v=2 + - --terminate-on-files=/var/run/secrets/serving-cert/tls.crt + - --terminate-on-files=/var/run/secrets/serving-cert/tls.key + - --guest-kubeconfig=/etc/guest-kubeconfig/kubeconfig + command: + - cluster-storage-operator + - start + env: + - name: OPERATOR_IMAGE_VERSION + value: "4.18" + - name: OPERAND_IMAGE_VERSION + value: "4.18" + - name: AWS_EBS_DRIVER_OPERATOR_IMAGE + value: aws-ebs-csi-driver-operator + - name: AWS_EBS_DRIVER_IMAGE + value: aws-ebs-csi-driver + - name: GCP_PD_DRIVER_OPERATOR_IMAGE + value: gcp-pd-csi-driver-operator + - name: GCP_PD_DRIVER_IMAGE + value: gcp-pd-csi-driver + - name: OPENSTACK_CINDER_DRIVER_OPERATOR_IMAGE + value: openstack-cinder-csi-driver-operator + - name: OPENSTACK_CINDER_DRIVER_IMAGE + value: openstack-cinder-csi-driver + - name: OVIRT_DRIVER_OPERATOR_IMAGE + value: ovirt-csi-driver-operator + - name: OVIRT_DRIVER_IMAGE + value: ovirt-csi-driver + - name: MANILA_DRIVER_OPERATOR_IMAGE + value: csi-driver-manila-operator + - name: MANILA_DRIVER_IMAGE + value: csi-driver-manila + - name: MANILA_NFS_DRIVER_IMAGE + value: csi-driver-nfs + - name: PROVISIONER_IMAGE + value: csi-external-provisioner + - name: ATTACHER_IMAGE + value: csi-external-attacher + - name: RESIZER_IMAGE + value: csi-external-resizer + - name: SNAPSHOTTER_IMAGE + value: csi-external-snapshotter + - name: NODE_DRIVER_REGISTRAR_IMAGE + value: csi-node-driver-registrar + - name: LIVENESS_PROBE_IMAGE + value: csi-livenessprobe + - name: VSPHERE_PROBLEM_DETECTOR_OPERATOR_IMAGE + value: vsphere-problem-detector + - name: AZURE_DISK_DRIVER_OPERATOR_IMAGE + value: azure-disk-csi-driver-operator + - name: AZURE_DISK_DRIVER_IMAGE + value: azure-disk-csi-driver + - name: AZURE_FILE_DRIVER_OPERATOR_IMAGE + value: azure-file-csi-driver-operator + - name: AZURE_FILE_DRIVER_IMAGE + value: azure-file-csi-driver + - name: KUBE_RBAC_PROXY_IMAGE + value: kube-rbac-proxy + - name: VMWARE_VSPHERE_DRIVER_OPERATOR_IMAGE + value: vsphere-csi-driver-operator + - name: VMWARE_VSPHERE_DRIVER_IMAGE + value: vsphere-csi-driver + - name: VMWARE_VSPHERE_SYNCER_IMAGE + value: vsphere-csi-driver-syncer + - name: CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE + value: cluster-cloud-controller-manager-operator + - name: IBM_VPC_BLOCK_DRIVER_OPERATOR_IMAGE + value: ibm-vpc-block-csi-driver-operator + - name: IBM_VPC_BLOCK_DRIVER_IMAGE + value: ibm-vpc-block-csi-driver + - name: POWERVS_BLOCK_CSI_DRIVER_OPERATOR_IMAGE + value: powervs-block-csi-driver-operator + - name: POWERVS_BLOCK_CSI_DRIVER_IMAGE + value: powervs-block-csi-driver + - name: HYPERSHIFT_IMAGE + value: token-minter + - name: AWS_EBS_DRIVER_CONTROL_PLANE_IMAGE + value: aws-ebs-csi-driver + - name: AZURE_DISK_DRIVER_CONTROL_PLANE_IMAGE + value: azure-disk-csi-driver + - name: LIVENESS_PROBE_CONTROL_PLANE_IMAGE + value: csi-livenessprobe + - name: AZURE_FILE_DRIVER_CONTROL_PLANE_IMAGE + value: azure-file-csi-driver + - name: OPENSTACK_CINDER_DRIVER_CONTROL_PLANE_IMAGE + value: openstack-cinder-csi-driver + - name: MANILA_DRIVER_CONTROL_PLANE_IMAGE + value: csi-driver-manila + - name: KUBE_RBAC_PROXY_CONTROL_PLANE_IMAGE + value: kube-rbac-proxy + - name: TOOLS_IMAGE + value: tools + image: cluster-storage-operator + imagePullPolicy: IfNotPresent + name: cluster-storage-operator + ports: + - containerPort: 8443 + name: metrics + resources: + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /etc/guest-kubeconfig + name: guest-kubeconfig + initContainers: + - command: + - /usr/bin/control-plane-operator + - availability-prober + - --target + - https://kube-apiserver:6443/readyz + - --kubeconfig=/var/kubeconfig/kubeconfig + - --required-api=operator.openshift.io,v1,Storage + image: availability-prober + imagePullPolicy: IfNotPresent + name: availability-prober + resources: {} + volumeMounts: + - mountPath: /var/kubeconfig + name: guest-kubeconfig + priorityClassName: hypershift-control-plane + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + serviceAccountName: cluster-storage-operator + tolerations: + - effect: NoSchedule + key: hypershift.openshift.io/control-plane + operator: Equal + value: "true" + - effect: NoSchedule + key: hypershift.openshift.io/cluster + operator: Equal + value: hcp-namespace + volumes: + - name: guest-kubeconfig + secret: + defaultMode: 416 + secretName: service-network-admin-kubeconfig +status: {} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_component.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_component.yaml new file mode 100644 index 0000000000..8a6f574371 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_component.yaml @@ -0,0 +1,31 @@ +apiVersion: hypershift.openshift.io/v1beta1 +kind: ControlPlaneComponent +metadata: + creationTimestamp: null + name: cluster-storage-operator + namespace: hcp-namespace + resourceVersion: "1" +spec: {} +status: + conditions: + - lastTransitionTime: null + message: cluster-storage-operator Deployment Available condition not found + reason: NotFound + status: "False" + type: Available + - lastTransitionTime: null + message: cluster-storage-operator Deployment Progressing condition not found + reason: NotFound + status: "False" + type: Progressing + resources: + - group: rbac.authorization.k8s.io + kind: Role + name: cluster-storage-operator + - group: rbac.authorization.k8s.io + kind: RoleBinding + name: cluster-storage-operator + - group: "" + kind: ServiceAccount + name: cluster-storage-operator + version: "4.18" diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_component_TechPreviewNoUpgrade.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_component_TechPreviewNoUpgrade.yaml new file mode 100644 index 0000000000..8a6f574371 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-storage-operator/zz_fixture_TestControlPlaneComponents_component_TechPreviewNoUpgrade.yaml @@ -0,0 +1,31 @@ +apiVersion: hypershift.openshift.io/v1beta1 +kind: ControlPlaneComponent +metadata: + creationTimestamp: null + name: cluster-storage-operator + namespace: hcp-namespace + resourceVersion: "1" +spec: {} +status: + conditions: + - lastTransitionTime: null + message: cluster-storage-operator Deployment Available condition not found + reason: NotFound + status: "False" + type: Available + - lastTransitionTime: null + message: cluster-storage-operator Deployment Progressing condition not found + reason: NotFound + status: "False" + type: Progressing + resources: + - group: rbac.authorization.k8s.io + kind: Role + name: cluster-storage-operator + - group: rbac.authorization.k8s.io + kind: RoleBinding + name: cluster-storage-operator + - group: "" + kind: ServiceAccount + name: cluster-storage-operator + version: "4.18" diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/README.md b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/README.md new file mode 100644 index 0000000000..c54ed5a0e2 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/README.md @@ -0,0 +1,11 @@ +# Templates of cluster-storage-operator objects. + +Generated files in this directory with "hypershift" name come from https://github.com/openshift/cluster-storage-operator. + +* To update the generated file, copy + [`manifests/10_deployment-hypershift.yaml`](https://github.com/openshift/cluster-storage-operator/blob/master/manifests/10_deployment-hypershift.yaml) + from github.com/openshift/cluster-storage-operator to `deployment.yaml` and change the container image from `quay.io/openshift/origin-cluster-storage-operator:latest` to `cluster-storage-operator` +* All changes in these files should be done in github.com/openshift/cluster-storage-operator repository, + so we keep authoritative source of all cluster-storage-operator objects on a single place. + +Role and rolebinding is custom for hypershift only. diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-disk-csi-config.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-disk-csi-config.yaml new file mode 100644 index 0000000000..4edab1c23b --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-disk-csi-config.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +data: + cloud.conf: "" +kind: Secret +metadata: + name: azure-disk-csi-config diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-disk-csi-secretprovider.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-disk-csi-secretprovider.yaml new file mode 100644 index 0000000000..cf10556b92 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-disk-csi-secretprovider.yaml @@ -0,0 +1,4 @@ +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: managed-azure-disk-csi diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-file-csi-config.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-file-csi-config.yaml new file mode 100644 index 0000000000..ec6480ed93 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-file-csi-config.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +data: + cloud.conf: "" +kind: Secret +metadata: + name: azure-disk-file-config diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-file-csi-secretprovider.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-file-csi-secretprovider.yaml new file mode 100644 index 0000000000..a73397bc27 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/azure-file-csi-secretprovider.yaml @@ -0,0 +1,4 @@ +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: managed-azure-file-csi diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/deployment.yaml new file mode 100644 index 0000000000..5ba0ecb406 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/deployment.yaml @@ -0,0 +1,140 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-storage-operator +spec: + replicas: 1 + selector: + matchLabels: + name: cluster-storage-operator + template: + metadata: + annotations: + openshift.io/required-scc: restricted-v2 + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + labels: + name: cluster-storage-operator + spec: + containers: + - args: + - start + - -v=2 + - --terminate-on-files=/var/run/secrets/serving-cert/tls.crt + - --terminate-on-files=/var/run/secrets/serving-cert/tls.key + - --guest-kubeconfig=/etc/guest-kubeconfig/kubeconfig + command: + - cluster-storage-operator + - start + env: + - name: OPERATOR_IMAGE_VERSION + value: 0.0.1-snapshot + - name: OPERAND_IMAGE_VERSION + value: 0.0.1-snapshot + - name: AWS_EBS_DRIVER_OPERATOR_IMAGE + value: quay.io/openshift/origin-aws-ebs-csi-driver-operator:latest + - name: AWS_EBS_DRIVER_IMAGE + value: quay.io/openshift/origin-aws-ebs-csi-driver:latest + - name: GCP_PD_DRIVER_OPERATOR_IMAGE + value: quay.io/openshift/origin-gcp-pd-csi-driver-operator:latest + - name: GCP_PD_DRIVER_IMAGE + value: quay.io/openshift/origin-gcp-pd-csi-driver:latest + - name: OPENSTACK_CINDER_DRIVER_OPERATOR_IMAGE + value: quay.io/openshift/origin-openstack-cinder-csi-driver-operator:latest + - name: OPENSTACK_CINDER_DRIVER_IMAGE + value: quay.io/openshift/origin-openstack-cinder-csi-driver:latest + - name: OVIRT_DRIVER_OPERATOR_IMAGE + value: quay.io/openshift/origin-ovirt-csi-driver-operator:latest + - name: OVIRT_DRIVER_IMAGE + value: quay.io/openshift/origin-ovirt-csi-driver:latest + - name: MANILA_DRIVER_OPERATOR_IMAGE + value: quay.io/openshift/origin-csi-driver-manila-operator:latest + - name: MANILA_DRIVER_IMAGE + value: quay.io/openshift/origin-csi-driver-manila:latest + - name: MANILA_NFS_DRIVER_IMAGE + value: quay.io/openshift/origin-csi-driver-nfs:latest + - name: PROVISIONER_IMAGE + value: quay.io/openshift/origin-csi-external-provisioner:latest + - name: ATTACHER_IMAGE + value: quay.io/openshift/origin-csi-external-attacher:latest + - name: RESIZER_IMAGE + value: quay.io/openshift/origin-csi-external-resizer:latest + - name: SNAPSHOTTER_IMAGE + value: quay.io/openshift/origin-csi-external-snapshotter:latest + - name: NODE_DRIVER_REGISTRAR_IMAGE + value: quay.io/openshift/origin-csi-node-driver-registrar:latest + - name: LIVENESS_PROBE_IMAGE + value: quay.io/openshift/origin-csi-livenessprobe:latest + - name: VSPHERE_PROBLEM_DETECTOR_OPERATOR_IMAGE + value: quay.io/openshift/origin-vsphere-problem-detector:latest + - name: AZURE_DISK_DRIVER_OPERATOR_IMAGE + value: quay.io/openshift/origin-azure-disk-csi-driver-operator:latest + - name: AZURE_DISK_DRIVER_IMAGE + value: quay.io/openshift/origin-azure-disk-csi-driver:latest + - name: AZURE_FILE_DRIVER_OPERATOR_IMAGE + value: quay.io/openshift/origin-azure-file-csi-driver-operator:latest + - name: AZURE_FILE_DRIVER_IMAGE + value: quay.io/openshift/origin-azure-file-csi-driver:latest + - name: KUBE_RBAC_PROXY_IMAGE + value: quay.io/openshift/origin-kube-rbac-proxy:latest + - name: VMWARE_VSPHERE_DRIVER_OPERATOR_IMAGE + value: quay.io/openshift/origin-vsphere-csi-driver-operator:latest + - name: VMWARE_VSPHERE_DRIVER_IMAGE + value: quay.io/openshift/origin-vsphere-csi-driver:latest + - name: VMWARE_VSPHERE_SYNCER_IMAGE + value: quay.io/openshift/origin-vsphere-csi-driver-syncer:latest + - name: CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE + value: quay.io/openshift/origin-cluster-cloud-controller-manager-operator:latest + - name: IBM_VPC_BLOCK_DRIVER_OPERATOR_IMAGE + value: quay.io/openshift/origin-ibm-vpc-block-csi-driver-operator:latest + - name: IBM_VPC_BLOCK_DRIVER_IMAGE + value: quay.io/openshift/origin-ibm-vpc-block-csi-driver:latest + - name: POWERVS_BLOCK_CSI_DRIVER_OPERATOR_IMAGE + value: quay.io/openshift/origin-powervs-block-csi-driver-operator:latest + - name: POWERVS_BLOCK_CSI_DRIVER_IMAGE + value: quay.io/openshift/origin-powervs-block-csi-driver:latest + - name: HYPERSHIFT_IMAGE + value: quay.io/openshift/origin-control-plane:latest + - name: AWS_EBS_DRIVER_CONTROL_PLANE_IMAGE + value: quay.io/openshift/origin-aws-ebs-csi-driver:latest + - name: AZURE_DISK_DRIVER_CONTROL_PLANE_IMAGE + value: quay.io/openshift/origin-azure-disk-csi-driver-operator:latest + - name: LIVENESS_PROBE_CONTROL_PLANE_IMAGE + value: quay.io/openshift/origin-csi-livenessprobe:latest + - name: AZURE_FILE_DRIVER_CONTROL_PLANE_IMAGE + value: quay.io/openshift/origin-azure-file-csi-driver-operator:latest + - name: OPENSTACK_CINDER_DRIVER_CONTROL_PLANE_IMAGE + value: quay.io/openshift/origin-openstack-cinder-csi-driver-operator:latest + - name: MANILA_DRIVER_CONTROL_PLANE_IMAGE + value: quay.io/openshift/origin-csi-driver-manila-operator:latest + - name: KUBE_RBAC_PROXY_CONTROL_PLANE_IMAGE + value: quay.io/openshift/origin-kube-rbac-proxy:latest + - name: TOOLS_IMAGE + value: quay.io/openshift/origin-tools:latest + image: cluster-storage-operator + imagePullPolicy: IfNotPresent + name: cluster-storage-operator + ports: + - containerPort: 8443 + name: metrics + resources: + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /etc/guest-kubeconfig + name: guest-kubeconfig + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + serviceAccountName: cluster-storage-operator + volumes: + - name: guest-kubeconfig + secret: + secretName: service-network-admin-kubeconfig \ No newline at end of file diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/role.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/role.yaml new file mode 100644 index 0000000000..4ed70ddfbe --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/role.yaml @@ -0,0 +1,66 @@ +# TODO: Prune the RBACs as cluster-storage-operator and aws-ebs-csi-driver-operator are pruned. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-storage-operator +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - '*' + - apiGroups: + - apps + resources: + - deployments + - replicasets + verbs: + - '*' + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - '*' + - apiGroups: + - "" + resources: + - serviceaccounts + - services + - configmaps + - pods + - endpoints + - events + - secrets + verbs: + - '*' + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - watch + - list + - get + - create + - delete + - patch + - update + - apiGroups: + - hypershift.openshift.io + resources: + - hostedcontrolplanes + verbs: + - watch + - list + - get diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/rolebinding.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/rolebinding.yaml new file mode 100644 index 0000000000..6c4f385e27 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + name: cluster-storage-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-storage-operator +subjects: +- kind: ServiceAccount + name: cluster-storage-operator diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/serviceaccount.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/serviceaccount.yaml new file mode 100644 index 0000000000..64949c93e3 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-storage-operator/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cluster-storage-operator + diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config.go index 34b9791fcf..bfcf74e627 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure/config.go @@ -17,7 +17,7 @@ import ( ) const ( - configKey = "cloud.conf" + ConfigKey = "cloud.conf" ) func adaptConfig(cpContext component.WorkloadContext, cm *corev1.ConfigMap) error { @@ -31,7 +31,7 @@ func adaptConfig(cpContext component.WorkloadContext, cm *corev1.ConfigMap) erro return fmt.Errorf("failed to serialize cloudconfig: %w", err) } - cm.Data[configKey] = string(serializedConfig) + cm.Data[ConfigKey] = string(serializedConfig) return nil } @@ -48,7 +48,7 @@ func adaptConfigSecret(cpContext component.WorkloadContext, secret *corev1.Secre return fmt.Errorf("failed to serialize cloudconfig: %w", err) } - secret.Data[configKey] = serializedConfig + secret.Data[ConfigKey] = serializedConfig return nil } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/storage/azure.go b/control-plane-operator/controllers/hostedcontrolplane/v2/storage/azure.go new file mode 100644 index 0000000000..df4577c8b0 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/storage/azure.go @@ -0,0 +1,70 @@ +package storage + +import ( + "encoding/json" + "fmt" + "path" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/secretproviderclass" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure" + hyperconfig "github.com/openshift/hypershift/support/config" + component "github.com/openshift/hypershift/support/controlplane-component" + + corev1 "k8s.io/api/core/v1" + + "sigs.k8s.io/controller-runtime/pkg/client" + secretsstorev1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" +) + +func adaptAzureCSISecret(cpContext component.WorkloadContext, managedIdentity hyperv1.ManagedIdentity, secret *corev1.Secret) error { + // Get the credentials secret so we can retrieve the tenant ID for the configuration + credentialsSecret := manifests.AzureCredentialInformation(cpContext.HCP.Namespace) + if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { + return fmt.Errorf("failed to get Azure credentials secret: %w", err) + } + + tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) + azureSpec := cpContext.HCP.Spec.Platform.Azure + + azureConfig := azure.AzureConfig{ + Cloud: azureSpec.Cloud, + TenantID: tenantID, + SubscriptionID: azureSpec.SubscriptionID, + ResourceGroup: azureSpec.ResourceGroupName, + Location: azureSpec.Location, + AADClientID: managedIdentity.ClientID, + AADClientCertPath: path.Join(hyperconfig.ManagedAzureCertificatePath, managedIdentity.CertificateName), + } + + serializedConfig, err := json.MarshalIndent(azureConfig, "", " ") + if err != nil { + return fmt.Errorf("failed to serialize cloudconfig: %w", err) + } + + secret.Data[azure.ConfigKey] = serializedConfig + return nil +} + +func adaptAzureCSIDiskSecret(cpContext component.WorkloadContext, secret *corev1.Secret) error { + managedIdentity := cpContext.HCP.Spec.Platform.Azure.ManagedIdentities.ControlPlane.Disk + return adaptAzureCSISecret(cpContext, managedIdentity, secret) +} + +func adaptAzureCSIDiskSecretProvider(cpContext component.WorkloadContext, secretProvider *secretsstorev1.SecretProviderClass) error { + managedIdentity := cpContext.HCP.Spec.Platform.Azure.ManagedIdentities.ControlPlane.Disk + secretproviderclass.ReconcileManagedAzureSecretProviderClass(secretProvider, cpContext.HCP, managedIdentity.CertificateName) + return nil +} + +func adaptAzureCSIFileSecret(cpContext component.WorkloadContext, secret *corev1.Secret) error { + managedIdentity := cpContext.HCP.Spec.Platform.Azure.ManagedIdentities.ControlPlane.File + return adaptAzureCSISecret(cpContext, managedIdentity, secret) +} + +func adaptAzureCSIFileSecretProvider(cpContext component.WorkloadContext, secretProvider *secretsstorev1.SecretProviderClass) error { + managedIdentity := cpContext.HCP.Spec.Platform.Azure.ManagedIdentities.ControlPlane.File + secretproviderclass.ReconcileManagedAzureSecretProviderClass(secretProvider, cpContext.HCP, managedIdentity.CertificateName) + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/storage/component.go b/control-plane-operator/controllers/hostedcontrolplane/v2/storage/component.go new file mode 100644 index 0000000000..3390b168d8 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/storage/component.go @@ -0,0 +1,80 @@ +package storage + +import ( + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + oapiv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/oapi" + "github.com/openshift/hypershift/support/azureutil" + component "github.com/openshift/hypershift/support/controlplane-component" + "github.com/openshift/hypershift/support/util" + + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const ( + ComponentName = "cluster-storage-operator" +) + +var _ component.ComponentOptions = &clusterStorageOperator{} + +type clusterStorageOperator struct { +} + +// IsRequestServing implements controlplanecomponent.ComponentOptions. +func (r *clusterStorageOperator) IsRequestServing() bool { + return false +} + +// MultiZoneSpread implements controlplanecomponent.ComponentOptions. +func (r *clusterStorageOperator) MultiZoneSpread() bool { + return false +} + +// NeedsManagementKASAccess implements controlplanecomponent.ComponentOptions. +func (r *clusterStorageOperator) NeedsManagementKASAccess() bool { + return true +} + +func NewComponent() component.ControlPlaneComponent { + return component.NewDeploymentComponent(ComponentName, &clusterStorageOperator{}). + WithAdaptFunction(adaptDeployment). + WithPredicate(isStorageAndCSIManaged). + WithManifestAdapter( + "azure-disk-csi-config.yaml", + component.WithAdaptFunction(adaptAzureCSIDiskSecret), + component.WithPredicate(isAroHCP), + ). + WithManifestAdapter( + "azure-disk-csi-secretprovider.yaml", + component.WithAdaptFunction(adaptAzureCSIDiskSecretProvider), + component.WithPredicate(isAroHCP), + ). + WithManifestAdapter( + "azure-file-csi-config.yaml", + component.WithAdaptFunction(adaptAzureCSIFileSecret), + component.WithPredicate(isAroHCP), + ). + WithManifestAdapter( + "azure-file-csi-secretprovider.yaml", + component.WithAdaptFunction(adaptAzureCSIFileSecretProvider), + component.WithPredicate(isAroHCP), + ). + WithDependencies(oapiv2.ComponentName). + InjectAvailabilityProberContainer(util.AvailabilityProberOpts{ + KubeconfigVolumeName: "guest-kubeconfig", + RequiredAPIs: []schema.GroupVersionKind{ + {Group: "operator.openshift.io", Version: "v1", Kind: "Storage"}, + }, + }). + Build() +} + +func isStorageAndCSIManaged(cpContext component.WorkloadContext) (bool, error) { + if cpContext.HCP.Spec.Platform.Type == hyperv1.IBMCloudPlatform || cpContext.HCP.Spec.Platform.Type == hyperv1.PowerVSPlatform { + return false, nil + } + return true, nil +} + +func isAroHCP(cpContext component.WorkloadContext) bool { + return azureutil.IsAroHCP() +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/storage/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/storage/deployment.go new file mode 100644 index 0000000000..04da69b1b8 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/storage/deployment.go @@ -0,0 +1,35 @@ +package storage + +import ( + "github.com/openshift/hypershift/support/azureutil" + "github.com/openshift/hypershift/support/config" + component "github.com/openshift/hypershift/support/controlplane-component" + "github.com/openshift/hypershift/support/util" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Deployment) error { + util.UpdateContainer(ComponentName, deployment.Spec.Template.Spec.Containers, func(c *corev1.Container) { + envReplacer := newEnvironmentReplacer(cpContext.ReleaseImageProvider, cpContext.UserReleaseImageProvider) + envReplacer.replaceEnvVars(c.Env) + + // For managed Azure, we need to supply a couple of environment variables for CSO to pass on to the CSI controllers for disk and file. + // CSO passes those on to the CSI deployment here - https://github.com/openshift/cluster-storage-operator/pull/517/files. + // CSI then mounts the Secrets Provider Class here - https://github.com/openshift/csi-operator/pull/309/files. + if azureutil.IsAroHCP() { + c.Env = append(c.Env, + corev1.EnvVar{ + Name: "ARO_HCP_SECRET_PROVIDER_CLASS_FOR_DISK", + Value: config.ManagedAzureDiskCSISecretStoreProviderClassName, + }, + corev1.EnvVar{ + Name: "ARO_HCP_SECRET_PROVIDER_CLASS_FOR_FILE", + Value: config.ManagedAzureFileCSISecretStoreProviderClassName, + }) + } + }) + + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/storage/envreplace.go b/control-plane-operator/controllers/hostedcontrolplane/v2/storage/envreplace.go new file mode 100644 index 0000000000..547524b407 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/storage/envreplace.go @@ -0,0 +1,115 @@ +package storage + +import ( + "strings" + + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/strings/slices" +) + +var ( + // map env. variable in CSO Deployment -> name of the image in payload. + operatorImageRefs = map[string]string{ + "AWS_EBS_DRIVER_OPERATOR_IMAGE": "aws-ebs-csi-driver-operator", + "AWS_EBS_DRIVER_IMAGE": "aws-ebs-csi-driver", + "GCP_PD_DRIVER_OPERATOR_IMAGE": "gcp-pd-csi-driver-operator", + "GCP_PD_DRIVER_IMAGE": "gcp-pd-csi-driver", + "OPENSTACK_CINDER_DRIVER_OPERATOR_IMAGE": "openstack-cinder-csi-driver-operator", + "OPENSTACK_CINDER_DRIVER_IMAGE": "openstack-cinder-csi-driver", + "OVIRT_DRIVER_OPERATOR_IMAGE": "ovirt-csi-driver-operator", + "OVIRT_DRIVER_IMAGE": "ovirt-csi-driver", + "MANILA_DRIVER_OPERATOR_IMAGE": "csi-driver-manila-operator", + "MANILA_DRIVER_IMAGE": "csi-driver-manila", + "MANILA_NFS_DRIVER_IMAGE": "csi-driver-nfs", + "PROVISIONER_IMAGE": "csi-external-provisioner", + "ATTACHER_IMAGE": "csi-external-attacher", + "RESIZER_IMAGE": "csi-external-resizer", + "SNAPSHOTTER_IMAGE": "csi-external-snapshotter", + "NODE_DRIVER_REGISTRAR_IMAGE": "csi-node-driver-registrar", + "LIVENESS_PROBE_IMAGE": "csi-livenessprobe", + "VSPHERE_PROBLEM_DETECTOR_OPERATOR_IMAGE": "vsphere-problem-detector", + "AZURE_DISK_DRIVER_OPERATOR_IMAGE": "azure-disk-csi-driver-operator", + "AZURE_DISK_DRIVER_IMAGE": "azure-disk-csi-driver", + "AZURE_FILE_DRIVER_OPERATOR_IMAGE": "azure-file-csi-driver-operator", + "AZURE_FILE_DRIVER_IMAGE": "azure-file-csi-driver", + "KUBE_RBAC_PROXY_IMAGE": "kube-rbac-proxy", + "VMWARE_VSPHERE_DRIVER_OPERATOR_IMAGE": "vsphere-csi-driver-operator", + "VMWARE_VSPHERE_DRIVER_IMAGE": "vsphere-csi-driver", + "VMWARE_VSPHERE_SYNCER_IMAGE": "vsphere-csi-driver-syncer", + "CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE": "cluster-cloud-controller-manager-operator", + "SHARED_RESOURCE_DRIVER_OPERATOR_IMAGE": "csi-driver-shared-resource-operator", + "SHARED_RESOURCE_DRIVER_IMAGE": "csi-driver-shared-resource", + "SHARED_RESOURCE_DRIVER_WEBHOOK_IMAGE": "csi-driver-shared-resource-webhook", + "ALIBABA_DISK_DRIVER_OPERATOR_IMAGE": "alibaba-disk-csi-driver-operator", + "ALIBABA_CLOUD_DRIVER_IMAGE": "alibaba-cloud-csi-driver", + "IBM_VPC_BLOCK_DRIVER_OPERATOR_IMAGE": "ibm-vpc-block-csi-driver-operator", + "IBM_VPC_BLOCK_DRIVER_IMAGE": "ibm-vpc-block-csi-driver", + "IBM_VPC_NODE_LABEL_UPDATER_IMAGE": "ibm-vpc-node-label-updater", + "POWERVS_BLOCK_CSI_DRIVER_OPERATOR_IMAGE": "powervs-block-csi-driver-operator", + "POWERVS_BLOCK_CSI_DRIVER_IMAGE": "powervs-block-csi-driver", + "HYPERSHIFT_IMAGE": "token-minter", + "AWS_EBS_DRIVER_CONTROL_PLANE_IMAGE": "aws-ebs-csi-driver", + "AZURE_DISK_DRIVER_CONTROL_PLANE_IMAGE": "azure-disk-csi-driver", + "AZURE_FILE_DRIVER_CONTROL_PLANE_IMAGE": "azure-file-csi-driver", + "OPENSTACK_CINDER_DRIVER_CONTROL_PLANE_IMAGE": "openstack-cinder-csi-driver", + "MANILA_DRIVER_CONTROL_PLANE_IMAGE": "csi-driver-manila", + "LIVENESS_PROBE_CONTROL_PLANE_IMAGE": "csi-livenessprobe", + "KUBE_RBAC_PROXY_CONTROL_PLANE_IMAGE": "kube-rbac-proxy", + "TOOLS_IMAGE": "tools", + } +) + +type environmentReplacer struct { + // map env variable name -> new env. variable value + values map[string]string +} + +func newEnvironmentReplacer(releaseImageProvider, userReleaseImageProvider imageprovider.ReleaseImageProvider) *environmentReplacer { + er := &environmentReplacer{values: map[string]string{}} + + version := userReleaseImageProvider.Version() + er.setVersions(version) + er.setOperatorImageReferences(releaseImageProvider, userReleaseImageProvider) + + return er +} + +func (er *environmentReplacer) setOperatorImageReferences(releaseImageProvider, userReleaseImageProvider imageprovider.ReleaseImageProvider) { + // `operatorImageRefs` is map from env. var name -> payload image name + // `images` is map from payload image name -> image URL + // Create map from env. var name -> image URL + + dataPlaneImageRefs := []string{ + "NODE_DRIVER_REGISTRAR_IMAGE", + "LIVENESS_PROBE_IMAGE", + "CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE", + "KUBE_RBAC_PROXY_IMAGE", + } + + for envVar, payloadName := range operatorImageRefs { + if slices.Contains(dataPlaneImageRefs, envVar) || strings.HasSuffix(envVar, "_DRIVER_IMAGE") { + if imageURL, ok := userReleaseImageProvider.ImageExist(payloadName); ok { + er.values[envVar] = imageURL + } + } else { + if imageURL, ok := releaseImageProvider.ImageExist(payloadName); ok { + er.values[envVar] = imageURL + } + } + } +} + +func (er *environmentReplacer) setVersions(ver string) { + er.values["OPERATOR_IMAGE_VERSION"] = ver + er.values["OPERAND_IMAGE_VERSION"] = ver +} + +func (er *environmentReplacer) replaceEnvVars(envVars []corev1.EnvVar) { + for i := range envVars { + if value, ok := er.values[envVars[i].Name]; ok { + envVars[i].Value = value + } + } +}