diff --git a/Makefile b/Makefile index 51cf8647f3..7648e32270 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ build: hypershift-operator control-plane-operator control-plane-pki-operator hyp .PHONY: sync sync: - $(GO) work sync + $(GO) work sync .PHONY: update update: sync api-deps api api-docs deps clients diff --git a/control-plane-operator/controllers/hostedcontrolplane/configoperator/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/configoperator/reconcile.go index 8a40bc6b0c..a9ef94280f 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/configoperator/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/configoperator/reconcile.go @@ -232,6 +232,7 @@ func ReconcileRole(role *rbacv1.Role, ownerRef config.OwnerRef, platform hyperv1 }, }...) } + // TODO (jparrill): Add RBAC specific needs for Agent platform return nil } @@ -443,7 +444,7 @@ func buildHCCContainerMain(image, hcpName, openShiftVersion, kubeVersion string, func buildHCCVolumeKubeconfig(v *corev1.Volume) { v.Secret = &corev1.SecretVolumeSource{ - SecretName: manifests.KASServiceKubeconfigSecret("").Name, + SecretName: manifests.HCCOKubeconfigSecret("").Name, DefaultMode: pointer.Int32(0640), } } diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 67c43ac447..cd49621d5b 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -2819,6 +2819,10 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex if err := r.Get(ctx, client.ObjectKeyFromObject(bootstrapClientCertSecret), bootstrapClientCertSecret); err != nil { return fmt.Errorf("failed to get bootstrap client cert secret: %w", err) } + hccoClientCertSecret := manifests.HCCOClientCertSecret(hcp.Namespace) + if err := r.Get(ctx, client.ObjectKeyFromObject(hccoClientCertSecret), hccoClientCertSecret); err != nil { + return fmt.Errorf("failed to get HCCO client cert secret: %w", err) + } serviceKubeconfigSecret := manifests.KASServiceKubeconfigSecret(hcp.Namespace) if _, err := createOrUpdate(ctx, r, serviceKubeconfigSecret, func() error { @@ -2838,6 +2842,13 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex return fmt.Errorf("failed to reconcile CAPI service admin kubeconfig secret: %w", err) } + hccoKubeconfigSecret := manifests.HCCOKubeconfigSecret(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, hccoKubeconfigSecret, func() error { + return kas.ReconcileHCCOKubeconfigSecret(hccoKubeconfigSecret, hccoClientCertSecret, rootCA, p.OwnerRef, hcp.Spec.Platform.Type) + }); err != nil { + return fmt.Errorf("failed to reconcile HCCO kubeconfig secret: %w", err) + } + localhostKubeconfigSecret := manifests.KASLocalhostKubeconfigSecret(hcp.Namespace) if _, err := createOrUpdate(ctx, r, localhostKubeconfigSecret, func() error { return kas.ReconcileLocalhostKubeconfigSecret(localhostKubeconfigSecret, clientCertSecret, rootCA, p.OwnerRef, p.KASPodPort) diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/config.go b/control-plane-operator/controllers/hostedcontrolplane/kas/config.go index 004b9f8cb2..25ef3b79f2 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/config.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/config.go @@ -181,6 +181,7 @@ func generateConfig(p KubeAPIServerConfigParams) *kcpv1.KubeAPIServerConfig { // TODO remove in 4.16 once we're able to have different featuregates for hypershift featureGates := append([]string{}, p.FeatureGates...) featureGates = append(featureGates, "StructuredAuthenticationConfiguration=true") + featureGates = append(featureGates, "ValidatingAdmissionPolicy=true") args.Set("feature-gates", featureGates...) args.Set("goaway-chance", "0") args.Set("http2-max-streams-per-connection", "2000") diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go index b34ea7aa8b..693ec6715c 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go @@ -756,6 +756,22 @@ cat </tmp/manifests/99_feature-gate.yaml %[3]s EOF +touch /tmp/manifests/hcco-rolebinding.yaml +cat </tmp/manifests/hcco-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hcco-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:hosted-cluster-config +EOF + /usr/bin/render \ --asset-output-dir /tmp/output \ --rendered-manifest-dir=/tmp/manifests \ diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/kubeconfig.go b/control-plane-operator/controllers/hostedcontrolplane/kas/kubeconfig.go index 065e426d96..e544ae054a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/kubeconfig.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/kubeconfig.go @@ -58,3 +58,8 @@ func ReconcileExternalKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.C func ReconcileBootstrapKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.ConfigMap, ownerRef config.OwnerRef, externalURL string) error { return pki.ReconcileKubeConfig(secret, cert, ca, externalURL, "", manifests.KubeconfigScopeBootstrap, ownerRef) } + +func ReconcileHCCOKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.ConfigMap, ownerRef config.OwnerRef, platformType hyperv1.PlatformType) error { + svcURL := InClusterKASURL(platformType) + return pki.ReconcileKubeConfig(secret, cert, ca, svcURL, "", "service", ownerRef) +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go b/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go index c3209d1b98..0facaab796 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go @@ -159,6 +159,29 @@ func (r *HostedControlPlaneReconciler) setupKASClientSigners( return err } + // ---------- + // HCCO signer + // ---------- + + hccoKubeconfigSigner, err := reconcileSigner( + manifests.HCCOSigner(hcp.Namespace), + pki.ReconcileHCCOSigner, + ) + + if err != nil { + return err + } + totalClientCABundle = append(totalClientCABundle, hccoKubeconfigSigner) + + // system:hosted-cluster-config client cert + if _, err := reconcileSub( + manifests.HCCOClientCertSecret(hcp.Namespace), + hccoKubeconfigSigner, + pki.ReconcileHCCOClientCertSecret, + ); err != nil { + return err + } + // ---------- // CSR signer // ---------- diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/configoperator.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/configoperator.go index 733fa837d1..111b0fb7d3 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/configoperator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/configoperator.go @@ -18,21 +18,21 @@ func ConfigOperatorDeployment(ns string) *appsv1.Deployment { func ConfigOperatorRole(ns string) *rbacv1.Role { r := &rbacv1.Role{} - r.Name = "hosted-cluster-config-operator" + r.Name = "hosted-cluster-config" r.Namespace = ns return r } func ConfigOperatorRoleBinding(ns string) *rbacv1.RoleBinding { rb := &rbacv1.RoleBinding{} - rb.Name = "hosted-cluster-config-operator" + rb.Name = "hosted-cluster-config" rb.Namespace = ns return rb } func ConfigOperatorServiceAccount(ns string) *corev1.ServiceAccount { sa := &corev1.ServiceAccount{} - sa.Name = "hosted-cluster-config-operator" + sa.Name = "hosted-cluster-config" sa.Namespace = ns return sa } @@ -40,6 +40,6 @@ func ConfigOperatorServiceAccount(ns string) *corev1.ServiceAccount { func ConfigOperatorPodMonitor(ns string) *prometheusoperatorv1.PodMonitor { return &prometheusoperatorv1.PodMonitor{ObjectMeta: metav1.ObjectMeta{ Namespace: ns, - Name: "hosted-cluster-config-operator", + Name: "hosted-cluster-config", }} } diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go index 7e354186a1..c90a7bcacf 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go @@ -63,6 +63,15 @@ func KASServiceCAPIKubeconfigSecret(controlPlaneNamespace, infraID string) *core } } +func HCCOKubeconfigSecret(controlPlaneNamespace string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hcco-kubeconfig", + Namespace: controlPlaneNamespace, + }, + } +} + func KASExternalKubeconfigSecret(controlPlaneNamespace string, ref *hyperv1.KubeconfigSecretRef) *corev1.Secret { s := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go index 626cc9a29d..92031576a4 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go @@ -224,6 +224,14 @@ func SystemAdminClientCertSecret(ns string) *corev1.Secret { return secretFor(ns, "system-admin-client") } +func HCCOSigner(ns string) *corev1.Secret { + return secretFor(ns, "hcco-signer") +} + +func HCCOClientCertSecret(ns string) *corev1.Secret { + return secretFor(ns, "hcco-client") +} + func KASMachineBootstrapClientCertSecret(ns string) *corev1.Secret { return secretFor(ns, "kas-bootstrap-client") } diff --git a/control-plane-operator/controllers/hostedcontrolplane/pki/ca.go b/control-plane-operator/controllers/hostedcontrolplane/pki/ca.go index cb4628a3a7..ce3b41fe47 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/pki/ca.go +++ b/control-plane-operator/controllers/hostedcontrolplane/pki/ca.go @@ -49,6 +49,10 @@ func ReconcileAdminKubeconfigSigner(secret *corev1.Secret, ownerRef config.Owner return reconcileSelfSignedCA(secret, ownerRef, "admin-kubeconfig-signer", "openshift") } +func ReconcileHCCOSigner(secret *corev1.Secret, ownerRef config.OwnerRef) error { + return reconcileSelfSignedCA(secret, ownerRef, "hcco-signer", "openshift") +} + func ReconcileKubeCSRSigner(secret *corev1.Secret, ownerRef config.OwnerRef) error { return reconcileSelfSignedCA(secret, ownerRef, "kube-csr-signer", "openshift") } diff --git a/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go b/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go index 2455d449ed..e8da0dbcb7 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go +++ b/control-plane-operator/controllers/hostedcontrolplane/pki/kas.go @@ -90,6 +90,10 @@ func ReconcileSystemAdminClientCertSecret(secret, ca *corev1.Secret, ownerRef co return reconcileSignedCert(secret, ca, ownerRef, "system:admin", []string{"system:masters"}, X509UsageClientAuth) } +func ReconcileHCCOClientCertSecret(secret, ca *corev1.Secret, ownerRef config.OwnerRef) error { + return reconcileSignedCert(secret, ca, ownerRef, fmt.Sprintf("system:%s", config.HCCOUser), []string{"kubernetes"}, X509UsageClientAuth) +} + func ReconcileServiceAccountKubeconfig(secret, csrSigner *corev1.Secret, ca *corev1.ConfigMap, hcp *hyperv1.HostedControlPlane, serviceAccountNamespace, serviceAccountName string) error { cn := serviceaccount.MakeUsername(serviceAccountNamespace, serviceAccountName) if err := reconcileSignedCert(secret, csrSigner, config.OwnerRef{}, cn, serviceaccount.MakeGroupNames(serviceAccountNamespace), X509UsageClientAuth); err != nil { diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies.go new file mode 100644 index 0000000000..d5417d00e5 --- /dev/null +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies.go @@ -0,0 +1,170 @@ +package kas + +import ( + "context" + "fmt" + + configv1 "github.com/openshift/api/config/v1" + operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests" + "github.com/openshift/hypershift/support/config" + "github.com/openshift/hypershift/support/upsert" + k8sadmissionv1beta1 "k8s.io/api/admissionregistration/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type AdmissionPolicy struct { + Name string + MatchConstraints *k8sadmissionv1beta1.MatchResources + Validations []k8sadmissionv1beta1.Validation +} + +const ( + AdmissionPolicyNameConfig = "config" + AdmissionPolicyNameMirror = "mirror" + AdmissionPolicyNameICSP = "icsp" +) + +var ( + allAdmissionPoliciesOperations = []k8sadmissionv1beta1.OperationType{"*"} + defaultMatchResourcesScope = k8sadmissionv1beta1.ScopeType("*") + defaultMatchPolicyType = k8sadmissionv1beta1.Equivalent + HCCOUserValidation = k8sadmissionv1beta1.Validation{ + Expression: fmt.Sprintf("request.userInfo.username == 'system:%s' || (has(object.spec) && has(oldObject.spec) && object.spec == oldObject.spec)", config.HCCOUser), + Message: "This resource cannot be created, updated, or deleted. Please ask your administrator to modify the resource in the HostedCluster object.", + Reason: ptr.To(metav1.StatusReasonInvalid), + } +) + +// ReconcileKASValidatingAdmissionPolicies will create ValidatingAdmissionPolicies which block certain resources +// from being updated/deleted from the DataPlane side. +func ReconcileKASValidatingAdmissionPolicies(ctx context.Context, hcp *hyperv1.HostedControlPlane, client client.Client, createOrUpdate upsert.CreateOrUpdateFN) error { + log := ctrl.LoggerFrom(ctx) + log.Info("reconciling validating admission policies") + + if err := reconcileConfigValidatingAdmissionPolicy(ctx, hcp, client, createOrUpdate); err != nil { + return fmt.Errorf("failed to reconcile Config Validating Admission Policy: %v", err) + } + + if err := reconcileMirrorValidatingAdmissionPolicy(ctx, hcp, client, createOrUpdate); err != nil { + return fmt.Errorf("failed to reconcile Mirror Validating Admission Policies: %v", err) + } + + return nil +} + +func reconcileConfigValidatingAdmissionPolicy(ctx context.Context, hcp *hyperv1.HostedControlPlane, client client.Client, createOrUpdate upsert.CreateOrUpdateFN) error { + // Config AdmissionPolicy + configAdmissionPolicy := AdmissionPolicy{Name: AdmissionPolicyNameConfig} + configAPIVersion := []string{configv1.GroupVersion.Version} + configAPIGroup := []string{configv1.GroupVersion.Group} + configResources := []string{ + "apiservers", + "authentications", + "featuregates", + "images", + "imagecontentpolicies", + "infrastructures", + "ingresses", + "proxies", + "schedulers", + "networks", + "oauths", + } + + if hcp.Spec.OLMCatalogPlacement == hyperv1.ManagementOLMCatalogPlacement { + configResources = append(configResources, "operatorhubs") + } + configAdmissionPolicy.Validations = []k8sadmissionv1beta1.Validation{HCCOUserValidation} + configAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(configResources, configAPIVersion, configAPIGroup, []k8sadmissionv1beta1.OperationType{"UPDATE", "DELETE"}) + if err := configAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil { + return fmt.Errorf("error reconciling Config Validating Admission Policy: %v", err) + } + + return nil +} + +func reconcileMirrorValidatingAdmissionPolicy(ctx context.Context, hcp *hyperv1.HostedControlPlane, client client.Client, createOrUpdate upsert.CreateOrUpdateFN) error { + // Mirroring AdmissionPolicies + mirrorAdmissionPolicy := AdmissionPolicy{Name: AdmissionPolicyNameMirror} + mirrorAPIVersion := []string{configv1.GroupVersion.Version} + mirrorAPIGroup := []string{configv1.GroupVersion.Group} + mirrorResources := []string{ + "imagedigestmirrorsets", + "imagetagmirrorsets", + } + + if hcp.Spec.OLMCatalogPlacement == hyperv1.ManagementOLMCatalogPlacement { + mirrorResources = append(mirrorResources, "operatorhubs") + } + mirrorAdmissionPolicy.Validations = []k8sadmissionv1beta1.Validation{HCCOUserValidation} + mirrorAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(mirrorResources, mirrorAPIVersion, mirrorAPIGroup, allAdmissionPoliciesOperations) + if err := mirrorAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil { + return fmt.Errorf("error reconciling Mirror Validating Admission Policy: %v", err) + } + + // ICSP lives in other API, this is why we need to create another vap and vap-binding + icspAdmissionPolicy := AdmissionPolicy{Name: AdmissionPolicyNameICSP} + icspAPIVersion := []string{operatorv1alpha1.GroupVersion.Version} + icspAPIGroup := []string{operatorv1alpha1.GroupVersion.Group} + icspResources := []string{"imagecontentsourcepolicies"} + + icspAdmissionPolicy.Validations = []k8sadmissionv1beta1.Validation{HCCOUserValidation} + icspAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(icspResources, icspAPIVersion, icspAPIGroup, allAdmissionPoliciesOperations) + if err := icspAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil { + return fmt.Errorf("error reconciling ICSP Validating Admission Policy: %v", err) + } + + return nil +} + +func (ap *AdmissionPolicy) reconcileAdmissionPolicy(ctx context.Context, client client.Client, createOrUpdate upsert.CreateOrUpdateFN) error { + vap := manifests.ValidatingAdmissionPolicy(ap.Name) + if _, err := createOrUpdate(ctx, client, vap, func() error { + if vap.Spec.MatchConstraints != nil { + vap.Spec.MatchConstraints.ResourceRules = ap.MatchConstraints.ResourceRules + vap.Spec.MatchConstraints.MatchPolicy = ap.MatchConstraints.MatchPolicy + } else { + vap.Spec.MatchConstraints = ap.MatchConstraints + } + vap.Spec.Validations = ap.Validations + + return nil + }); err != nil { + return fmt.Errorf("failed to create/update Validating Admission Policy with name %s: %v", ap.Name, err) + } + + policyBinding := manifests.ValidatingAdmissionPolicyBinding(fmt.Sprintf("%s-binding", ap.Name)) + if _, err := createOrUpdate(ctx, client, policyBinding, func() error { + policyBinding.Spec.PolicyName = ap.Name + policyBinding.Spec.ValidationActions = []k8sadmissionv1beta1.ValidationAction{k8sadmissionv1beta1.Deny} + return nil + }); err != nil { + return fmt.Errorf("failed to create/update Validating Admission Policy Binding with name %s: %v", ap.Name, err) + } + + return nil +} + +func constructPolicyMatchConstraints(resources, apiVersion, apiGroup []string, operations []k8sadmissionv1beta1.OperationType) *k8sadmissionv1beta1.MatchResources { + return &k8sadmissionv1beta1.MatchResources{ + ResourceRules: []k8sadmissionv1beta1.NamedRuleWithOperations{ + { + RuleWithOperations: k8sadmissionv1beta1.RuleWithOperations{ + Operations: operations, + Rule: k8sadmissionv1beta1.Rule{ + APIGroups: apiGroup, + APIVersions: apiVersion, + Resources: resources, + Scope: &defaultMatchResourcesScope, + }, + }, + }, + }, + MatchPolicy: &defaultMatchPolicyType, + } +} diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests/admissionpolicies.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests/admissionpolicies.go new file mode 100644 index 0000000000..9ee41dcf19 --- /dev/null +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests/admissionpolicies.go @@ -0,0 +1,22 @@ +package manifests + +import ( + k8sadmissionv1beta1 "k8s.io/api/admissionregistration/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func ValidatingAdmissionPolicy(name string) *k8sadmissionv1beta1.ValidatingAdmissionPolicy { + return &k8sadmissionv1beta1.ValidatingAdmissionPolicy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: k8sadmissionv1beta1.SchemeGroupVersion.String(), + Kind: "ValidatingAdmissionPolicy", + }, + ObjectMeta: metav1.ObjectMeta{Name: name}, + } +} + +func ValidatingAdmissionPolicyBinding(bindingName string) *k8sadmissionv1beta1.ValidatingAdmissionPolicyBinding { + return &k8sadmissionv1beta1.ValidatingAdmissionPolicyBinding{ + ObjectMeta: metav1.ObjectMeta{Name: bindingName}, + } +} diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go index fe0f9d08dd..6c79ef1758 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go @@ -43,6 +43,7 @@ import ( openshiftcpv1 "github.com/openshift/api/openshiftcontrolplane/v1" operatorv1 "github.com/openshift/api/operator/v1" + "github.com/blang/semver" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/cloud/azure" @@ -279,6 +280,20 @@ func (r *reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result errs = append(errs, fmt.Errorf("failed to reconcile kubernetes.default endpoints and endpointslice: %w", err)) } + releaseImageVersion, err := semver.Parse(releaseImage.Version()) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to parse release image version: %w", err) + } + + // The exception for IBMCloudPlatform is due to the fact that the IBM will include new certificates for HCCO from 4.17 version + if !(hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform && (releaseImageVersion.Major == 4 && releaseImageVersion.Minor < 17)) { + // Apply new ValidatingAdmissionPolicy to restrict the modification/deletion of certain + // objects from the DataPlane which are managed by the HCCO. + if err := kas.ReconcileKASValidatingAdmissionPolicies(ctx, hcp, r.client, r.CreateOrUpdate); err != nil { + errs = append(errs, fmt.Errorf("failed to reconcile validating admission policies: %w", err)) + } + } + log.Info("reconciling install configmap") if err := r.reconcileInstallConfigMap(ctx, releaseImage); err != nil { errs = append(errs, fmt.Errorf("failed to reconcile install configmap: %w", err)) diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go index 55f60be5af..8ffda46458 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go @@ -21,6 +21,7 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" cpomanifests "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/api" + "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas" "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests" "github.com/openshift/hypershift/support/globalconfig" fakereleaseprovider "github.com/openshift/hypershift/support/releaseinfo/fake" @@ -61,6 +62,13 @@ var initialObjects = []client.Object{ manifests.NodeTuningClusterOperator(), manifests.NamespaceKubeSystem(), &configv1.ClusterVersion{ObjectMeta: metav1.ObjectMeta{Name: "version"}}, + manifests.ValidatingAdmissionPolicy(kas.AdmissionPolicyNameConfig), + manifests.ValidatingAdmissionPolicy(kas.AdmissionPolicyNameMirror), + manifests.ValidatingAdmissionPolicy(kas.AdmissionPolicyNameICSP), + manifests.ValidatingAdmissionPolicyBinding(fmt.Sprintf("%s-binding", kas.AdmissionPolicyNameConfig)), + manifests.ValidatingAdmissionPolicyBinding(fmt.Sprintf("%s-binding", kas.AdmissionPolicyNameMirror)), + manifests.ValidatingAdmissionPolicyBinding(fmt.Sprintf("%s-binding", kas.AdmissionPolicyNameICSP)), + fakeOperatorHub(), } diff --git a/control-plane-operator/hostedclusterconfigoperator/operator/config.go b/control-plane-operator/hostedclusterconfigoperator/operator/config.go index 25c402d137..3796eeaebf 100644 --- a/control-plane-operator/hostedclusterconfigoperator/operator/config.go +++ b/control-plane-operator/hostedclusterconfigoperator/operator/config.go @@ -25,6 +25,7 @@ import ( operatorv1 "github.com/openshift/api/operator/v1" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/api" + "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/labelenforcingclient" "github.com/openshift/hypershift/support/releaseinfo" "github.com/openshift/hypershift/support/upsert" @@ -75,7 +76,7 @@ type HostedClusterConfigOperatorConfig struct { } func Mgr(cfg, cpConfig *rest.Config, namespace string) ctrl.Manager { - cfg.UserAgent = "hosted-cluster-config-operator-manager" + cfg.UserAgent = config.HCCOUserAgent allSelector := cache.ByObject{ Label: labels.Everything(), } diff --git a/go.mod b/go.mod index 567a733a9f..e3e78b0176 100644 --- a/go.mod +++ b/go.mod @@ -254,9 +254,9 @@ replace github.com/google/cel-go => github.com/google/cel-go v0.17.7 // These are because of github.com/openshift/cluster-node-tuning-operator@v0.0.0-20240131125539-0e319439e65a replace ( - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.3 - k8s.io/kubernetes => k8s.io/kubernetes v0.29.3 - k8s.io/mount-utils => k8s.io/mount-utils v0.29.3 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.30.2 + k8s.io/kubernetes => k8s.io/kubernetes v0.30.2 + k8s.io/mount-utils => k8s.io/mount-utils v0.30.2 ) // There is an error with newer versions of prometheus diff --git a/go.work.sum b/go.work.sum index 452a335aa9..47b94c892b 100644 --- a/go.work.sum +++ b/go.work.sum @@ -698,6 +698,7 @@ k8s.io/component-helpers v0.30.1/go.mod h1:b1Xk27UJ3p/AmPqDx7khrnSxrdwQy9gTP7O1y k8s.io/controller-manager v0.27.4/go.mod h1:5+Fo0k+t3MDyuNLjmXzU/dJcD2c34ii8Wef+OmqhkVg= k8s.io/cri-api v0.28.4/go.mod h1:QaLIWi4Ejw0uHZlGRUIDmc2IlNlwc9Wp4gb6tEjeQCs= k8s.io/csi-translation-lib v0.29.3/go.mod h1:snAzieA58/oiQXQZr27b0+b6/3+ZzitwI+57cUsMKKQ= +k8s.io/csi-translation-lib v0.30.2/go.mod h1:jFT8vquP6eSDUwDHk0mKT6uKFWlZp60ecUEUhmlGsOY= k8s.io/dynamic-resource-allocation v0.26.2/go.mod h1:LvsFr+Mp6LvEYypFPkB/xq0u+W368qVldQckybgQTgM= k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 h1:pWEwq4Asjm4vjW7vcsmijwBhOr1/shsbSYiWXmNGlks= k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -708,7 +709,7 @@ k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/metrics v0.30.1/go.mod h1:gVAhTTgfNKsn9D1kB7Nmb1T31relBuXzzGUE7klyOkM= -k8s.io/mount-utils v0.29.3/go.mod h1:9IWJTMe8tG0MYMLEp60xK9GYVeCdA3g4LowmnVi+t9Y= +k8s.io/mount-utils v0.30.2/go.mod h1:9sCVmwGLcV1MPvbZ+rToMDnl1QcGozy+jBPd0MsQLIo= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/support/config/types.go b/support/config/types.go index 37fa7f7d3a..6e7f69aacc 100644 --- a/support/config/types.go +++ b/support/config/types.go @@ -4,4 +4,9 @@ const ( // PodSafeToEvictLocalVolumesKey is an annotation used by the CA operator which makes sure // all the pods annotated with it and the picking the desired local volumes that are safe to evict, could be drained properly. PodSafeToEvictLocalVolumesKey = "cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes" + + // HCCOUser references the user used by the HostedClusterConfigOperator + HCCOUser = "hosted-cluster-config" + // HCCOUserAgent references the userAgent used by the HostedClusterConfigOperator + HCCOUserAgent = "hosted-cluster-config-operator-manager" ) diff --git a/test/e2e/util/hypershift_framework.go b/test/e2e/util/hypershift_framework.go index 96de106a0d..4b700cbfd6 100644 --- a/test/e2e/util/hypershift_framework.go +++ b/test/e2e/util/hypershift_framework.go @@ -147,6 +147,7 @@ func (h *hypershiftTest) after(hostedCluster *hyperv1.HostedCluster, platform hy EnsureHCPPodsAffinitiesAndTolerations(t, context.Background(), h.client, hostedCluster) } EnsureSATokenNotMountedUnlessNecessary(t, context.Background(), h.client, hostedCluster) + EnsureAdmissionPolicies(t, context.Background(), h.client, hostedCluster) }) } diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index 7496a7abb3..287ad2cb11 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -17,6 +17,7 @@ import ( routev1 "github.com/openshift/api/route/v1" routev1client "github.com/openshift/client-go/route/clientset/versioned" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + hccokasvap "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas" hcmetrics "github.com/openshift/hypershift/hypershift-operator/controllers/hostedcluster/metrics" "github.com/openshift/hypershift/hypershift-operator/controllers/manifests" "github.com/openshift/hypershift/support/conditions" @@ -27,6 +28,7 @@ import ( "github.com/prometheus/common/model" "go.uber.org/zap/zaptest" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + k8sadmissionv1beta1 "k8s.io/api/admissionregistration/v1beta1" appsv1 "k8s.io/api/apps/v1" authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" @@ -1205,6 +1207,107 @@ func EnsureGuestWebhooksValidated(t *testing.T, ctx context.Context, guestClient }) } +func EnsureAdmissionPolicies(t *testing.T, ctx context.Context, mgmtClient crclient.Client, hc *hyperv1.HostedCluster) { + if !util.IsPublicHC(hc) { + return // Admission policies are only validated in public clusters does not worth to test it in private ones. + } + guestClient := WaitForGuestClient(t, ctx, mgmtClient, hc) + t.Run("EnsureValidatingAdmissionPoliciesExists", func(t *testing.T) { + t.Log("Waiting for ValidatingAdmissionPolicies to exist") + var expectedVAPCount int = 3 + g := NewWithT(t) + start := time.Now() + var validatingAdmissionPolicies k8sadmissionv1beta1.ValidatingAdmissionPolicyList + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { + + if err := guestClient.List(ctx, &validatingAdmissionPolicies); err != nil { + t.Logf("Failed to list ValidatingAdmissionPolicies: %v", err) + return false, nil + } + + if len(validatingAdmissionPolicies.Items) == 0 { + t.Log("No ValidatingAdmissionPolicies found") + return false, nil + } + + if len(validatingAdmissionPolicies.Items) == expectedVAPCount { + t.Logf("Found at least %d ValidatingAdmissionPolicies", expectedVAPCount) + return true, nil + } + + return true, nil + }) + duration := time.Since(start).Round(time.Second) + g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to wait for ValidatingAdmissionPolicies to exist in %s: %v", duration, err)) + + for _, vap := range validatingAdmissionPolicies.Items { + switch vap.Name { + case hccokasvap.AdmissionPolicyNameConfig: + g.Expect(vap.Name).To(Equal(hccokasvap.AdmissionPolicyNameConfig), fmt.Sprintf("ValidatingAdmissionPolicy %s not found in the list", hccokasvap.AdmissionPolicyNameConfig)) + case hccokasvap.AdmissionPolicyNameMirror: + g.Expect(vap.Name).To(Equal(hccokasvap.AdmissionPolicyNameMirror), fmt.Sprintf("ValidatingAdmissionPolicy %s not found in the list", hccokasvap.AdmissionPolicyNameMirror)) + case hccokasvap.AdmissionPolicyNameICSP: + g.Expect(vap.Name).To(Equal(hccokasvap.AdmissionPolicyNameICSP), fmt.Sprintf("ValidatingAdmissionPolicy %s not found in the list", hccokasvap.AdmissionPolicyNameICSP)) + default: + t.Errorf("Unexpected ValidatingAdmissionPolicy %s found in the list", vap.Name) + } + } + g.Expect(expectedVAPCount).To(Equal(len(validatingAdmissionPolicies.Items)), fmt.Sprintf("Failed checking ValidatingAdmissionPolicies, there are %d VAP deployed and %d is expected", len(validatingAdmissionPolicies.Items), expectedVAPCount)) + t.Logf("Successfully waited for ValidatingAdmissionPolicies to exist in %s", duration) + }) + t.Run("EnsureValidatingAdmissionPoliciesCheckDeniedRequests", func(t *testing.T) { + g := NewWithT(t) + t.Log("Checking Denied KAS Requests for ValidatingAdmissionPolicies") + apiServer := &configv1.APIServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + } + err := guestClient.Get(ctx, client.ObjectKeyFromObject(apiServer), apiServer) + g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to wait grabbing HostedCluster apiserver configuration: %v", err)) + g.Expect(apiServer).NotTo(BeNil(), "Apiserver configuration is nil") + apiServerCP := apiServer.DeepCopy() + apiServerCP.Spec.Audit.Profile = configv1.AllRequestBodiesAuditProfileType + err = guestClient.Update(ctx, apiServerCP) + g.Expect(err).To(HaveOccurred(), fmt.Sprintf("Failed block apiservers configuration update: %v", err)) + + }) + t.Run("EnsureValidatingAdmissionPoliciesDontBlockStatusModifications", func(t *testing.T) { + g := NewWithT(t) + t.Log("Checking ClusterOperator status modifications are allowed") + network := &configv1.Network{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + } + err := guestClient.Get(ctx, client.ObjectKeyFromObject(network), network) + g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to wait grabbing HostedCluster network configuration: %v", err)) + g.Expect(network).NotTo(BeNil(), "network configuration is nil") + cpNetwork := network.DeepCopy() + cpNetwork.Status.ClusterNetworkMTU = 9180 + err = guestClient.Update(ctx, cpNetwork) + g.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed updating network status ClusterNetworkMTU field: %v", err)) + }) + if hc.Spec.OLMCatalogPlacement == hyperv1.GuestOLMCatalogPlacement { + t.Run("EnsureValidatingAdmissionPoliciesCheckAllowedRequest", func(t *testing.T) { + g := NewWithT(t) + t.Log("Checking Allowed KAS Requests for ValidatingAdmissionPolicies") + operatorHub := &configv1.OperatorHub{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + } + err := guestClient.Get(ctx, client.ObjectKeyFromObject(operatorHub), operatorHub) + g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to wait grabbing HostedCluster network configuration: %v", err)) + g.Expect(operatorHub).NotTo(BeNil(), "OperatorHub configuration is nil") + operatorHubCP := operatorHub.DeepCopy() + operatorHubCP.Spec.DisableAllDefaultSources = true + err = guestClient.Update(ctx, operatorHubCP) + g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to update OperatorHub configuration: %v", err)) + }) + } +} + const ( // Metrics // TODO (jparrill): We need to separate the metrics.go from the main pkg in the hypershift-operator. diff --git a/vendor/modules.txt b/vendor/modules.txt index cef7fec59e..a2bae7d52f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2367,8 +2367,8 @@ sigs.k8s.io/yaml sigs.k8s.io/yaml/goyaml.v2 sigs.k8s.io/yaml/goyaml.v3 # github.com/google/cel-go => github.com/google/cel-go v0.17.7 -# k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.3 -# k8s.io/kubernetes => k8s.io/kubernetes v0.29.3 -# k8s.io/mount-utils => k8s.io/mount-utils v0.29.3 +# k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.30.2 +# k8s.io/kubernetes => k8s.io/kubernetes v0.30.2 +# k8s.io/mount-utils => k8s.io/mount-utils v0.30.2 # github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.18.0 # github.com/prometheus/common => github.com/prometheus/common v0.45.0