Skip to content

Commit

Permalink
Merge pull request #4303 from jparrill/OCPBUGS-34816
Browse files Browse the repository at this point in the history
OCPBUGS-34816: Block data plane HC configuration requests
  • Loading branch information
openshift-merge-bot[bot] authored Aug 20, 2024
2 parents 4531612 + ea1be5d commit b8b5251
Show file tree
Hide file tree
Showing 23 changed files with 422 additions and 14 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ build: hypershift-operator control-plane-operator control-plane-pki-operator hyp

.PHONY: sync
sync:
$(GO) work sync
$(GO) work sync

.PHONY: update
update: sync api-deps api api-docs deps clients
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,7 @@ func ReconcileRole(role *rbacv1.Role, ownerRef config.OwnerRef, platform hyperv1
},
}...)
}
// TODO (jparrill): Add RBAC specific needs for Agent platform
return nil
}

Expand Down Expand Up @@ -443,7 +444,7 @@ func buildHCCContainerMain(image, hcpName, openShiftVersion, kubeVersion string,

func buildHCCVolumeKubeconfig(v *corev1.Volume) {
v.Secret = &corev1.SecretVolumeSource{
SecretName: manifests.KASServiceKubeconfigSecret("").Name,
SecretName: manifests.HCCOKubeconfigSecret("").Name,
DefaultMode: pointer.Int32(0640),
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2819,6 +2819,10 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex
if err := r.Get(ctx, client.ObjectKeyFromObject(bootstrapClientCertSecret), bootstrapClientCertSecret); err != nil {
return fmt.Errorf("failed to get bootstrap client cert secret: %w", err)
}
hccoClientCertSecret := manifests.HCCOClientCertSecret(hcp.Namespace)
if err := r.Get(ctx, client.ObjectKeyFromObject(hccoClientCertSecret), hccoClientCertSecret); err != nil {
return fmt.Errorf("failed to get HCCO client cert secret: %w", err)
}

serviceKubeconfigSecret := manifests.KASServiceKubeconfigSecret(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, serviceKubeconfigSecret, func() error {
Expand All @@ -2838,6 +2842,13 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex
return fmt.Errorf("failed to reconcile CAPI service admin kubeconfig secret: %w", err)
}

hccoKubeconfigSecret := manifests.HCCOKubeconfigSecret(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, hccoKubeconfigSecret, func() error {
return kas.ReconcileHCCOKubeconfigSecret(hccoKubeconfigSecret, hccoClientCertSecret, rootCA, p.OwnerRef, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile HCCO kubeconfig secret: %w", err)
}

localhostKubeconfigSecret := manifests.KASLocalhostKubeconfigSecret(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, localhostKubeconfigSecret, func() error {
return kas.ReconcileLocalhostKubeconfigSecret(localhostKubeconfigSecret, clientCertSecret, rootCA, p.OwnerRef, p.KASPodPort)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ func generateConfig(p KubeAPIServerConfigParams) *kcpv1.KubeAPIServerConfig {
// TODO remove in 4.16 once we're able to have different featuregates for hypershift
featureGates := append([]string{}, p.FeatureGates...)
featureGates = append(featureGates, "StructuredAuthenticationConfiguration=true")
featureGates = append(featureGates, "ValidatingAdmissionPolicy=true")
args.Set("feature-gates", featureGates...)
args.Set("goaway-chance", "0")
args.Set("http2-max-streams-per-connection", "2000")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -756,6 +756,22 @@ cat <<EOF >/tmp/manifests/99_feature-gate.yaml
%[3]s
EOF
touch /tmp/manifests/hcco-rolebinding.yaml
cat <<EOF >/tmp/manifests/hcco-rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: hcco-cluster-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:hosted-cluster-config
EOF
/usr/bin/render \
--asset-output-dir /tmp/output \
--rendered-manifest-dir=/tmp/manifests \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,8 @@ func ReconcileExternalKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.C
func ReconcileBootstrapKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.ConfigMap, ownerRef config.OwnerRef, externalURL string) error {
return pki.ReconcileKubeConfig(secret, cert, ca, externalURL, "", manifests.KubeconfigScopeBootstrap, ownerRef)
}

func ReconcileHCCOKubeconfigSecret(secret, cert *corev1.Secret, ca *corev1.ConfigMap, ownerRef config.OwnerRef, platformType hyperv1.PlatformType) error {
svcURL := InClusterKASURL(platformType)
return pki.ReconcileKubeConfig(secret, cert, ca, svcURL, "", "service", ownerRef)
}
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,29 @@ func (r *HostedControlPlaneReconciler) setupKASClientSigners(
return err
}

// ----------
// HCCO signer
// ----------

hccoKubeconfigSigner, err := reconcileSigner(
manifests.HCCOSigner(hcp.Namespace),
pki.ReconcileHCCOSigner,
)

if err != nil {
return err
}
totalClientCABundle = append(totalClientCABundle, hccoKubeconfigSigner)

// system:hosted-cluster-config client cert
if _, err := reconcileSub(
manifests.HCCOClientCertSecret(hcp.Namespace),
hccoKubeconfigSigner,
pki.ReconcileHCCOClientCertSecret,
); err != nil {
return err
}

// ----------
// CSR signer
// ----------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,28 @@ func ConfigOperatorDeployment(ns string) *appsv1.Deployment {

func ConfigOperatorRole(ns string) *rbacv1.Role {
r := &rbacv1.Role{}
r.Name = "hosted-cluster-config-operator"
r.Name = "hosted-cluster-config"
r.Namespace = ns
return r
}

func ConfigOperatorRoleBinding(ns string) *rbacv1.RoleBinding {
rb := &rbacv1.RoleBinding{}
rb.Name = "hosted-cluster-config-operator"
rb.Name = "hosted-cluster-config"
rb.Namespace = ns
return rb
}

func ConfigOperatorServiceAccount(ns string) *corev1.ServiceAccount {
sa := &corev1.ServiceAccount{}
sa.Name = "hosted-cluster-config-operator"
sa.Name = "hosted-cluster-config"
sa.Namespace = ns
return sa
}

func ConfigOperatorPodMonitor(ns string) *prometheusoperatorv1.PodMonitor {
return &prometheusoperatorv1.PodMonitor{ObjectMeta: metav1.ObjectMeta{
Namespace: ns,
Name: "hosted-cluster-config-operator",
Name: "hosted-cluster-config",
}}
}
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,15 @@ func KASServiceCAPIKubeconfigSecret(controlPlaneNamespace, infraID string) *core
}
}

func HCCOKubeconfigSecret(controlPlaneNamespace string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "hcco-kubeconfig",
Namespace: controlPlaneNamespace,
},
}
}

func KASExternalKubeconfigSecret(controlPlaneNamespace string, ref *hyperv1.KubeconfigSecretRef) *corev1.Secret {
s := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,14 @@ func SystemAdminClientCertSecret(ns string) *corev1.Secret {
return secretFor(ns, "system-admin-client")
}

func HCCOSigner(ns string) *corev1.Secret {
return secretFor(ns, "hcco-signer")
}

func HCCOClientCertSecret(ns string) *corev1.Secret {
return secretFor(ns, "hcco-client")
}

func KASMachineBootstrapClientCertSecret(ns string) *corev1.Secret {
return secretFor(ns, "kas-bootstrap-client")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ func ReconcileAdminKubeconfigSigner(secret *corev1.Secret, ownerRef config.Owner
return reconcileSelfSignedCA(secret, ownerRef, "admin-kubeconfig-signer", "openshift")
}

func ReconcileHCCOSigner(secret *corev1.Secret, ownerRef config.OwnerRef) error {
return reconcileSelfSignedCA(secret, ownerRef, "hcco-signer", "openshift")
}

func ReconcileKubeCSRSigner(secret *corev1.Secret, ownerRef config.OwnerRef) error {
return reconcileSelfSignedCA(secret, ownerRef, "kube-csr-signer", "openshift")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,10 @@ func ReconcileSystemAdminClientCertSecret(secret, ca *corev1.Secret, ownerRef co
return reconcileSignedCert(secret, ca, ownerRef, "system:admin", []string{"system:masters"}, X509UsageClientAuth)
}

func ReconcileHCCOClientCertSecret(secret, ca *corev1.Secret, ownerRef config.OwnerRef) error {
return reconcileSignedCert(secret, ca, ownerRef, fmt.Sprintf("system:%s", config.HCCOUser), []string{"kubernetes"}, X509UsageClientAuth)
}

func ReconcileServiceAccountKubeconfig(secret, csrSigner *corev1.Secret, ca *corev1.ConfigMap, hcp *hyperv1.HostedControlPlane, serviceAccountNamespace, serviceAccountName string) error {
cn := serviceaccount.MakeUsername(serviceAccountNamespace, serviceAccountName)
if err := reconcileSignedCert(secret, csrSigner, config.OwnerRef{}, cn, serviceaccount.MakeGroupNames(serviceAccountNamespace), X509UsageClientAuth); err != nil {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
package kas

import (
"context"
"fmt"

configv1 "github.com/openshift/api/config/v1"
operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1"
hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests"
"github.com/openshift/hypershift/support/config"
"github.com/openshift/hypershift/support/upsert"
k8sadmissionv1beta1 "k8s.io/api/admissionregistration/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type AdmissionPolicy struct {
Name string
MatchConstraints *k8sadmissionv1beta1.MatchResources
Validations []k8sadmissionv1beta1.Validation
}

const (
AdmissionPolicyNameConfig = "config"
AdmissionPolicyNameMirror = "mirror"
AdmissionPolicyNameICSP = "icsp"
)

var (
allAdmissionPoliciesOperations = []k8sadmissionv1beta1.OperationType{"*"}
defaultMatchResourcesScope = k8sadmissionv1beta1.ScopeType("*")
defaultMatchPolicyType = k8sadmissionv1beta1.Equivalent
HCCOUserValidation = k8sadmissionv1beta1.Validation{
Expression: fmt.Sprintf("request.userInfo.username == 'system:%s' || (has(object.spec) && has(oldObject.spec) && object.spec == oldObject.spec)", config.HCCOUser),
Message: "This resource cannot be created, updated, or deleted. Please ask your administrator to modify the resource in the HostedCluster object.",
Reason: ptr.To(metav1.StatusReasonInvalid),
}
)

// ReconcileKASValidatingAdmissionPolicies will create ValidatingAdmissionPolicies which block certain resources
// from being updated/deleted from the DataPlane side.
func ReconcileKASValidatingAdmissionPolicies(ctx context.Context, hcp *hyperv1.HostedControlPlane, client client.Client, createOrUpdate upsert.CreateOrUpdateFN) error {
log := ctrl.LoggerFrom(ctx)
log.Info("reconciling validating admission policies")

if err := reconcileConfigValidatingAdmissionPolicy(ctx, hcp, client, createOrUpdate); err != nil {
return fmt.Errorf("failed to reconcile Config Validating Admission Policy: %v", err)
}

if err := reconcileMirrorValidatingAdmissionPolicy(ctx, hcp, client, createOrUpdate); err != nil {
return fmt.Errorf("failed to reconcile Mirror Validating Admission Policies: %v", err)
}

return nil
}

func reconcileConfigValidatingAdmissionPolicy(ctx context.Context, hcp *hyperv1.HostedControlPlane, client client.Client, createOrUpdate upsert.CreateOrUpdateFN) error {
// Config AdmissionPolicy
configAdmissionPolicy := AdmissionPolicy{Name: AdmissionPolicyNameConfig}
configAPIVersion := []string{configv1.GroupVersion.Version}
configAPIGroup := []string{configv1.GroupVersion.Group}
configResources := []string{
"apiservers",
"authentications",
"featuregates",
"images",
"imagecontentpolicies",
"infrastructures",
"ingresses",
"proxies",
"schedulers",
"networks",
"oauths",
}

if hcp.Spec.OLMCatalogPlacement == hyperv1.ManagementOLMCatalogPlacement {
configResources = append(configResources, "operatorhubs")
}
configAdmissionPolicy.Validations = []k8sadmissionv1beta1.Validation{HCCOUserValidation}
configAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(configResources, configAPIVersion, configAPIGroup, []k8sadmissionv1beta1.OperationType{"UPDATE", "DELETE"})
if err := configAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil {
return fmt.Errorf("error reconciling Config Validating Admission Policy: %v", err)
}

return nil
}

func reconcileMirrorValidatingAdmissionPolicy(ctx context.Context, hcp *hyperv1.HostedControlPlane, client client.Client, createOrUpdate upsert.CreateOrUpdateFN) error {
// Mirroring AdmissionPolicies
mirrorAdmissionPolicy := AdmissionPolicy{Name: AdmissionPolicyNameMirror}
mirrorAPIVersion := []string{configv1.GroupVersion.Version}
mirrorAPIGroup := []string{configv1.GroupVersion.Group}
mirrorResources := []string{
"imagedigestmirrorsets",
"imagetagmirrorsets",
}

if hcp.Spec.OLMCatalogPlacement == hyperv1.ManagementOLMCatalogPlacement {
mirrorResources = append(mirrorResources, "operatorhubs")
}
mirrorAdmissionPolicy.Validations = []k8sadmissionv1beta1.Validation{HCCOUserValidation}
mirrorAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(mirrorResources, mirrorAPIVersion, mirrorAPIGroup, allAdmissionPoliciesOperations)
if err := mirrorAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil {
return fmt.Errorf("error reconciling Mirror Validating Admission Policy: %v", err)
}

// ICSP lives in other API, this is why we need to create another vap and vap-binding
icspAdmissionPolicy := AdmissionPolicy{Name: AdmissionPolicyNameICSP}
icspAPIVersion := []string{operatorv1alpha1.GroupVersion.Version}
icspAPIGroup := []string{operatorv1alpha1.GroupVersion.Group}
icspResources := []string{"imagecontentsourcepolicies"}

icspAdmissionPolicy.Validations = []k8sadmissionv1beta1.Validation{HCCOUserValidation}
icspAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(icspResources, icspAPIVersion, icspAPIGroup, allAdmissionPoliciesOperations)
if err := icspAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil {
return fmt.Errorf("error reconciling ICSP Validating Admission Policy: %v", err)
}

return nil
}

func (ap *AdmissionPolicy) reconcileAdmissionPolicy(ctx context.Context, client client.Client, createOrUpdate upsert.CreateOrUpdateFN) error {
vap := manifests.ValidatingAdmissionPolicy(ap.Name)
if _, err := createOrUpdate(ctx, client, vap, func() error {
if vap.Spec.MatchConstraints != nil {
vap.Spec.MatchConstraints.ResourceRules = ap.MatchConstraints.ResourceRules
vap.Spec.MatchConstraints.MatchPolicy = ap.MatchConstraints.MatchPolicy
} else {
vap.Spec.MatchConstraints = ap.MatchConstraints
}
vap.Spec.Validations = ap.Validations

return nil
}); err != nil {
return fmt.Errorf("failed to create/update Validating Admission Policy with name %s: %v", ap.Name, err)
}

policyBinding := manifests.ValidatingAdmissionPolicyBinding(fmt.Sprintf("%s-binding", ap.Name))
if _, err := createOrUpdate(ctx, client, policyBinding, func() error {
policyBinding.Spec.PolicyName = ap.Name
policyBinding.Spec.ValidationActions = []k8sadmissionv1beta1.ValidationAction{k8sadmissionv1beta1.Deny}
return nil
}); err != nil {
return fmt.Errorf("failed to create/update Validating Admission Policy Binding with name %s: %v", ap.Name, err)
}

return nil
}

func constructPolicyMatchConstraints(resources, apiVersion, apiGroup []string, operations []k8sadmissionv1beta1.OperationType) *k8sadmissionv1beta1.MatchResources {
return &k8sadmissionv1beta1.MatchResources{
ResourceRules: []k8sadmissionv1beta1.NamedRuleWithOperations{
{
RuleWithOperations: k8sadmissionv1beta1.RuleWithOperations{
Operations: operations,
Rule: k8sadmissionv1beta1.Rule{
APIGroups: apiGroup,
APIVersions: apiVersion,
Resources: resources,
Scope: &defaultMatchResourcesScope,
},
},
},
},
MatchPolicy: &defaultMatchPolicyType,
}
}
Loading

0 comments on commit b8b5251

Please sign in to comment.