diff --git a/Dockerfile b/Dockerfile index cfb0b507d8..f307d21594 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,12 +4,13 @@ WORKDIR /hypershift COPY . . -RUN make hypershift hypershift-operator product-cli +RUN make hypershift hypershift-operator product-cli karpenter-operator FROM registry.access.redhat.com/ubi9:latest COPY --from=builder /hypershift/bin/hypershift \ /hypershift/bin/hcp \ /hypershift/bin/hypershift-operator \ + /hypershift/bin/karpenter-operator \ /usr/bin/ ENTRYPOINT ["/usr/bin/hypershift"] diff --git a/Makefile b/Makefile index 0bf255ec11..650a7f8fb5 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ OUT_DIR ?= bin # run the HO locally HYPERSHIFT_INSTALL_AWS := ./hack/dev/aws/hypershft-install-aws.sh -RUN_OPERATOR_LOCALLY_AWS := ./hack/dev/aws/run-operator-locally-aws.sh +RUN_OPERATOR_LOCALLY_AWS := ./hack/dev/aws/run-operator-locally-aws-dev.sh # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -89,6 +89,10 @@ tests: hypershift-operator: $(GO_BUILD_RECIPE) -o $(OUT_DIR)/hypershift-operator ./hypershift-operator +.PHONY: karpenter-operator +karpenter-operator: + $(GO_BUILD_RECIPE) -o $(OUT_DIR)/karpenter-operator ./karpenter-operator + .PHONY: control-plane-operator control-plane-operator: $(GO_BUILD_RECIPE) -o $(OUT_DIR)/control-plane-operator ./control-plane-operator diff --git a/api/hypershift/v1beta1/featuregates/featureGate-Hypershift-Default.yaml b/api/hypershift/v1beta1/featuregates/featureGate-Hypershift-Default.yaml index 8576664946..1c370c8571 100644 --- a/api/hypershift/v1beta1/featuregates/featureGate-Hypershift-Default.yaml +++ b/api/hypershift/v1beta1/featuregates/featureGate-Hypershift-Default.yaml @@ -15,6 +15,9 @@ "featureGates": [ { "disabled": [ + { + "name": "AutoNodeKarpenter", + }, { "name": "AROHCPManagedIdentities" }, diff --git a/api/hypershift/v1beta1/featuregates/featureGate-Hypershift-TechPreviewNoUpgrade.yaml b/api/hypershift/v1beta1/featuregates/featureGate-Hypershift-TechPreviewNoUpgrade.yaml index c67a7d1639..1e26eceb21 100644 --- a/api/hypershift/v1beta1/featuregates/featureGate-Hypershift-TechPreviewNoUpgrade.yaml +++ b/api/hypershift/v1beta1/featuregates/featureGate-Hypershift-TechPreviewNoUpgrade.yaml @@ -16,6 +16,9 @@ { "disabled": [], "enabled": [ + { + "name": "AutoNodeKarpenter", + }, { "name": "AROHCPManagedIdentities" }, diff --git a/api/hypershift/v1beta1/featuregates/featureGate-SelfManagedHA-Default.yaml b/api/hypershift/v1beta1/featuregates/featureGate-SelfManagedHA-Default.yaml index 8576664946..1c370c8571 100644 --- a/api/hypershift/v1beta1/featuregates/featureGate-SelfManagedHA-Default.yaml +++ b/api/hypershift/v1beta1/featuregates/featureGate-SelfManagedHA-Default.yaml @@ -15,6 +15,9 @@ "featureGates": [ { "disabled": [ + { + "name": "AutoNodeKarpenter", + }, { "name": "AROHCPManagedIdentities" }, diff --git a/api/hypershift/v1beta1/featuregates/featureGate-SelfManagedHA-TechPreviewNoUpgrade.yaml b/api/hypershift/v1beta1/featuregates/featureGate-SelfManagedHA-TechPreviewNoUpgrade.yaml index c67a7d1639..6b9ca32850 100644 --- a/api/hypershift/v1beta1/featuregates/featureGate-SelfManagedHA-TechPreviewNoUpgrade.yaml +++ b/api/hypershift/v1beta1/featuregates/featureGate-SelfManagedHA-TechPreviewNoUpgrade.yaml @@ -16,6 +16,9 @@ { "disabled": [], "enabled": [ + { + "name": "AutoNodeKarpenter", + }, { "name": "AROHCPManagedIdentities" }, diff --git a/api/hypershift/v1beta1/hosted_controlplane.go b/api/hypershift/v1beta1/hosted_controlplane.go index 0c46641f8d..7dffc7e199 100644 --- a/api/hypershift/v1beta1/hosted_controlplane.go +++ b/api/hypershift/v1beta1/hosted_controlplane.go @@ -175,6 +175,10 @@ type HostedControlPlaneSpec struct { // +optional Autoscaling ClusterAutoscaling `json:"autoscaling,omitempty"` + // autoNode specifies the configuration for the autoNode feature. + // +openshift:enable:FeatureGate=AutoNodeKarpenter + AutoNode *AutoNode `json:"autoNode,omitempty"` + // NodeSelector when specified, must be true for the pods managed by the HostedCluster to be scheduled. // // +optional diff --git a/api/hypershift/v1beta1/hostedcluster_types.go b/api/hypershift/v1beta1/hostedcluster_types.go index da3e55f8b1..c541d9639b 100644 --- a/api/hypershift/v1beta1/hostedcluster_types.go +++ b/api/hypershift/v1beta1/hostedcluster_types.go @@ -460,6 +460,10 @@ type HostedClusterSpec struct { // +optional Autoscaling ClusterAutoscaling `json:"autoscaling,omitempty"` + // autoNode specifies the configuration for the autoNode feature. + // +openshift:enable:FeatureGate=AutoNodeKarpenter + AutoNode *AutoNode `json:"autoNode,omitempty"` + // etcd specifies configuration for the control plane etcd cluster. The // default managementType is Managed. Once set, the managementType cannot be // changed. @@ -1096,6 +1100,47 @@ type Release struct { Image string `json:"image"` } +// We expose here internal configuration knobs that won't be exposed to the service. +type AutoNode struct { + // provisioner is the implementation used for Node auto provisioning. + // +required + Provisioner *ProvisionerConfig `json:"provisionerConfig"` +} + +// ProvisionerConfig is a enum specifying the strategy for auto managing Nodes. +type ProvisionerConfig struct { + // name specifies the name of the provisioner to use. + // +required + // +kubebuilder:validation:Enum=Karpenter + Name Provisioner `json:"name"` + // karpenter specifies the configuration for the Karpenter provisioner. + // +optional + Karpenter *KarpenterConfig `json:"karpenter,omitempty"` +} + +type KarpenterConfig struct { + // platform specifies the platform-specific configuration for Karpenter. + // +required + Platform PlatformType `json:"platform"` + // aws specifies the AWS-specific configuration for Karpenter. + // +optional + AWS *KarpenterAWSConfig `json:"aws,omitempty"` +} + +type KarpenterAWSConfig struct { + //arn specifies the ARN of the Karpenter provisioner. + // +required + RoleARN string `json:"roleARN"` +} + +const ( + ProvisionerKarpeneter Provisioner = "Karpenter" +) + +// provisioner is a enum specifying the strategy for auto managing Nodes. +// +kubebuilder:validation:Enum=Karpenter +type Provisioner string + // ClusterAutoscaling specifies auto-scaling behavior that applies to all // NodePools associated with a control plane. type ClusterAutoscaling struct { diff --git a/api/hypershift/v1beta1/zz_generated.deepcopy.go b/api/hypershift/v1beta1/zz_generated.deepcopy.go index f5c45b362d..850f73bfbc 100644 --- a/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -554,6 +554,26 @@ func (in *AllocationPool) DeepCopy() *AllocationPool { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoNode) DeepCopyInto(out *AutoNode) { + *out = *in + if in.Provisioner != nil { + in, out := &in.Provisioner, &out.Provisioner + *out = new(ProvisionerConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoNode. +func (in *AutoNode) DeepCopy() *AutoNode { + if in == nil { + return nil + } + out := new(AutoNode) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureKMSKey) DeepCopyInto(out *AzureKMSKey) { *out = *in @@ -1327,6 +1347,11 @@ func (in *HostedClusterSpec) DeepCopyInto(out *HostedClusterSpec) { in.DNS.DeepCopyInto(&out.DNS) in.Networking.DeepCopyInto(&out.Networking) in.Autoscaling.DeepCopyInto(&out.Autoscaling) + if in.AutoNode != nil { + in, out := &in.AutoNode, &out.AutoNode + *out = new(AutoNode) + (*in).DeepCopyInto(*out) + } in.Etcd.DeepCopyInto(&out.Etcd) if in.Services != nil { in, out := &in.Services, &out.Services @@ -1573,6 +1598,11 @@ func (in *HostedControlPlaneSpec) DeepCopyInto(out *HostedControlPlaneSpec) { **out = **in } in.Autoscaling.DeepCopyInto(&out.Autoscaling) + if in.AutoNode != nil { + in, out := &in.AutoNode, &out.AutoNode + *out = new(AutoNode) + (*in).DeepCopyInto(*out) + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) @@ -1840,6 +1870,41 @@ func (in *KMSSpec) DeepCopy() *KMSSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KarpenterAWSConfig) DeepCopyInto(out *KarpenterAWSConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterAWSConfig. +func (in *KarpenterAWSConfig) DeepCopy() *KarpenterAWSConfig { + if in == nil { + return nil + } + out := new(KarpenterAWSConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KarpenterConfig) DeepCopyInto(out *KarpenterConfig) { + *out = *in + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = new(KarpenterAWSConfig) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterConfig. +func (in *KarpenterConfig) DeepCopy() *KarpenterConfig { + if in == nil { + return nil + } + out := new(KarpenterConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeVirtNodePoolStatus) DeepCopyInto(out *KubeVirtNodePoolStatus) { *out = *in @@ -2986,6 +3051,26 @@ func (in *PowerVSVPC) DeepCopy() *PowerVSVPC { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProvisionerConfig) DeepCopyInto(out *ProvisionerConfig) { + *out = *in + if in.Karpenter != nil { + in, out := &in.Karpenter, &out.Karpenter + *out = new(KarpenterConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerConfig. +func (in *ProvisionerConfig) DeepCopy() *ProvisionerConfig { + if in == nil { + return nil + } + out := new(ProvisionerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Release) DeepCopyInto(out *Release) { *out = *in diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests.yaml index 608af7ce1f..2b6cc4f5d7 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests.yaml @@ -96,6 +96,7 @@ hostedclusters.hypershift.openshift.io: Category: "" FeatureGates: - AROHCPManagedIdentities + - AutoNodeKarpenter - DynamicResourceAllocation - ExternalOIDC - HCPPodsLabels @@ -149,6 +150,7 @@ hostedcontrolplanes.hypershift.openshift.io: Category: cluster-api FeatureGates: - AROHCPManagedIdentities + - AutoNodeKarpenter - DynamicResourceAllocation - ExternalOIDC - NetworkDiagnosticsConfig diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AutoNodeKarpenter.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AutoNodeKarpenter.yaml new file mode 100644 index 0000000000..cd05ae9463 --- /dev/null +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AutoNodeKarpenter.yaml @@ -0,0 +1,4655 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + feature-gate.release.openshift.io/AutoNodeKarpenter: "true" + name: hostedclusters.hypershift.openshift.io +spec: + group: hypershift.openshift.io + names: + kind: HostedCluster + listKind: HostedClusterList + plural: hostedclusters + shortNames: + - hc + - hcs + singular: hostedcluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Version + jsonPath: .status.version.history[?(@.state=="Completed")].version + name: Version + type: string + - description: KubeConfig Secret + jsonPath: .status.kubeconfig.name + name: KubeConfig + type: string + - description: Progress + jsonPath: .status.version.history[?(@.state!="")].state + name: Progress + type: string + - description: Available + jsonPath: .status.conditions[?(@.type=="Available")].status + name: Available + type: string + - description: Progressing + jsonPath: .status.conditions[?(@.type=="Progressing")].status + name: Progressing + type: string + - description: Message + jsonPath: .status.conditions[?(@.type=="Available")].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + HostedCluster is the primary representation of a HyperShift cluster and encapsulates + the control plane and common data plane configuration. Creating a HostedCluster + results in a fully functional OpenShift control plane with no attached nodes. + To support workloads (e.g. pods), a HostedCluster may have one or more associated + NodePool resources. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec is the desired behavior of the HostedCluster. + properties: + additionalTrustBundle: + description: |- + additionalTrustBundle is a local reference to a ConfigMap that must have a "ca-bundle.crt" key + whose content must be a PEM-encoded X.509 certificate bundle that will be added to the hosted controlplane and nodes + If the reference is set but none of the above requirements are met, the HostedCluster will enter a degraded state. + This will be part of every payload generated by the controllers for any NodePool of the HostedCluster. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + auditWebhook: + description: |- + AuditWebhook contains metadata for configuring an audit webhook endpoint + for a cluster to process cluster audit events. It references a secret that + contains the webhook information for the audit webhook endpoint. It is a + secret because if the endpoint has mTLS the kubeconfig will contain client + keys. The kubeconfig needs to be stored in the secret with a secret key + name that corresponds to the constant AuditWebhookKubeconfigKey. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + autoNode: + description: autoNode specifies the configuration for the autoNode + feature. + properties: + provisionerConfig: + description: provisioner is the implementation used for Node auto + provisioning. + properties: + karpenter: + description: karpenter specifies the configuration for the + Karpenter provisioner. + properties: + aws: + description: aws specifies the AWS-specific configuration + for Karpenter. + properties: + roleARN: + description: arn specifies the ARN of the Karpenter + provisioner. + type: string + required: + - roleARN + type: object + platform: + description: platform specifies the platform-specific + configuration for Karpenter. + type: string + required: + - platform + type: object + name: + allOf: + - enum: + - Karpenter + - enum: + - Karpenter + description: name specifies the name of the provisioner to + use. + type: string + required: + - name + type: object + required: + - provisionerConfig + type: object + autoscaling: + description: |- + autoscaling specifies auto-scaling behavior that applies to all NodePools + associated with this HostedCluster. + properties: + maxNodeProvisionTime: + description: |- + maxNodeProvisionTime is the maximum time to wait for node provisioning + before considering the provisioning to be unsuccessful, expressed as a Go + duration string. The default is 15 minutes. + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + maxNodesTotal: + description: |- + maxNodesTotal is the maximum allowable number of nodes for the Autoscaler scale out to be operational. + The autoscaler will not grow the cluster beyond this number. + If omitted, the autoscaler will not have a maximum limit. + number. + format: int32 + minimum: 0 + type: integer + maxPodGracePeriod: + description: |- + maxPodGracePeriod is the maximum seconds to wait for graceful pod + termination before scaling down a NodePool. The default is 600 seconds. + format: int32 + minimum: 0 + type: integer + podPriorityThreshold: + description: |- + podPriorityThreshold enables users to schedule "best-effort" pods, which + shouldn't trigger autoscaler actions, but only run when there are spare + resources available. The default is -10. + + See the following for more details: + https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#how-does-cluster-autoscaler-work-with-pod-priority-and-preemption + format: int32 + type: integer + type: object + channel: + description: |- + channel is an identifier for explicitly requesting that a non-default set of updates be applied to this cluster. + If omitted no particular upgrades are suggested. + maxLength: 100 + minLength: 1 + type: string + clusterID: + description: |- + clusterID uniquely identifies this cluster. This is expected to be an RFC4122 UUID value (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx in hexadecimal digits). + As with a Kubernetes metadata.uid, this ID uniquely identifies this cluster in space and time. + This value identifies the cluster in metrics pushed to telemetry and metrics produced by the control plane operators. + If a value is not specified, a random clusterID will be generated and set by the controller. + Once set, this value is immutable. + maxLength: 36 + minLength: 36 + type: string + x-kubernetes-validations: + - message: clusterID must be an RFC4122 UUID value (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + in hexadecimal digits) + rule: self.matches('[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}') + - message: clusterID is immutable + rule: oldSelf == "" || self == oldSelf + configuration: + description: |- + Configuration specifies configuration for individual OCP components in the + cluster, represented as embedded resources that correspond to the openshift + configuration API. + properties: + apiServer: + description: |- + APIServer holds configuration (like serving certificates, client CA and CORS domains) + shared by all API servers in the system, among them especially kube-apiserver + and openshift-apiserver. + properties: + additionalCORSAllowedOrigins: + description: |- + additionalCORSAllowedOrigins lists additional, user-defined regular expressions describing hosts for which the + API server allows access using the CORS headers. This may be needed to access the API and the integrated OAuth + server from JavaScript applications. + The values are regular expressions that correspond to the Golang regular expression language. + items: + type: string + type: array + audit: + default: + profile: Default + description: |- + audit specifies the settings for audit configuration to be applied to all OpenShift-provided + API servers in the cluster. + properties: + customRules: + description: |- + customRules specify profiles per group. These profile take precedence over the + top-level profile field if they apply. They are evaluation from top to bottom and + the first one that matches, applies. + items: + description: |- + AuditCustomRule describes a custom rule for an audit profile that takes precedence over + the top-level profile. + properties: + group: + description: group is a name of group a request + user must be member of in order to this profile + to apply. + minLength: 1 + type: string + profile: + description: |- + profile specifies the name of the desired audit policy configuration to be deployed to + all OpenShift-provided API servers in the cluster. + + The following profiles are provided: + - Default: the existing default policy. + - WriteRequestBodies: like 'Default', but logs request and response HTTP payloads for + write requests (create, update, patch). + - AllRequestBodies: like 'WriteRequestBodies', but also logs request and response + HTTP payloads for read requests (get, list). + - None: no requests are logged at all, not even oauthaccesstokens and oauthauthorizetokens. + + If unset, the 'Default' profile is used as the default. + enum: + - Default + - WriteRequestBodies + - AllRequestBodies + - None + type: string + required: + - group + - profile + type: object + type: array + x-kubernetes-list-map-keys: + - group + x-kubernetes-list-type: map + profile: + default: Default + description: |- + profile specifies the name of the desired top-level audit profile to be applied to all requests + sent to any of the OpenShift-provided API servers in the cluster (kube-apiserver, + openshift-apiserver and oauth-apiserver), with the exception of those requests that match + one or more of the customRules. + + The following profiles are provided: + - Default: default policy which means MetaData level logging with the exception of events + (not logged at all), oauthaccesstokens and oauthauthorizetokens (both logged at RequestBody + level). + - WriteRequestBodies: like 'Default', but logs request and response HTTP payloads for + write requests (create, update, patch). + - AllRequestBodies: like 'WriteRequestBodies', but also logs request and response + HTTP payloads for read requests (get, list). + - None: no requests are logged at all, not even oauthaccesstokens and oauthauthorizetokens. + + Warning: It is not recommended to disable audit logging by using the `None` profile unless you + are fully aware of the risks of not logging data that can be beneficial when troubleshooting issues. + If you disable audit logging and a support situation arises, you might need to enable audit logging + and reproduce the issue in order to troubleshoot properly. + + If unset, the 'Default' profile is used as the default. + enum: + - Default + - WriteRequestBodies + - AllRequestBodies + - None + type: string + type: object + clientCA: + description: |- + clientCA references a ConfigMap containing a certificate bundle for the signers that will be recognized for + incoming client certificates in addition to the operator managed signers. If this is empty, then only operator managed signers are valid. + You usually only have to set this if you have your own PKI you wish to honor client certificates from. + The ConfigMap must exist in the openshift-config namespace and contain the following required fields: + - ConfigMap.Data["ca-bundle.crt"] - CA bundle. + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + required: + - name + type: object + encryption: + description: encryption allows the configuration of encryption + of resources at the datastore layer. + properties: + type: + description: |- + type defines what encryption type should be used to encrypt resources at the datastore layer. + When this field is unset (i.e. when it is set to the empty string), identity is implied. + The behavior of unset can and will change over time. Even if encryption is enabled by default, + the meaning of unset may change to a different encryption type based on changes in best practices. + + When encryption is enabled, all sensitive resources shipped with the platform are encrypted. + This list of sensitive resources can and will change over time. The current authoritative list is: + + 1. secrets + 2. configmaps + 3. routes.route.openshift.io + 4. oauthaccesstokens.oauth.openshift.io + 5. oauthauthorizetokens.oauth.openshift.io + enum: + - "" + - identity + - aescbc + - aesgcm + type: string + type: object + servingCerts: + description: |- + servingCert is the TLS cert info for serving secure traffic. If not specified, operator managed certificates + will be used for serving secure traffic. + properties: + namedCertificates: + description: |- + namedCertificates references secrets containing the TLS cert info for serving secure traffic to specific hostnames. + If no named certificates are provided, or no named certificates match the server name as understood by a client, + the defaultServingCertificate will be used. + items: + description: APIServerNamedServingCert maps a server + DNS name, as understood by a client, to a certificate. + properties: + names: + description: |- + names is a optional list of explicit DNS names (leading wildcards allowed) that should use this certificate to + serve secure traffic. If no names are provided, the implicit names will be extracted from the certificates. + Exact names trump over wildcard names. Explicit names defined here trump over extracted implicit names. + items: + type: string + type: array + servingCertificate: + description: |- + servingCertificate references a kubernetes.io/tls type secret containing the TLS cert info for serving secure traffic. + The secret must exist in the openshift-config namespace and contain the following required fields: + - Secret.Data["tls.key"] - TLS private key. + - Secret.Data["tls.crt"] - TLS certificate. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + type: object + type: array + type: object + tlsSecurityProfile: + description: |- + tlsSecurityProfile specifies settings for TLS connections for externally exposed servers. + + If unset, a default (which may change between releases) is chosen. Note that only Old, + Intermediate and Custom profiles are currently supported, and the maximum available + minTLSVersion is VersionTLS12. + properties: + custom: + description: |- + custom is a user-defined TLS security profile. Be extremely careful using a custom + profile as invalid configurations can be catastrophic. An example custom profile + looks like this: + + ciphers: + + - ECDHE-ECDSA-CHACHA20-POLY1305 + + - ECDHE-RSA-CHACHA20-POLY1305 + + - ECDHE-RSA-AES128-GCM-SHA256 + + - ECDHE-ECDSA-AES128-GCM-SHA256 + + minTLSVersion: VersionTLS11 + nullable: true + properties: + ciphers: + description: |- + ciphers is used to specify the cipher algorithms that are negotiated + during the TLS handshake. Operators may remove entries their operands + do not support. For example, to use DES-CBC3-SHA (yaml): + + ciphers: + - DES-CBC3-SHA + items: + type: string + type: array + minTLSVersion: + description: |- + minTLSVersion is used to specify the minimal version of the TLS protocol + that is negotiated during the TLS handshake. For example, to use TLS + versions 1.1, 1.2 and 1.3 (yaml): + + minTLSVersion: VersionTLS11 + + NOTE: currently the highest minTLSVersion allowed is VersionTLS12 + enum: + - VersionTLS10 + - VersionTLS11 + - VersionTLS12 + - VersionTLS13 + type: string + type: object + intermediate: + description: |- + intermediate is a TLS security profile based on: + + https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29 + + and looks like this (yaml): + + ciphers: + + - TLS_AES_128_GCM_SHA256 + + - TLS_AES_256_GCM_SHA384 + + - TLS_CHACHA20_POLY1305_SHA256 + + - ECDHE-ECDSA-AES128-GCM-SHA256 + + - ECDHE-RSA-AES128-GCM-SHA256 + + - ECDHE-ECDSA-AES256-GCM-SHA384 + + - ECDHE-RSA-AES256-GCM-SHA384 + + - ECDHE-ECDSA-CHACHA20-POLY1305 + + - ECDHE-RSA-CHACHA20-POLY1305 + + - DHE-RSA-AES128-GCM-SHA256 + + - DHE-RSA-AES256-GCM-SHA384 + + minTLSVersion: VersionTLS12 + nullable: true + type: object + modern: + description: |- + modern is a TLS security profile based on: + + https://wiki.mozilla.org/Security/Server_Side_TLS#Modern_compatibility + + and looks like this (yaml): + + ciphers: + + - TLS_AES_128_GCM_SHA256 + + - TLS_AES_256_GCM_SHA384 + + - TLS_CHACHA20_POLY1305_SHA256 + + minTLSVersion: VersionTLS13 + nullable: true + type: object + old: + description: |- + old is a TLS security profile based on: + + https://wiki.mozilla.org/Security/Server_Side_TLS#Old_backward_compatibility + + and looks like this (yaml): + + ciphers: + + - TLS_AES_128_GCM_SHA256 + + - TLS_AES_256_GCM_SHA384 + + - TLS_CHACHA20_POLY1305_SHA256 + + - ECDHE-ECDSA-AES128-GCM-SHA256 + + - ECDHE-RSA-AES128-GCM-SHA256 + + - ECDHE-ECDSA-AES256-GCM-SHA384 + + - ECDHE-RSA-AES256-GCM-SHA384 + + - ECDHE-ECDSA-CHACHA20-POLY1305 + + - ECDHE-RSA-CHACHA20-POLY1305 + + - DHE-RSA-AES128-GCM-SHA256 + + - DHE-RSA-AES256-GCM-SHA384 + + - DHE-RSA-CHACHA20-POLY1305 + + - ECDHE-ECDSA-AES128-SHA256 + + - ECDHE-RSA-AES128-SHA256 + + - ECDHE-ECDSA-AES128-SHA + + - ECDHE-RSA-AES128-SHA + + - ECDHE-ECDSA-AES256-SHA384 + + - ECDHE-RSA-AES256-SHA384 + + - ECDHE-ECDSA-AES256-SHA + + - ECDHE-RSA-AES256-SHA + + - DHE-RSA-AES128-SHA256 + + - DHE-RSA-AES256-SHA256 + + - AES128-GCM-SHA256 + + - AES256-GCM-SHA384 + + - AES128-SHA256 + + - AES256-SHA256 + + - AES128-SHA + + - AES256-SHA + + - DES-CBC3-SHA + + minTLSVersion: VersionTLS10 + nullable: true + type: object + type: + description: |- + type is one of Old, Intermediate, Modern or Custom. Custom provides + the ability to specify individual TLS security profile parameters. + Old, Intermediate and Modern are TLS security profiles based on: + + https://wiki.mozilla.org/Security/Server_Side_TLS#Recommended_configurations + + The profiles are intent based, so they may change over time as new ciphers are developed and existing ciphers + are found to be insecure. Depending on precisely which ciphers are available to a process, the list may be + reduced. + + Note that the Modern profile is currently not supported because it is not + yet well adopted by common software libraries. + enum: + - Old + - Intermediate + - Modern + - Custom + type: string + type: object + type: object + authentication: + description: |- + Authentication specifies cluster-wide settings for authentication (like OAuth and + webhook token authenticators). + properties: + oauthMetadata: + description: |- + oauthMetadata contains the discovery endpoint data for OAuth 2.0 + Authorization Server Metadata for an external OAuth server. + This discovery document can be viewed from its served location: + oc get --raw '/.well-known/oauth-authorization-server' + For further details, see the IETF Draft: + https://tools.ietf.org/html/draft-ietf-oauth-discovery-04#section-2 + If oauthMetadata.name is non-empty, this value has precedence + over any metadata reference stored in status. + The key "oauthMetadata" is used to locate the data. + If specified and the config map or expected key is not found, no metadata is served. + If the specified metadata is not valid, no metadata is served. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + required: + - name + type: object + serviceAccountIssuer: + description: |- + serviceAccountIssuer is the identifier of the bound service account token + issuer. + The default is https://kubernetes.default.svc + WARNING: Updating this field will not result in immediate invalidation of all bound tokens with the + previous issuer value. Instead, the tokens issued by previous service account issuer will continue to + be trusted for a time period chosen by the platform (currently set to 24h). + This time period is subject to change over time. + This allows internal components to transition to use new service account issuer without service distruption. + type: string + type: + description: |- + type identifies the cluster managed, user facing authentication mode in use. + Specifically, it manages the component that responds to login attempts. + The default is IntegratedOAuth. + type: string + webhookTokenAuthenticator: + description: |- + webhookTokenAuthenticator configures a remote token reviewer. + These remote authentication webhooks can be used to verify bearer tokens + via the tokenreviews.authentication.k8s.io REST API. This is required to + honor bearer tokens that are provisioned by an external authentication service. + + Can only be set if "Type" is set to "None". + properties: + kubeConfig: + description: |- + kubeConfig references a secret that contains kube config file data which + describes how to access the remote webhook service. + The namespace for the referenced secret is openshift-config. + + For further details, see: + + https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication + + The key "kubeConfig" is used to locate the data. + If the secret or expected key is not found, the webhook is not honored. + If the specified kube config data is not valid, the webhook is not honored. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + required: + - kubeConfig + type: object + webhookTokenAuthenticators: + description: webhookTokenAuthenticators is DEPRECATED, setting + it has no effect. + items: + description: |- + deprecatedWebhookTokenAuthenticator holds the necessary configuration options for a remote token authenticator. + It's the same as WebhookTokenAuthenticator but it's missing the 'required' validation on KubeConfig field. + properties: + kubeConfig: + description: |- + kubeConfig contains kube config file data which describes how to access the remote webhook service. + For further details, see: + https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication + The key "kubeConfig" is used to locate the data. + If the secret or expected key is not found, the webhook is not honored. + If the specified kube config data is not valid, the webhook is not honored. + The namespace for this secret is determined by the point of use. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + featureGate: + description: FeatureGate holds cluster-wide information about + feature gates. + properties: + customNoUpgrade: + description: |- + customNoUpgrade allows the enabling or disabling of any feature. Turning this feature set on IS NOT SUPPORTED, CANNOT BE UNDONE, and PREVENTS UPGRADES. + Because of its nature, this setting cannot be validated. If you have any typos or accidentally apply invalid combinations + your cluster may fail in an unrecoverable way. featureSet must equal "CustomNoUpgrade" must be set to use this field. + nullable: true + properties: + disabled: + description: disabled is a list of all feature gates that + you want to force off + items: + description: FeatureGateName is a string to enforce + patterns on the name of a FeatureGate + pattern: ^([A-Za-z0-9-]+\.)*[A-Za-z0-9-]+\.?$ + type: string + type: array + enabled: + description: enabled is a list of all feature gates that + you want to force on + items: + description: FeatureGateName is a string to enforce + patterns on the name of a FeatureGate + pattern: ^([A-Za-z0-9-]+\.)*[A-Za-z0-9-]+\.?$ + type: string + type: array + type: object + featureSet: + description: |- + featureSet changes the list of features in the cluster. The default is empty. Be very careful adjusting this setting. + Turning on or off features may cause irreversible changes in your cluster which cannot be undone. + type: string + x-kubernetes-validations: + - message: CustomNoUpgrade may not be changed + rule: 'oldSelf == ''CustomNoUpgrade'' ? self == ''CustomNoUpgrade'' + : true' + - message: TechPreviewNoUpgrade may not be changed + rule: 'oldSelf == ''TechPreviewNoUpgrade'' ? self == ''TechPreviewNoUpgrade'' + : true' + - message: DevPreviewNoUpgrade may not be changed + rule: 'oldSelf == ''DevPreviewNoUpgrade'' ? self == ''DevPreviewNoUpgrade'' + : true' + type: object + image: + description: |- + Image governs policies related to imagestream imports and runtime configuration + for external registries. It allows cluster admins to configure which registries + OpenShift is allowed to import images from, extra CA trust bundles for external + registries, and policies to block or allow registry hostnames. + When exposing OpenShift's image registry to the public, this also lets cluster + admins specify the external hostname. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + additionalTrustedCA: + description: |- + additionalTrustedCA is a reference to a ConfigMap containing additional CAs that + should be trusted during imagestream import, pod image pull, build image pull, and + imageregistry pullthrough. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + required: + - name + type: object + allowedRegistriesForImport: + description: |- + allowedRegistriesForImport limits the container image registries that normal users may import + images from. Set this list to the registries that you trust to contain valid Docker + images and that you want applications to be able to import from. Users with + permission to create Images or ImageStreamMappings via the API are not affected by + this policy - typically only administrators or system integrations will have those + permissions. + items: + description: |- + RegistryLocation contains a location of the registry specified by the registry domain + name. The domain name might include wildcards, like '*' or '??'. + properties: + domainName: + description: |- + domainName specifies a domain name for the registry + In case the registry use non-standard (80 or 443) port, the port should be included + in the domain name as well. + type: string + insecure: + description: |- + insecure indicates whether the registry is secure (https) or insecure (http) + By default (if not specified) the registry is assumed as secure. + type: boolean + type: object + type: array + externalRegistryHostnames: + description: |- + externalRegistryHostnames provides the hostnames for the default external image + registry. The external hostname should be set only when the image registry + is exposed externally. The first value is used in 'publicDockerImageRepository' + field in ImageStreams. The value must be in "hostname[:port]" format. + items: + type: string + type: array + registrySources: + description: |- + registrySources contains configuration that determines how the container runtime + should treat individual registries when accessing images for builds+pods. (e.g. + whether or not to allow insecure access). It does not contain configuration for the + internal cluster registry. + properties: + allowedRegistries: + description: |- + allowedRegistries are the only registries permitted for image pull and push actions. All other registries are denied. + + Only one of BlockedRegistries or AllowedRegistries may be set. + items: + type: string + type: array + blockedRegistries: + description: |- + blockedRegistries cannot be used for image pull and push actions. All other registries are permitted. + + Only one of BlockedRegistries or AllowedRegistries may be set. + items: + type: string + type: array + containerRuntimeSearchRegistries: + description: |- + containerRuntimeSearchRegistries are registries that will be searched when pulling images that do not have fully qualified + domains in their pull specs. Registries will be searched in the order provided in the list. + Note: this search list only works with the container runtime, i.e CRI-O. Will NOT work with builds or imagestream imports. + format: hostname + items: + type: string + minItems: 1 + type: array + x-kubernetes-list-type: set + insecureRegistries: + description: insecureRegistries are registries which do + not have a valid TLS certificates or only support HTTP + connections. + items: + type: string + type: array + type: object + type: object + ingress: + description: |- + Ingress holds cluster-wide information about ingress, including the default ingress domain + used for routes. + properties: + appsDomain: + description: |- + appsDomain is an optional domain to use instead of the one specified + in the domain field when a Route is created without specifying an explicit + host. If appsDomain is nonempty, this value is used to generate default + host values for Route. Unlike domain, appsDomain may be modified after + installation. + This assumes a new ingresscontroller has been setup with a wildcard + certificate. + type: string + componentRoutes: + description: |- + componentRoutes is an optional list of routes that are managed by OpenShift components + that a cluster-admin is able to configure the hostname and serving certificate for. + The namespace and name of each route in this list should match an existing entry in the + status.componentRoutes list. + + To determine the set of configurable Routes, look at namespace and name of entries in the + .status.componentRoutes list, where participating operators write the status of + configurable routes. + items: + description: ComponentRouteSpec allows for configuration + of a route's hostname and serving certificate. + properties: + hostname: + description: hostname is the hostname that should be + used by the route. + pattern: ^([a-zA-Z0-9\p{S}\p{L}]((-?[a-zA-Z0-9\p{S}\p{L}]{0,62})?)|([a-zA-Z0-9\p{S}\p{L}](([a-zA-Z0-9-\p{S}\p{L}]{0,61}[a-zA-Z0-9\p{S}\p{L}])?)(\.)){1,}([a-zA-Z\p{L}]){2,63})$|^(([a-z0-9][-a-z0-9]{0,61}[a-z0-9]|[a-z0-9]{1,63})[\.]){0,}([a-z0-9][-a-z0-9]{0,61}[a-z0-9]|[a-z0-9]{1,63})$ + type: string + name: + description: |- + name is the logical name of the route to customize. + + The namespace and name of this componentRoute must match a corresponding + entry in the list of status.componentRoutes if the route is to be customized. + maxLength: 256 + minLength: 1 + type: string + namespace: + description: |- + namespace is the namespace of the route to customize. + + The namespace and name of this componentRoute must match a corresponding + entry in the list of status.componentRoutes if the route is to be customized. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + servingCertKeyPairSecret: + description: |- + servingCertKeyPairSecret is a reference to a secret of type `kubernetes.io/tls` in the openshift-config namespace. + The serving cert/key pair must match and will be used by the operator to fulfill the intent of serving with this name. + If the custom hostname uses the default routing suffix of the cluster, + the Secret specification for a serving certificate will not be needed. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + required: + - hostname + - name + - namespace + type: object + type: array + x-kubernetes-list-map-keys: + - namespace + - name + x-kubernetes-list-type: map + domain: + description: |- + domain is used to generate a default host name for a route when the + route's host name is empty. The generated host name will follow this + pattern: "..". + + It is also used as the default wildcard domain suffix for ingress. The + default ingresscontroller domain will follow this pattern: "*.". + + Once set, changing domain is not currently supported. + type: string + loadBalancer: + description: |- + loadBalancer contains the load balancer details in general which are not only specific to the underlying infrastructure + provider of the current cluster and are required for Ingress Controller to work on OpenShift. + properties: + platform: + description: |- + platform holds configuration specific to the underlying + infrastructure provider for the ingress load balancers. + When omitted, this means the user has no opinion and the platform is left + to choose reasonable defaults. These defaults are subject to change over time. + properties: + aws: + description: aws contains settings specific to the + Amazon Web Services infrastructure provider. + properties: + type: + description: |- + type allows user to set a load balancer type. + When this field is set the default ingresscontroller will get created using the specified LBType. + If this field is not set then the default ingress controller of LBType Classic will be created. + Valid values are: + + * "Classic": A Classic Load Balancer that makes routing decisions at either + the transport layer (TCP/SSL) or the application layer (HTTP/HTTPS). See + the following for additional details: + + https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#clb + + * "NLB": A Network Load Balancer that makes routing decisions at the + transport layer (TCP/SSL). See the following for additional details: + + https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#nlb + enum: + - NLB + - Classic + type: string + required: + - type + type: object + type: + description: |- + type is the underlying infrastructure provider for the cluster. + Allowed values are "AWS", "Azure", "BareMetal", "GCP", "Libvirt", + "OpenStack", "VSphere", "oVirt", "KubeVirt", "EquinixMetal", "PowerVS", + "AlibabaCloud", "Nutanix" and "None". Individual components may not support all platforms, + and must handle unrecognized platforms as None if they do not support that platform. + enum: + - "" + - AWS + - Azure + - BareMetal + - GCP + - Libvirt + - OpenStack + - None + - VSphere + - oVirt + - IBMCloud + - KubeVirt + - EquinixMetal + - PowerVS + - AlibabaCloud + - Nutanix + - External + type: string + type: object + type: object + requiredHSTSPolicies: + description: |- + requiredHSTSPolicies specifies HSTS policies that are required to be set on newly created or updated routes + matching the domainPattern/s and namespaceSelector/s that are specified in the policy. + Each requiredHSTSPolicy must have at least a domainPattern and a maxAge to validate a route HSTS Policy route + annotation, and affect route admission. + + A candidate route is checked for HSTS Policies if it has the HSTS Policy route annotation: + "haproxy.router.openshift.io/hsts_header" + E.g. haproxy.router.openshift.io/hsts_header: max-age=31536000;preload;includeSubDomains + + - For each candidate route, if it matches a requiredHSTSPolicy domainPattern and optional namespaceSelector, + then the maxAge, preloadPolicy, and includeSubdomainsPolicy must be valid to be admitted. Otherwise, the route + is rejected. + - The first match, by domainPattern and optional namespaceSelector, in the ordering of the RequiredHSTSPolicies + determines the route's admission status. + - If the candidate route doesn't match any requiredHSTSPolicy domainPattern and optional namespaceSelector, + then it may use any HSTS Policy annotation. + + The HSTS policy configuration may be changed after routes have already been created. An update to a previously + admitted route may then fail if the updated route does not conform to the updated HSTS policy configuration. + However, changing the HSTS policy configuration will not cause a route that is already admitted to stop working. + + Note that if there are no RequiredHSTSPolicies, any HSTS Policy annotation on the route is valid. + items: + properties: + domainPatterns: + description: |- + domainPatterns is a list of domains for which the desired HSTS annotations are required. + If domainPatterns is specified and a route is created with a spec.host matching one of the domains, + the route must specify the HSTS Policy components described in the matching RequiredHSTSPolicy. + + The use of wildcards is allowed like this: *.foo.com matches everything under foo.com. + foo.com only matches foo.com, so to cover foo.com and everything under it, you must specify *both*. + items: + type: string + minItems: 1 + type: array + includeSubDomainsPolicy: + description: |- + includeSubDomainsPolicy means the HSTS Policy should apply to any subdomains of the host's + domain name. Thus, for the host bar.foo.com, if includeSubDomainsPolicy was set to RequireIncludeSubDomains: + - the host app.bar.foo.com would inherit the HSTS Policy of bar.foo.com + - the host bar.foo.com would inherit the HSTS Policy of bar.foo.com + - the host foo.com would NOT inherit the HSTS Policy of bar.foo.com + - the host def.foo.com would NOT inherit the HSTS Policy of bar.foo.com + enum: + - RequireIncludeSubDomains + - RequireNoIncludeSubDomains + - NoOpinion + type: string + maxAge: + description: |- + maxAge is the delta time range in seconds during which hosts are regarded as HSTS hosts. + If set to 0, it negates the effect, and hosts are removed as HSTS hosts. + If set to 0 and includeSubdomains is specified, all subdomains of the host are also removed as HSTS hosts. + maxAge is a time-to-live value, and if this policy is not refreshed on a client, the HSTS + policy will eventually expire on that client. + properties: + largestMaxAge: + description: |- + The largest allowed value (in seconds) of the RequiredHSTSPolicy max-age + This value can be left unspecified, in which case no upper limit is enforced. + format: int32 + maximum: 2147483647 + minimum: 0 + type: integer + smallestMaxAge: + description: |- + The smallest allowed value (in seconds) of the RequiredHSTSPolicy max-age + Setting max-age=0 allows the deletion of an existing HSTS header from a host. This is a necessary + tool for administrators to quickly correct mistakes. + This value can be left unspecified, in which case no lower limit is enforced. + format: int32 + maximum: 2147483647 + minimum: 0 + type: integer + type: object + namespaceSelector: + description: |- + namespaceSelector specifies a label selector such that the policy applies only to those routes that + are in namespaces with labels that match the selector, and are in one of the DomainPatterns. + Defaults to the empty LabelSelector, which matches everything. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + preloadPolicy: + description: |- + preloadPolicy directs the client to include hosts in its host preload list so that + it never needs to do an initial load to get the HSTS header (note that this is not defined + in RFC 6797 and is therefore client implementation-dependent). + enum: + - RequirePreload + - RequireNoPreload + - NoOpinion + type: string + required: + - domainPatterns + type: object + type: array + type: object + network: + description: |- + Network holds cluster-wide information about the network. It is used to configure the desired network configuration, such as: IP address pools for services/pod IPs, network plugin, etc. + Please view network.spec for an explanation on what applies when configuring this resource. + properties: + clusterNetwork: + description: |- + IP address pool to use for pod IPs. + This field is immutable after installation. + items: + description: |- + ClusterNetworkEntry is a contiguous block of IP addresses from which pod IPs + are allocated. + properties: + cidr: + description: The complete block for pod IPs. + type: string + hostPrefix: + description: |- + The size (prefix) of block to allocate to each node. If this + field is not used by the plugin, it can be left unset. + format: int32 + minimum: 0 + type: integer + type: object + type: array + x-kubernetes-list-type: atomic + externalIP: + description: |- + externalIP defines configuration for controllers that + affect Service.ExternalIP. If nil, then ExternalIP is + not allowed to be set. + properties: + autoAssignCIDRs: + description: |- + autoAssignCIDRs is a list of CIDRs from which to automatically assign + Service.ExternalIP. These are assigned when the service is of type + LoadBalancer. In general, this is only useful for bare-metal clusters. + In Openshift 3.x, this was misleadingly called "IngressIPs". + Automatically assigned External IPs are not affected by any + ExternalIPPolicy rules. + Currently, only one entry may be provided. + items: + type: string + type: array + x-kubernetes-list-type: atomic + policy: + description: |- + policy is a set of restrictions applied to the ExternalIP field. + If nil or empty, then ExternalIP is not allowed to be set. + properties: + allowedCIDRs: + description: allowedCIDRs is the list of allowed CIDRs. + items: + type: string + type: array + x-kubernetes-list-type: atomic + rejectedCIDRs: + description: |- + rejectedCIDRs is the list of disallowed CIDRs. These take precedence + over allowedCIDRs. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + type: object + networkType: + description: |- + NetworkType is the plugin that is to be deployed (e.g. OpenShiftSDN). + This should match a value that the cluster-network-operator understands, + or else no networking will be installed. + Currently supported values are: + - OpenShiftSDN + This field is immutable after installation. + type: string + serviceNetwork: + description: |- + IP address pool for services. + Currently, we only support a single entry here. + This field is immutable after installation. + items: + type: string + type: array + x-kubernetes-list-type: atomic + serviceNodePortRange: + description: |- + The port range allowed for Services of type NodePort. + If not specified, the default of 30000-32767 will be used. + Such Services without a NodePort specified will have one + automatically allocated from this range. + This parameter can be updated after the cluster is + installed. + pattern: ^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$ + type: string + type: object + oauth: + description: |- + OAuth holds cluster-wide information about OAuth. + It is used to configure the integrated OAuth server. + This configuration is only honored when the top level Authentication config has type set to IntegratedOAuth. + properties: + identityProviders: + description: |- + identityProviders is an ordered list of ways for a user to identify themselves. + When this list is empty, no identities are provisioned for users. + items: + description: IdentityProvider provides identities for users + authenticating using credentials + properties: + basicAuth: + description: basicAuth contains configuration options + for the BasicAuth IdP + properties: + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + tlsClientCert: + description: |- + tlsClientCert is an optional reference to a secret by name that contains the + PEM-encoded TLS client certificate to present when connecting to the server. + The key "tls.crt" is used to locate the data. + If specified and the secret or expected key is not found, the identity provider is not honored. + If the specified certificate data is not valid, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + tlsClientKey: + description: |- + tlsClientKey is an optional reference to a secret by name that contains the + PEM-encoded TLS private key for the client certificate referenced in tlsClientCert. + The key "tls.key" is used to locate the data. + If specified and the secret or expected key is not found, the identity provider is not honored. + If the specified certificate data is not valid, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + url: + description: url is the remote URL to connect to + type: string + type: object + github: + description: github enables user authentication using + GitHub credentials + properties: + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + This can only be configured when hostname is set to a non-empty value. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + clientID: + description: clientID is the oauth client ID + type: string + clientSecret: + description: |- + clientSecret is a required reference to the secret by name containing the oauth client secret. + The key "clientSecret" is used to locate the data. + If the secret or expected key is not found, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + hostname: + description: |- + hostname is the optional domain (e.g. "mycompany.com") for use with a hosted instance of + GitHub Enterprise. + It must match the GitHub Enterprise settings value configured at /setup/settings#hostname. + type: string + organizations: + description: organizations optionally restricts + which organizations are allowed to log in + items: + type: string + type: array + teams: + description: teams optionally restricts which teams + are allowed to log in. Format is /. + items: + type: string + type: array + type: object + gitlab: + description: gitlab enables user authentication using + GitLab credentials + properties: + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + clientID: + description: clientID is the oauth client ID + type: string + clientSecret: + description: |- + clientSecret is a required reference to the secret by name containing the oauth client secret. + The key "clientSecret" is used to locate the data. + If the secret or expected key is not found, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + url: + description: url is the oauth server base URL + type: string + type: object + google: + description: google enables user authentication using + Google credentials + properties: + clientID: + description: clientID is the oauth client ID + type: string + clientSecret: + description: |- + clientSecret is a required reference to the secret by name containing the oauth client secret. + The key "clientSecret" is used to locate the data. + If the secret or expected key is not found, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + hostedDomain: + description: hostedDomain is the optional Google + App domain (e.g. "mycompany.com") to restrict + logins to + type: string + type: object + htpasswd: + description: htpasswd enables user authentication using + an HTPasswd file to validate credentials + properties: + fileData: + description: |- + fileData is a required reference to a secret by name containing the data to use as the htpasswd file. + The key "htpasswd" is used to locate the data. + If the secret or expected key is not found, the identity provider is not honored. + If the specified htpasswd data is not valid, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + type: object + keystone: + description: keystone enables user authentication using + keystone password credentials + properties: + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + domainName: + description: domainName is required for keystone + v3 + type: string + tlsClientCert: + description: |- + tlsClientCert is an optional reference to a secret by name that contains the + PEM-encoded TLS client certificate to present when connecting to the server. + The key "tls.crt" is used to locate the data. + If specified and the secret or expected key is not found, the identity provider is not honored. + If the specified certificate data is not valid, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + tlsClientKey: + description: |- + tlsClientKey is an optional reference to a secret by name that contains the + PEM-encoded TLS private key for the client certificate referenced in tlsClientCert. + The key "tls.key" is used to locate the data. + If specified and the secret or expected key is not found, the identity provider is not honored. + If the specified certificate data is not valid, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + url: + description: url is the remote URL to connect to + type: string + type: object + ldap: + description: ldap enables user authentication using + LDAP credentials + properties: + attributes: + description: attributes maps LDAP attributes to + identities + properties: + email: + description: |- + email is the list of attributes whose values should be used as the email address. Optional. + If unspecified, no email is set for the identity + items: + type: string + type: array + id: + description: |- + id is the list of attributes whose values should be used as the user ID. Required. + First non-empty attribute is used. At least one attribute is required. If none of the listed + attribute have a value, authentication fails. + LDAP standard identity attribute is "dn" + items: + type: string + type: array + name: + description: |- + name is the list of attributes whose values should be used as the display name. Optional. + If unspecified, no display name is set for the identity + LDAP standard display name attribute is "cn" + items: + type: string + type: array + preferredUsername: + description: |- + preferredUsername is the list of attributes whose values should be used as the preferred username. + LDAP standard login attribute is "uid" + items: + type: string + type: array + type: object + bindDN: + description: bindDN is an optional DN to bind with + during the search phase. + type: string + bindPassword: + description: |- + bindPassword is an optional reference to a secret by name + containing a password to bind with during the search phase. + The key "bindPassword" is used to locate the data. + If specified and the secret or expected key is not found, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + insecure: + description: |- + insecure, if true, indicates the connection should not use TLS + WARNING: Should not be set to `true` with the URL scheme "ldaps://" as "ldaps://" URLs always + attempt to connect using TLS, even when `insecure` is set to `true` + When `true`, "ldap://" URLS connect insecurely. When `false`, "ldap://" URLs are upgraded to + a TLS connection using StartTLS as specified in https://tools.ietf.org/html/rfc2830. + type: boolean + url: + description: |- + url is an RFC 2255 URL which specifies the LDAP search parameters to use. + The syntax of the URL is: + ldap://host:port/basedn?attribute?scope?filter + type: string + type: object + mappingMethod: + description: |- + mappingMethod determines how identities from this provider are mapped to users + Defaults to "claim" + type: string + name: + description: |- + name is used to qualify the identities returned by this provider. + - It MUST be unique and not shared by any other identity provider used + - It MUST be a valid path segment: name cannot equal "." or ".." or contain "/" or "%" or ":" + Ref: https://godoc.org/github.com/openshift/origin/pkg/user/apis/user/validation#ValidateIdentityProviderName + type: string + openID: + description: openID enables user authentication using + OpenID credentials + properties: + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + claims: + description: claims mappings + properties: + email: + description: |- + email is the list of claims whose values should be used as the email address. Optional. + If unspecified, no email is set for the identity + items: + type: string + type: array + x-kubernetes-list-type: atomic + groups: + description: |- + groups is the list of claims value of which should be used to synchronize groups + from the OIDC provider to OpenShift for the user. + If multiple claims are specified, the first one with a non-empty value is used. + items: + description: |- + OpenIDClaim represents a claim retrieved from an OpenID provider's tokens or userInfo + responses + minLength: 1 + type: string + type: array + x-kubernetes-list-type: atomic + name: + description: |- + name is the list of claims whose values should be used as the display name. Optional. + If unspecified, no display name is set for the identity + items: + type: string + type: array + x-kubernetes-list-type: atomic + preferredUsername: + description: |- + preferredUsername is the list of claims whose values should be used as the preferred username. + If unspecified, the preferred username is determined from the value of the sub claim + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + clientID: + description: clientID is the oauth client ID + type: string + clientSecret: + description: |- + clientSecret is a required reference to the secret by name containing the oauth client secret. + The key "clientSecret" is used to locate the data. + If the secret or expected key is not found, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + extraAuthorizeParameters: + additionalProperties: + type: string + description: extraAuthorizeParameters are any custom + parameters to add to the authorize request. + type: object + extraScopes: + description: extraScopes are any scopes to request + in addition to the standard "openid" scope. + items: + type: string + type: array + issuer: + description: |- + issuer is the URL that the OpenID Provider asserts as its Issuer Identifier. + It must use the https scheme with no query or fragment component. + type: string + type: object + requestHeader: + description: requestHeader enables user authentication + using request header credentials + properties: + ca: + description: |- + ca is a required reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + Specifically, it allows verification of incoming requests to prevent header spoofing. + The key "ca.crt" is used to locate the data. + If the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + challengeURL: + description: |- + challengeURL is a URL to redirect unauthenticated /authorize requests to + Unauthenticated requests from OAuth clients which expect WWW-Authenticate challenges will be + redirected here. + ${url} is replaced with the current URL, escaped to be safe in a query parameter + https://www.example.com/sso-login?then=${url} + ${query} is replaced with the current query string + https://www.example.com/auth-proxy/oauth/authorize?${query} + Required when challenge is set to true. + type: string + clientCommonNames: + description: |- + clientCommonNames is an optional list of common names to require a match from. If empty, any + client certificate validated against the clientCA bundle is considered authoritative. + items: + type: string + type: array + emailHeaders: + description: emailHeaders is the set of headers + to check for the email address + items: + type: string + type: array + headers: + description: headers is the set of headers to check + for identity information + items: + type: string + type: array + loginURL: + description: |- + loginURL is a URL to redirect unauthenticated /authorize requests to + Unauthenticated requests from OAuth clients which expect interactive logins will be redirected here + ${url} is replaced with the current URL, escaped to be safe in a query parameter + https://www.example.com/sso-login?then=${url} + ${query} is replaced with the current query string + https://www.example.com/auth-proxy/oauth/authorize?${query} + Required when login is set to true. + type: string + nameHeaders: + description: nameHeaders is the set of headers to + check for the display name + items: + type: string + type: array + preferredUsernameHeaders: + description: preferredUsernameHeaders is the set + of headers to check for the preferred username + items: + type: string + type: array + type: object + type: + description: type identifies the identity provider type + for this entry. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + templates: + description: templates allow you to customize pages like the + login page. + properties: + error: + description: |- + error is the name of a secret that specifies a go template to use to render error pages + during the authentication or grant flow. + The key "errors.html" is used to locate the template data. + If specified and the secret or expected key is not found, the default error page is used. + If the specified template is not valid, the default error page is used. + If unspecified, the default error page is used. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + login: + description: |- + login is the name of a secret that specifies a go template to use to render the login page. + The key "login.html" is used to locate the template data. + If specified and the secret or expected key is not found, the default login page is used. + If the specified template is not valid, the default login page is used. + If unspecified, the default login page is used. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + providerSelection: + description: |- + providerSelection is the name of a secret that specifies a go template to use to render + the provider selection page. + The key "providers.html" is used to locate the template data. + If specified and the secret or expected key is not found, the default provider selection page is used. + If the specified template is not valid, the default provider selection page is used. + If unspecified, the default provider selection page is used. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + type: object + tokenConfig: + description: tokenConfig contains options for authorization + and access tokens + properties: + accessTokenInactivityTimeout: + description: |- + accessTokenInactivityTimeout defines the token inactivity timeout + for tokens granted by any client. + The value represents the maximum amount of time that can occur between + consecutive uses of the token. Tokens become invalid if they are not + used within this temporal window. The user will need to acquire a new + token to regain access once a token times out. Takes valid time + duration string such as "5m", "1.5h" or "2h45m". The minimum allowed + value for duration is 300s (5 minutes). If the timeout is configured + per client, then that value takes precedence. If the timeout value is + not specified and the client does not override the value, then tokens + are valid until their lifetime. + + WARNING: existing tokens' timeout will not be affected (lowered) by changing this value + type: string + accessTokenInactivityTimeoutSeconds: + description: 'accessTokenInactivityTimeoutSeconds - DEPRECATED: + setting this field has no effect.' + format: int32 + type: integer + accessTokenMaxAgeSeconds: + description: accessTokenMaxAgeSeconds defines the maximum + age of access tokens + format: int32 + type: integer + type: object + type: object + x-kubernetes-validations: + - message: spec.configuration.oauth.tokenConfig.accessTokenInactivityTimeout + minimum acceptable token timeout value is 300 seconds + rule: '!has(self.tokenConfig) || !has(self.tokenConfig.accessTokenInactivityTimeout) + || duration(self.tokenConfig.accessTokenInactivityTimeout).getSeconds() + >= 300' + operatorhub: + description: |- + OperatorHub specifies the configuration for the Operator Lifecycle Manager in the HostedCluster. This is only configured at deployment time but the controller are not reconcilling over it. + The OperatorHub configuration will be constantly reconciled if catalog placement is management, but only on cluster creation otherwise. + properties: + disableAllDefaultSources: + description: |- + disableAllDefaultSources allows you to disable all the default hub + sources. If this is true, a specific entry in sources can be used to + enable a default source. If this is false, a specific entry in + sources can be used to disable or enable a default source. + type: boolean + sources: + description: |- + sources is the list of default hub sources and their configuration. + If the list is empty, it implies that the default hub sources are + enabled on the cluster unless disableAllDefaultSources is true. + If disableAllDefaultSources is true and sources is not empty, + the configuration present in sources will take precedence. The list of + default hub sources and their current state will always be reflected in + the status block. + items: + description: HubSource is used to specify the hub source + and its configuration + properties: + disabled: + description: disabled is used to disable a default hub + source on cluster + type: boolean + name: + description: name is the name of one of the default + hub sources + maxLength: 253 + minLength: 1 + type: string + type: object + type: array + type: object + proxy: + description: ProxySpec contains cluster proxy creation configuration. + properties: + httpProxy: + description: httpProxy is the URL of the proxy for HTTP requests. Empty + means unset and will not result in an env var. + type: string + httpsProxy: + description: httpsProxy is the URL of the proxy for HTTPS + requests. Empty means unset and will not result in an env + var. + type: string + noProxy: + description: |- + noProxy is a comma-separated list of hostnames and/or CIDRs and/or IPs for which the proxy should not be used. + Empty means unset and will not result in an env var. + type: string + readinessEndpoints: + description: readinessEndpoints is a list of endpoints used + to verify readiness of the proxy. + items: + type: string + type: array + trustedCA: + description: |- + trustedCA is a reference to a ConfigMap containing a CA certificate bundle. + The trustedCA field should only be consumed by a proxy validator. The + validator is responsible for reading the certificate bundle from the required + key "ca-bundle.crt", merging it with the system default trust bundle, + and writing the merged trust bundle to a ConfigMap named "trusted-ca-bundle" + in the "openshift-config-managed" namespace. Clients that expect to make + proxy connections must use the trusted-ca-bundle for all HTTPS requests to + the proxy, and may use the trusted-ca-bundle for non-proxy HTTPS requests as + well. + + The namespace for the ConfigMap referenced by trustedCA is + "openshift-config". Here is an example ConfigMap (in yaml): + + apiVersion: v1 + kind: ConfigMap + metadata: + name: user-ca-bundle + namespace: openshift-config + data: + ca-bundle.crt: | + -----BEGIN CERTIFICATE----- + Custom CA certificate bundle. + -----END CERTIFICATE----- + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + required: + - name + type: object + type: object + scheduler: + description: |- + Scheduler holds cluster-wide config information to run the Kubernetes Scheduler + and influence its placement decisions. The canonical name for this config is `cluster`. + properties: + defaultNodeSelector: + description: |- + defaultNodeSelector helps set the cluster-wide default node selector to + restrict pod placement to specific nodes. This is applied to the pods + created in all namespaces and creates an intersection with any existing + nodeSelectors already set on a pod, additionally constraining that pod's selector. + For example, + defaultNodeSelector: "type=user-node,region=east" would set nodeSelector + field in pod spec to "type=user-node,region=east" to all pods created + in all namespaces. Namespaces having project-wide node selectors won't be + impacted even if this field is set. This adds an annotation section to + the namespace. + For example, if a new namespace is created with + node-selector='type=user-node,region=east', + the annotation openshift.io/node-selector: type=user-node,region=east + gets added to the project. When the openshift.io/node-selector annotation + is set on the project the value is used in preference to the value we are setting + for defaultNodeSelector field. + For instance, + openshift.io/node-selector: "type=user-node,region=west" means + that the default of "type=user-node,region=east" set in defaultNodeSelector + would not be applied. + type: string + mastersSchedulable: + description: |- + MastersSchedulable allows masters nodes to be schedulable. When this flag is + turned on, all the master nodes in the cluster will be made schedulable, + so that workload pods can run on them. The default value for this field is false, + meaning none of the master nodes are schedulable. + Important Note: Once the workload pods start running on the master nodes, + extreme care must be taken to ensure that cluster-critical control plane components + are not impacted. + Please turn on this field after doing due diligence. + type: boolean + policy: + description: |- + DEPRECATED: the scheduler Policy API has been deprecated and will be removed in a future release. + policy is a reference to a ConfigMap containing scheduler policy which has + user specified predicates and priorities. If this ConfigMap is not available + scheduler will default to use DefaultAlgorithmProvider. + The namespace for this configmap is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + required: + - name + type: object + profile: + description: |- + profile sets which scheduling profile should be set in order to configure scheduling + decisions for new pods. + + Valid values are "LowNodeUtilization", "HighNodeUtilization", "NoScoring" + Defaults to "LowNodeUtilization" + enum: + - "" + - LowNodeUtilization + - HighNodeUtilization + - NoScoring + type: string + type: object + type: object + controlPlaneRelease: + description: |- + controlPlaneRelease is like spec.release but only for the components running on the management cluster. + This excludes any operand which will land in the hosted cluster data plane. + It is useful when you need to apply patch management side like a CVE, transparently for the hosted cluster. + Version input for this field is free, no validation is performed against spec.release or maximum and minimum is performed. + If defined, it will dicate the version of the components running management side, while spec.release will dictate the version of the components landing in the hosted cluster data plane. + If not defined, spec.release is used for both. + Changing this field will trigger a rollout of the control plane. + The behavior of the rollout will be driven by the ControllerAvailabilityPolicy and InfrastructureAvailabilityPolicy for PDBs and maxUnavailable and surce policies. + properties: + image: + description: |- + Image is the image pullspec of an OCP release payload image. + See https://quay.io/repository/openshift-release-dev/ocp-release?tab=tags for a list of available images. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: Image must start with a word character (letters, digits, + or underscores) and contain no white spaces + rule: self.matches('^(\\w+\\S+)$') + required: + - image + type: object + controllerAvailabilityPolicy: + default: HighlyAvailable + description: |- + controllerAvailabilityPolicy specifies the availability policy applied to critical control plane components like the Kube API Server. + Possible values are HighlyAvailable and SingleReplica. The default value is HighlyAvailable. + enum: + - HighlyAvailable + - SingleReplica + type: string + dns: + description: dns specifies the DNS configuration for the hosted cluster + ingress. + properties: + baseDomain: + description: |- + baseDomain is the base domain of the hosted cluster. + It will be used to configure ingress in the hosted cluster through the subdomain baseDomainPrefix.baseDomain. + If baseDomainPrefix is omitted, the hostedCluster.name will be used as the subdomain. + Once set, this field is immutable. + When the value is the empty string "", the controller might default to a value depending on the platform. + maxLength: 253 + type: string + x-kubernetes-validations: + - message: baseDomain must be a valid domain name (e.g., example, + example.com, sub.example.com) + rule: self == "" || self.matches('^(?:(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}|[a-zA-Z0-9-]+)$') + - message: baseDomain is immutable + rule: oldSelf == "" || self == oldSelf + baseDomainPrefix: + description: |- + baseDomainPrefix is the base domain prefix for the hosted cluster ingress. + It will be used to configure ingress in the hosted cluster through the subdomain baseDomainPrefix.baseDomain. + If baseDomainPrefix is omitted, the hostedCluster.name will be used as the subdomain. + Set baseDomainPrefix to an empty string "", if you don't want a prefix at all (not even hostedCluster.name) to be prepended to baseDomain. + This field is immutable. + maxLength: 253 + type: string + x-kubernetes-validations: + - message: baseDomainPrefix must be a valid domain name (e.g., + example, example.com, sub.example.com) + rule: self == "" || self.matches('^(?:(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}|[a-zA-Z0-9-]+)$') + - message: baseDomainPrefix is immutable + rule: self == oldSelf + privateZoneID: + description: |- + privateZoneID is the Hosted Zone ID where all the DNS records that are only available internally to the cluster exist. + This field is optional and mainly leveraged in cloud environments where the DNS records for the .baseDomain are created by controllers in this zone. + Once set, this value is immutable. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: privateZoneID is immutable + rule: oldSelf == "" || self == oldSelf + publicZoneID: + description: |- + publicZoneID is the Hosted Zone ID where all the DNS records that are publicly accessible to the internet exist. + This field is optional and mainly leveraged in cloud environments where the DNS records for the .baseDomain are created by controllers in this zone. + Once set, this value is immutable. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: publicZoneID is immutable + rule: oldSelf == "" || self == oldSelf + required: + - baseDomain + type: object + etcd: + default: + managed: + storage: + persistentVolume: + size: 8Gi + type: PersistentVolume + managementType: Managed + description: |- + etcd specifies configuration for the control plane etcd cluster. The + default managementType is Managed. Once set, the managementType cannot be + changed. + properties: + managed: + description: managed specifies the behavior of an etcd cluster + managed by HyperShift. + properties: + storage: + description: storage specifies how etcd data is persisted. + properties: + persistentVolume: + description: |- + persistentVolume is the configuration for PersistentVolume etcd storage. + With this implementation, a PersistentVolume will be allocated for every + etcd member (either 1 or 3 depending on the HostedCluster control plane + availability configuration). + properties: + size: + anyOf: + - type: integer + - type: string + default: 8Gi + description: |- + size is the minimum size of the data volume for each etcd member. + Default is 8Gi. + This field is immutable + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + x-kubernetes-validations: + - message: Etcd PV storage size is immutable + rule: self == oldSelf + storageClassName: + description: |- + storageClassName is the StorageClass of the data volume for each etcd member. + See https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1. + type: string + x-kubernetes-validations: + - message: storageClassName is immutable + rule: self == oldSelf + type: object + restoreSnapshotURL: + description: |- + restoreSnapshotURL allows an optional URL to be provided where + an etcd snapshot can be downloaded, for example a pre-signed URL + referencing a storage service. + This snapshot will be restored on initial startup, only when the etcd PV + is empty. + items: + type: string + type: array + x-kubernetes-validations: + - message: RestoreSnapshotURL shouldn't contain more than + 1 entry + rule: self.size() <= 1 + type: + description: |- + type is the kind of persistent storage implementation to use for etcd. + Only PersistentVolume is supported at the moment. + enum: + - PersistentVolume + type: string + required: + - type + type: object + required: + - storage + type: object + managementType: + description: |- + managementType defines how the etcd cluster is managed. + This can be either Managed or Unmanaged. + This field is immutable. + enum: + - Managed + - Unmanaged + type: string + x-kubernetes-validations: + - message: managementType is immutable + rule: self == oldSelf + unmanaged: + description: |- + unmanaged specifies configuration which enables the control plane to + integrate with an externally managed etcd cluster. + properties: + endpoint: + description: |- + endpoint is the full etcd cluster client endpoint URL. For example: + + https://etcd-client:2379 + + If the URL uses an HTTPS scheme, the TLS field is required. + pattern: ^https:// + type: string + tls: + description: tls specifies TLS configuration for HTTPS etcd + client endpoints. + properties: + clientSecret: + description: |- + ClientSecret refers to a secret for client mTLS authentication with the etcd cluster. It + may have the following key/value pairs: + + etcd-client-ca.crt: Certificate Authority value + etcd-client.crt: Client certificate value + etcd-client.key: Client certificate key value + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - clientSecret + type: object + required: + - endpoint + - tls + type: object + required: + - managementType + type: object + x-kubernetes-validations: + - message: Only managed configuration must be set when managementType + is Managed + rule: 'self.managementType == ''Managed'' ? has(self.managed) : + !has(self.managed)' + - message: Only unmanaged configuration must be set when managementType + is Unmanaged + rule: 'self.managementType == ''Unmanaged'' ? has(self.unmanaged) + : !has(self.unmanaged)' + fips: + description: |- + fips indicates whether this cluster's nodes will be running in FIPS mode. + If set to true, the control plane's ignition server will be configured to + expect that nodes joining the cluster will be FIPS-enabled. + type: boolean + x-kubernetes-validations: + - message: fips is immutable + rule: self == oldSelf + imageContentSources: + description: |- + imageContentSources specifies image mirrors that can be used by cluster + nodes to pull content. + When imageContentSources is set, the controllers will generate a machineConfig. + This MachineConfig will be part of every payload generated by the controllers for any NodePool of the HostedCluster. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + items: + description: |- + ImageContentSource specifies image mirrors that can be used by cluster nodes + to pull content. For cluster workloads, if a container image registry host of + the pullspec matches Source then one of the Mirrors are substituted as hosts + in the pullspec and tried in order to fetch the image. + properties: + mirrors: + description: Mirrors are one or more repositories that may also + contain the same images. + items: + type: string + type: array + source: + description: |- + Source is the repository that users refer to, e.g. in image pull + specifications. + type: string + required: + - source + type: object + type: array + infraID: + description: |- + infraID is a globally unique identifier for the cluster. + It must consist of lowercase alphanumeric characters and hyphens ('-') only, and start and end with an alphanumeric character. + It must be no more than 253 characters in length. + This identifier will be used to associate various cloud resources with the HostedCluster and its associated NodePools. + infraID is used to compute and tag created resources with "kubernetes.io/cluster/"+hcluster.Spec.InfraID which has contractual meaning for the cloud provider implementations. + If a value is not specified, a random infraID will be generated and set by the controller. + Once set, this value is immutable. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: infraID must consist of lowercase alphanumeric characters + or '-', start and end with an alphanumeric character, and be between + 1 and 253 characters + rule: self.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') + - message: infraID is immutable + rule: oldSelf == "" || self == oldSelf + infrastructureAvailabilityPolicy: + default: SingleReplica + description: |- + infrastructureAvailabilityPolicy specifies the availability policy applied to infrastructure services which run on the hosted cluster data plane like the ingress controller and image registry controller. + Possible values are HighlyAvailable and SingleReplica. The default value is SingleReplica. + enum: + - HighlyAvailable + - SingleReplica + type: string + issuerURL: + default: https://kubernetes.default.svc + description: |- + issuerURL is an OIDC issuer URL which will be used as the issuer in all + ServiceAccount tokens generated by the control plane API server via --service-account-issuer kube api server flag. + https://k8s-docs.netlify.app/en/docs/reference/command-line-tools-reference/kube-apiserver/ + https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#serviceaccount-token-volume-projection + The default value is kubernetes.default.svc, which only works for in-cluster + validation. + If the platform is AWS and this value is set, the controller will update an s3 object with the appropriate OIDC documents (using the serviceAccountSigningKey info) into that issuerURL. + The expectation is for this s3 url to be backed by an OIDC provider in the AWS IAM. + type: string + x-kubernetes-validations: + - message: issuerURL is immutable + rule: self == oldSelf + - message: issuerURL must be a valid absolute URL + rule: isURL(self) + networking: + default: + clusterNetwork: + - cidr: 10.132.0.0/14 + networkType: OVNKubernetes + serviceNetwork: + - cidr: 172.31.0.0/16 + description: |- + networking specifies network configuration for the hosted cluster. + Defaults to OVNKubernetes with a cluster network of cidr: "10.132.0.0/14" and a service network of cidr: "172.31.0.0/16". + properties: + apiServer: + description: |- + apiServer contains advanced network settings for the API server that affect + how the APIServer is exposed inside a hosted cluster node. + properties: + advertiseAddress: + description: |- + advertiseAddress is the address that pods within the nodes will use to talk to the API + server. This is an address associated with the loopback adapter of each + node. If not specified, the controller will take default values. + The default values will be set as 172.20.0.1 or fd00::1. + This value is immutable. + type: string + x-kubernetes-validations: + - message: advertiseAddress is immutable + rule: self == oldSelf + allowedCIDRBlocks: + description: |- + allowedCIDRBlocks is an allow list of CIDR blocks that can access the APIServer + If not specified, traffic is allowed from all addresses. + This depends on underlying support by the cloud provider for Service LoadBalancerSourceRanges + items: + pattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(3[0-2]|[1-2][0-9]|[0-9]))$ + type: string + type: array + port: + description: |- + port is the port at which the APIServer is exposed inside a node. Other + pods using host networking cannot listen on this port. + If omitted 6443 is used. + This is useful to choose a port other than the default one which might interfere with customer environments e.g. https://github.com/openshift/hypershift/pull/356. + Setting this to 443 is possible only for backward compatibility reasons and it's discouraged. + Doing so, it would result in the controller overriding the KAS endpoint in the guest cluster having a discrepancy with the KAS Pod and potentially causing temporarily network failures. + This value is immutable. + format: int32 + type: integer + x-kubernetes-validations: + - message: port is immutable + rule: self == oldSelf + type: object + clusterNetwork: + default: + - cidr: 10.132.0.0/14 + description: |- + clusterNetwork is the list of IP address pools for pods. + Defaults to cidr: "10.132.0.0/14". + Currently only one entry is supported. + This field is immutable. + items: + description: |- + ClusterNetworkEntry is a single IP address block for pod IP blocks. IP blocks + are allocated with size 2^HostSubnetLength. + properties: + cidr: + description: cidr is the IP block address pool. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') + hostPrefix: + description: |- + hostPrefix is the prefix size to allocate to each node from the CIDR. + For example, 24 would allocate 2^(32-24)=2^8=256 addresses to each node. If this + field is not used by the plugin, it can be left unset. + format: int32 + type: integer + required: + - cidr + type: object + maxItems: 2 + minItems: 1 + type: array + x-kubernetes-validations: + - message: clusterNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf + machineNetwork: + description: |- + machineNetwork is the list of IP address pools for machines. + This might be used among other things to generate appropriate networking security groups in some clouds providers. + Currently only one entry or two for dual stack is supported. + This field is immutable. + items: + description: MachineNetworkEntry is a single IP address block + for node IP blocks. + properties: + cidr: + description: CIDR is the IP block address pool for machines + within the cluster. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') + required: + - cidr + type: object + maxItems: 2 + minItems: 1 + type: array + x-kubernetes-validations: + - message: machineNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf + networkType: + default: OVNKubernetes + description: |- + networkType specifies the SDN provider used for cluster networking. + Defaults to OVNKubernetes. + This field is required and immutable. + kubebuilder:validation:XValidation:rule="self == oldSelf", message="networkType is immutable" + enum: + - OpenShiftSDN + - Calico + - OVNKubernetes + - Other + type: string + serviceNetwork: + default: + - cidr: 172.31.0.0/16 + description: |- + serviceNetwork is the list of IP address pools for services. + Defaults to cidr: "172.31.0.0/16". + Currently only one entry is supported. + This field is immutable. + items: + description: ServiceNetworkEntry is a single IP address block + for the service network. + properties: + cidr: + description: cidr is the IP block address pool for services + within the cluster in CIDR format (e.g., 192.168.1.0/24 + or 2001:0db8::/64) + maxLength: 43 + type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') + required: + - cidr + type: object + maxItems: 2 + minItems: 1 + type: array + x-kubernetes-validations: + - message: serviceNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf + type: object + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector when specified, is propagated to all control plane Deployments and Stateful sets running management side. + It must be satisfied by the management Nodes for the pods to be scheduled. Otherwise the HostedCluster will enter a degraded state. + Changes to this field will propagate to existing Deployments and StatefulSets. + type: object + x-kubernetes-validations: + - message: nodeSelector map can have at most 20 entries + rule: size(self) <= 20 + olmCatalogPlacement: + default: management + description: |- + OLMCatalogPlacement specifies the placement of OLM catalog components. By default, + this is set to management and OLM catalog components are deployed onto the management + cluster. If set to guest, the OLM catalog components will be deployed onto the guest + cluster. + enum: + - management + - guest + type: string + x-kubernetes-validations: + - message: OLMCatalogPlacement is immutable + rule: self == oldSelf + pausedUntil: + description: |- + pausedUntil is a field that can be used to pause reconciliation on the HostedCluster controller, resulting in any change to the HostedCluster being ignored. + Either a date can be provided in RFC3339 format or a boolean as in 'true', 'false', 'True', 'False'. If a date is + provided: reconciliation is paused on the resource until that date. If the boolean true is + provided: reconciliation is paused on the resource until the field is removed. + maxLength: 35 + minLength: 4 + type: string + x-kubernetes-validations: + - message: PausedUntil must be a date in RFC3339 format or 'True', + 'true', 'False' or 'false' + rule: self.matches('^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.*$') + || self in ['true', 'false', 'True', 'False'] + platform: + description: |- + platform specifies the underlying infrastructure provider for the cluster + and is used to configure platform specific behavior. + properties: + agent: + description: Agent specifies configuration for agent-based installations. + properties: + agentNamespace: + description: AgentNamespace is the namespace where to search + for Agents for this cluster + type: string + required: + - agentNamespace + type: object + aws: + description: AWS specifies configuration for clusters running + on Amazon Web Services. + properties: + additionalAllowedPrincipals: + description: |- + AdditionalAllowedPrincipals specifies a list of additional allowed principal ARNs + to be added to the hosted control plane's VPC Endpoint Service to enable additional + VPC Endpoint connection requests to be automatically accepted. + See https://docs.aws.amazon.com/vpc/latest/privatelink/configure-endpoint-service.html + for more details around VPC Endpoint Service allowed principals. + items: + type: string + type: array + cloudProviderConfig: + description: |- + CloudProviderConfig specifies AWS networking configuration for the control + plane. + This is mainly used for cloud provider controller config: + https://github.com/kubernetes/kubernetes/blob/f5be5052e3d0808abb904aebd3218fe4a5c2dd82/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go#L1347-L1364 + properties: + subnet: + description: Subnet is the subnet to use for control plane + cloud resources. + properties: + filters: + description: |- + Filters is a set of key/value pairs used to identify a resource + They are applied according to the rules defined by the AWS API: + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html + items: + description: Filter is a filter used to identify + an AWS resource + properties: + name: + description: Name of the filter. Filter names + are case-sensitive. + type: string + values: + description: Values includes one or more filter + values. Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + vpc: + description: VPC is the VPC to use for control plane cloud + resources. + type: string + zone: + description: |- + Zone is the availability zone where control plane cloud resources are + created. + type: string + required: + - vpc + type: object + endpointAccess: + default: Public + description: |- + EndpointAccess specifies the publishing scope of cluster endpoints. The + default is Public. + enum: + - Public + - PublicAndPrivate + - Private + type: string + multiArch: + default: false + description: |- + MultiArch specifies whether the Hosted Cluster will be expected to support NodePools with different + CPU architectures, i.e., supporting arm64 NodePools and supporting amd64 NodePools on the same Hosted Cluster. + Deprecated: This field is no longer used. The HyperShift Operator now performs multi-arch validations + automatically despite the platform type. The HyperShift Operator will set HostedCluster.Status.PayloadArch based + on the HostedCluster release image. This field is used by the NodePool controller to validate the + NodePool.Spec.Arch is supported. + type: boolean + region: + description: |- + Region is the AWS region in which the cluster resides. This configures the + OCP control plane cloud integrations, and is used by NodePool to resolve + the correct boot AMI for a given release. + type: string + resourceTags: + description: |- + ResourceTags is a list of additional tags to apply to AWS resources created + for the cluster. See + https://docs.aws.amazon.com/general/latest/gr/aws_tagging.html for + information on tagging AWS resources. AWS supports a maximum of 50 tags per + resource. OpenShift reserves 25 tags for its use, leaving 25 tags available + for the user. + items: + description: AWSResourceTag is a tag to apply to AWS resources + created for the cluster. + properties: + key: + description: Key is the key of the tag. + maxLength: 128 + minLength: 1 + pattern: ^[0-9A-Za-z_.:/=+-@]+$ + type: string + value: + description: |- + Value is the value of the tag. + + Some AWS service do not support empty values. Since tags are added to + resources in many services, the length of the tag value must meet the + requirements of all services. + maxLength: 256 + minLength: 1 + pattern: ^[0-9A-Za-z_.:/=+-@]+$ + type: string + required: + - key + - value + type: object + maxItems: 25 + type: array + rolesRef: + description: |- + RolesRef contains references to various AWS IAM roles required to enable + integrations such as OIDC. + properties: + controlPlaneOperatorARN: + description: "ControlPlaneOperatorARN is an ARN value + referencing a role appropriate for the Control Plane + Operator.\n\nThe following is an example of a valid + policy document:\n\n{\n\t\"Version\": \"2012-10-17\",\n\t\"Statement\": + [\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": + [\n\t\t\t\t\"ec2:CreateVpcEndpoint\",\n\t\t\t\t\"ec2:DescribeVpcEndpoints\",\n\t\t\t\t\"ec2:ModifyVpcEndpoint\",\n\t\t\t\t\"ec2:DeleteVpcEndpoints\",\n\t\t\t\t\"ec2:CreateTags\",\n\t\t\t\t\"route53:ListHostedZones\",\n\t\t\t\t\"ec2:CreateSecurityGroup\",\n\t\t\t\t\"ec2:AuthorizeSecurityGroupIngress\",\n\t\t\t\t\"ec2:AuthorizeSecurityGroupEgress\",\n\t\t\t\t\"ec2:DeleteSecurityGroup\",\n\t\t\t\t\"ec2:RevokeSecurityGroupIngress\",\n\t\t\t\t\"ec2:RevokeSecurityGroupEgress\",\n\t\t\t\t\"ec2:DescribeSecurityGroups\",\n\t\t\t\t\"ec2:DescribeVpcs\",\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t},\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": + [\n\t\t\t\t\"route53:ChangeResourceRecordSets\",\n\t\t\t\t\"route53:ListResourceRecordSets\"\n\t\t\t],\n\t\t\t\"Resource\": + \"arn:aws:route53:::%s\"\n\t\t}\n\t]\n}" + type: string + imageRegistryARN: + description: "ImageRegistryARN is an ARN value referencing + a role appropriate for the Image Registry Operator.\n\nThe + following is an example of a valid policy document:\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"s3:CreateBucket\",\n\t\t\t\t\"s3:DeleteBucket\",\n\t\t\t\t\"s3:PutBucketTagging\",\n\t\t\t\t\"s3:GetBucketTagging\",\n\t\t\t\t\"s3:PutBucketPublicAccessBlock\",\n\t\t\t\t\"s3:GetBucketPublicAccessBlock\",\n\t\t\t\t\"s3:PutEncryptionConfiguration\",\n\t\t\t\t\"s3:GetEncryptionConfiguration\",\n\t\t\t\t\"s3:PutLifecycleConfiguration\",\n\t\t\t\t\"s3:GetLifecycleConfiguration\",\n\t\t\t\t\"s3:GetBucketLocation\",\n\t\t\t\t\"s3:ListBucket\",\n\t\t\t\t\"s3:GetObject\",\n\t\t\t\t\"s3:PutObject\",\n\t\t\t\t\"s3:DeleteObject\",\n\t\t\t\t\"s3:ListBucketMultipartUploads\",\n\t\t\t\t\"s3:AbortMultipartUpload\",\n\t\t\t\t\"s3:ListMultipartUploadParts\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t}\n\t]\n}" + type: string + ingressARN: + description: "The referenced role must have a trust relationship + that allows it to be assumed via web identity.\nhttps://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html.\nExample:\n{\n\t\t\"Version\": + \"2012-10-17\",\n\t\t\"Statement\": [\n\t\t\t{\n\t\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\t\"Principal\": {\n\t\t\t\t\t\"Federated\": + \"{{ .ProviderARN }}\"\n\t\t\t\t},\n\t\t\t\t\t\"Action\": + \"sts:AssumeRoleWithWebIdentity\",\n\t\t\t\t\"Condition\": + {\n\t\t\t\t\t\"StringEquals\": {\n\t\t\t\t\t\t\"{{ .ProviderName + }}:sub\": {{ .ServiceAccounts }}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t]\n\t}\n\nIngressARN + is an ARN value referencing a role appropriate for the + Ingress Operator.\n\nThe following is an example of + a valid policy document:\n\n{\n\t\"Version\": \"2012-10-17\",\n\t\"Statement\": + [\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": + [\n\t\t\t\t\"elasticloadbalancing:DescribeLoadBalancers\",\n\t\t\t\t\"tag:GetResources\",\n\t\t\t\t\"route53:ListHostedZones\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t},\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": + [\n\t\t\t\t\"route53:ChangeResourceRecordSets\"\n\t\t\t],\n\t\t\t\"Resource\": + [\n\t\t\t\t\"arn:aws:route53:::PUBLIC_ZONE_ID\",\n\t\t\t\t\"arn:aws:route53:::PRIVATE_ZONE_ID\"\n\t\t\t]\n\t\t}\n\t]\n}" + type: string + kubeCloudControllerARN: + description: |- + KubeCloudControllerARN is an ARN value referencing a role appropriate for the KCM/KCC. + Source: https://cloud-provider-aws.sigs.k8s.io/prerequisites/#iam-policies + + The following is an example of a valid policy document: + + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstances", + "ec2:DescribeImages", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ], + "Effect": "Allow" + } + ] + } + type: string + networkARN: + description: "NetworkARN is an ARN value referencing a + role appropriate for the Network Operator.\n\nThe following + is an example of a valid policy document:\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"ec2:DescribeInstances\",\n + \ \"ec2:DescribeInstanceStatus\",\n \"ec2:DescribeInstanceTypes\",\n + \ \"ec2:UnassignPrivateIpAddresses\",\n \"ec2:AssignPrivateIpAddresses\",\n + \ \"ec2:UnassignIpv6Addresses\",\n \"ec2:AssignIpv6Addresses\",\n + \ \"ec2:DescribeSubnets\",\n \"ec2:DescribeNetworkInterfaces\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t}\n\t]\n}" + type: string + nodePoolManagementARN: + description: "NodePoolManagementARN is an ARN value referencing + a role appropriate for the CAPI Controller.\n\nThe following + is an example of a valid policy document:\n\n{\n \"Version\": + \"2012-10-17\",\n \"Statement\": [\n {\n \"Action\": + [\n \"ec2:AssociateRouteTable\",\n \"ec2:AttachInternetGateway\",\n + \ \"ec2:AuthorizeSecurityGroupIngress\",\n \"ec2:CreateInternetGateway\",\n + \ \"ec2:CreateNatGateway\",\n \"ec2:CreateRoute\",\n + \ \"ec2:CreateRouteTable\",\n \"ec2:CreateSecurityGroup\",\n + \ \"ec2:CreateSubnet\",\n \"ec2:CreateTags\",\n + \ \"ec2:DeleteInternetGateway\",\n \"ec2:DeleteNatGateway\",\n + \ \"ec2:DeleteRouteTable\",\n \"ec2:DeleteSecurityGroup\",\n + \ \"ec2:DeleteSubnet\",\n \"ec2:DeleteTags\",\n + \ \"ec2:DescribeAccountAttributes\",\n \"ec2:DescribeAddresses\",\n + \ \"ec2:DescribeAvailabilityZones\",\n \"ec2:DescribeImages\",\n + \ \"ec2:DescribeInstances\",\n \"ec2:DescribeInternetGateways\",\n + \ \"ec2:DescribeNatGateways\",\n \"ec2:DescribeNetworkInterfaces\",\n + \ \"ec2:DescribeNetworkInterfaceAttribute\",\n + \ \"ec2:DescribeRouteTables\",\n \"ec2:DescribeSecurityGroups\",\n + \ \"ec2:DescribeSubnets\",\n \"ec2:DescribeVpcs\",\n + \ \"ec2:DescribeVpcAttribute\",\n \"ec2:DescribeVolumes\",\n + \ \"ec2:DetachInternetGateway\",\n \"ec2:DisassociateRouteTable\",\n + \ \"ec2:DisassociateAddress\",\n \"ec2:ModifyInstanceAttribute\",\n + \ \"ec2:ModifyNetworkInterfaceAttribute\",\n \"ec2:ModifySubnetAttribute\",\n + \ \"ec2:RevokeSecurityGroupIngress\",\n \"ec2:RunInstances\",\n + \ \"ec2:TerminateInstances\",\n \"tag:GetResources\",\n + \ \"ec2:CreateLaunchTemplate\",\n \"ec2:CreateLaunchTemplateVersion\",\n + \ \"ec2:DescribeLaunchTemplates\",\n \"ec2:DescribeLaunchTemplateVersions\",\n + \ \"ec2:DeleteLaunchTemplate\",\n \"ec2:DeleteLaunchTemplateVersions\"\n + \ ],\n \"Resource\": [\n \"*\"\n ],\n + \ \"Effect\": \"Allow\"\n },\n {\n \"Condition\": + {\n \"StringLike\": {\n \"iam:AWSServiceName\": + \"elasticloadbalancing.amazonaws.com\"\n }\n },\n + \ \"Action\": [\n \"iam:CreateServiceLinkedRole\"\n + \ ],\n \"Resource\": [\n \"arn:*:iam::*:role/aws-service-role/elasticloadbalancing.amazonaws.com/AWSServiceRoleForElasticLoadBalancing\"\n + \ ],\n \"Effect\": \"Allow\"\n },\n {\n \"Action\": + [\n \"iam:PassRole\"\n ],\n \"Resource\": + [\n \"arn:*:iam::*:role/*-worker-role\"\n ],\n + \ \"Effect\": \"Allow\"\n },\n\t {\n\t \t\"Effect\": + \"Allow\",\n\t \t\"Action\": [\n\t \t\t\"kms:Decrypt\",\n\t + \ \t\t\"kms:ReEncrypt\",\n\t \t\t\"kms:GenerateDataKeyWithoutPlainText\",\n\t + \ \t\t\"kms:DescribeKey\"\n\t \t],\n\t \t\"Resource\": + \"*\"\n\t },\n\t {\n\t \t\"Effect\": \"Allow\",\n\t + \ \t\"Action\": [\n\t \t\t\"kms:CreateGrant\"\n\t \t],\n\t + \ \t\"Resource\": \"*\",\n\t \t\"Condition\": {\n\t + \ \t\t\"Bool\": {\n\t \t\t\t\"kms:GrantIsForAWSResource\": + true\n\t \t\t}\n\t \t}\n\t }\n ]\n}" + type: string + storageARN: + description: "StorageARN is an ARN value referencing a + role appropriate for the Storage Operator.\n\nThe following + is an example of a valid policy document:\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"ec2:AttachVolume\",\n\t\t\t\t\"ec2:CreateSnapshot\",\n\t\t\t\t\"ec2:CreateTags\",\n\t\t\t\t\"ec2:CreateVolume\",\n\t\t\t\t\"ec2:DeleteSnapshot\",\n\t\t\t\t\"ec2:DeleteTags\",\n\t\t\t\t\"ec2:DeleteVolume\",\n\t\t\t\t\"ec2:DescribeInstances\",\n\t\t\t\t\"ec2:DescribeSnapshots\",\n\t\t\t\t\"ec2:DescribeTags\",\n\t\t\t\t\"ec2:DescribeVolumes\",\n\t\t\t\t\"ec2:DescribeVolumesModifications\",\n\t\t\t\t\"ec2:DetachVolume\",\n\t\t\t\t\"ec2:ModifyVolume\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t}\n\t]\n}" + type: string + required: + - controlPlaneOperatorARN + - imageRegistryARN + - ingressARN + - kubeCloudControllerARN + - networkARN + - nodePoolManagementARN + - storageARN + type: object + serviceEndpoints: + description: |- + ServiceEndpoints specifies optional custom endpoints which will override + the default service endpoint of specific AWS Services. + + There must be only one ServiceEndpoint for a given service name. + items: + description: |- + AWSServiceEndpoint stores the configuration for services to + override existing defaults of AWS Services. + properties: + name: + description: |- + Name is the name of the AWS service. + This must be provided and cannot be empty. + type: string + url: + description: |- + URL is fully qualified URI with scheme https, that overrides the default generated + endpoint for a client. + This must be provided and cannot be empty. + pattern: ^https:// + type: string + required: + - name + - url + type: object + type: array + sharedVPC: + description: |- + SharedVPC contains fields that must be specified if the HostedCluster must use a VPC that is + created in a different AWS account and is shared with the AWS account where the HostedCluster + will be created. + properties: + localZoneID: + description: |- + LocalZoneID is the ID of the route53 hosted zone for [cluster-name].hypershift.local that is + associated with the HostedCluster's VPC and exists in the VPC owner account. + maxLength: 32 + type: string + rolesRef: + description: |- + RolesRef contains references to roles in the VPC owner account that enable a + HostedCluster on a shared VPC. + properties: + controlPlaneARN: + description: "ControlPlaneARN is an ARN value referencing + the role in the VPC owner account that allows\nthe + control plane operator in the cluster account to + create and manage a VPC endpoint, its\ncorresponding + Security Group, and DNS records in the hypershift + local hosted zone.\n\nThe referenced role must have + a trust relationship that allows it to be assumed + by the\ncontrol plane operator role in the VPC creator + account.\nExample:\n{\n\t \"Version\": \"2012-10-17\",\n\t + \"Statement\": [\n\t \t{\n\t \t\t\"Sid\": \"Statement1\",\n\t + \t\t\"Effect\": \"Allow\",\n\t \t\t\"Principal\": + {\n\t \t\t\t\"AWS\": \"arn:aws:iam::[cluster-creator-account-id]:role/[infra-id]-control-plane-operator\"\n\t + \t\t},\n\t \t\t\"Action\": \"sts:AssumeRole\"\n\t + \t}\n\t ]\n}\n\nThe following is an example of the + policy document for this role.\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"ec2:CreateVpcEndpoint\",\n\t\t\t\t\"ec2:DescribeVpcEndpoints\",\n\t\t\t\t\"ec2:ModifyVpcEndpoint\",\n\t\t\t\t\"ec2:DeleteVpcEndpoints\",\n\t\t\t\t\"ec2:CreateTags\",\n\t\t\t\t\"route53:ListHostedZones\",\n\t\t\t\t\"ec2:CreateSecurityGroup\",\n\t\t\t\t\"ec2:AuthorizeSecurityGroupIngress\",\n\t\t\t\t\"ec2:AuthorizeSecurityGroupEgress\",\n\t\t\t\t\"ec2:DeleteSecurityGroup\",\n\t\t\t\t\"ec2:RevokeSecurityGroupIngress\",\n\t\t\t\t\"ec2:RevokeSecurityGroupEgress\",\n\t\t\t\t\"ec2:DescribeSecurityGroups\",\n\t\t\t\t\"ec2:DescribeVpcs\",\n\t\t\t\t\"route53:ChangeResourceRecordSets\",\n\t\t\t\t\"route53:ListResourceRecordSets\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t}\n\t]\n}" + pattern: ^arn:(aws|aws-cn|aws-us-gov):iam::[0-9]{12}:role\/.*$ + type: string + ingressARN: + description: "IngressARN is an ARN value referencing + the role in the VPC owner account that allows the\ningress + operator in the cluster account to create and manage + records in the private DNS\nhosted zone.\n\nThe + referenced role must have a trust relationship that + allows it to be assumed by the\ningress operator + role in the VPC creator account.\nExample:\n{\n\t + \"Version\": \"2012-10-17\",\n\t \"Statement\": + [\n\t \t{\n\t \t\t\"Sid\": \"Statement1\",\n\t \t\t\"Effect\": + \"Allow\",\n\t \t\t\"Principal\": {\n\t \t\t\t\"AWS\": + \"arn:aws:iam::[cluster-creator-account-id]:role/[infra-id]-openshift-ingress\"\n\t + \t\t},\n\t \t\t\"Action\": \"sts:AssumeRole\"\n\t + \t}\n\t ]\n}\n\nThe following is an example of the + policy document for this role.\n(Based on https://docs.openshift.com/rosa/rosa_install_access_delete_clusters/rosa-shared-vpc-config.html#rosa-sharing-vpc-dns-and-roles_rosa-shared-vpc-config)\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"route53:ListHostedZones\",\n\t\t\t\t\"route53:ListHostedZonesByName\",\n\t\t\t\t\"route53:ChangeTagsForResource\",\n\t\t\t\t\"route53:GetAccountLimit\",\n\t\t\t\t\"route53:GetChange\",\n\t\t\t\t\"route53:GetHostedZone\",\n\t\t\t\t\"route53:ListTagsForResource\",\n\t\t\t\t\"route53:UpdateHostedZoneComment\",\n\t\t\t\t\"tag:GetResources\",\n\t\t\t\t\"tag:UntagResources\"\n\t\t\t\t\"route53:ChangeResourceRecordSets\",\n\t\t\t\t\"route53:ListResourceRecordSets\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t},\n\t]\n}" + pattern: ^arn:(aws|aws-cn|aws-us-gov):iam::[0-9]{12}:role\/.*$ + type: string + required: + - controlPlaneARN + - ingressARN + type: object + required: + - localZoneID + - rolesRef + type: object + required: + - region + - rolesRef + type: object + azure: + description: Azure defines azure specific settings + properties: + cloud: + default: AzurePublicCloud + description: 'Cloud is the cloud environment identifier, valid + values could be found here: https://github.com/Azure/go-autorest/blob/4c0e21ca2bbb3251fe7853e6f9df6397f53dd419/autorest/azure/environments.go#L33' + enum: + - AzurePublicCloud + - AzureUSGovernmentCloud + - AzureChinaCloud + - AzureGermanCloud + - AzureStackCloud + type: string + credentials: + description: |- + Credentials is the object containing existing Azure credentials needed for creating and managing cloud + infrastructure resources. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + location: + description: |- + Location is the Azure region in where all the cloud infrastructure resources will be created. + + Example: eastus + type: string + x-kubernetes-validations: + - message: Location is immutable + rule: self == oldSelf + resourceGroup: + default: default + description: |- + ResourceGroupName is the name of an existing resource group where all cloud resources created by the Hosted + Cluster are to be placed. The resource group is expected to exist under the same subscription as SubscriptionID. + + In ARO HCP, this will be the managed resource group where customer cloud resources will be created. + + Resource group naming requirements can be found here: https://azure.github.io/PSRule.Rules.Azure/en/rules/Azure.ResourceGroup.Name/. + + Example: if your resource group ID is /subscriptions//resourceGroups/, your + ResourceGroupName is . + pattern: ^[a-zA-Z0-9_()\-\.]{1,89}[a-zA-Z0-9_()\-]$ + type: string + x-kubernetes-validations: + - message: ResourceGroupName is immutable + rule: self == oldSelf + securityGroupID: + description: |- + SecurityGroupID is the ID of an existing security group on the SubnetID. This field is provided as part of the + configuration for the Azure cloud provider, aka Azure cloud controller manager (CCM). This security group is + expected to exist under the same subscription as SubscriptionID. + type: string + x-kubernetes-validations: + - message: SecurityGroupID is immutable + rule: self == oldSelf + subnetID: + description: |- + subnetID is the subnet ID of an existing subnet where the nodes in the nodepool will be created. This can be a + different subnet than the one listed in the HostedCluster, HostedCluster.Spec.Platform.Azure.SubnetID, but must + exist in the same network, HostedCluster.Spec.Platform.Azure.VnetID, and must exist under the same subscription ID, + HostedCluster.Spec.Platform.Azure.SubscriptionID. + subnetID is immutable once set. + The subnetID should be in the format `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}`. + The subscriptionId in the encryptionSetID must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + The resourceGroupName should be between 1 and 90 characters, consisting only of alphanumeric characters, hyphens, underscores, periods and parenthesis and must not end with a period (.) character. + The vnetName should be between 2 and 64 characters, consisting only of alphanumeric characters, hyphens, underscores and periods and must not end with either a period (.) or hyphen (-) character. + The subnetName should be between 1 and 80 characters, consisting only of alphanumeric characters, hyphens and underscores and must start with an alphanumeric character and must not end with a period (.) or hyphen (-) character. + maxLength: 355 + minLength: 1 + type: string + x-kubernetes-validations: + - message: encryptionSetID must be in the format `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}` + rule: size(self.split('/')) == 11 && self.matches('^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Network/virtualNetworks/.*/subnets/.*$') + - message: The resourceGroupName should be between 1 and 90 + characters, consisting only of alphanumeric characters, + hyphens, underscores, periods and parenthesis + rule: self.split('/')[4].matches('[a-zA-Z0-9-_\\(\\)\\.]{1,90}') + - message: the resourceGroupName in the subnetID must not + end with a period (.) character + rule: '!self.split(''/'')[4].endsWith(''.'')' + - message: The vnetName should be between 2 and 64 characters, + consisting only of alphanumeric characters, hyphens, underscores + and periods + rule: self.split('/')[8].matches('[a-zA-Z0-9-_\\.]{2,64}') + - message: the vnetName in the subnetID must not end with + either a period (.) or hyphen (-) character + rule: '!self.split(''/'')[8].endsWith(''.'') && !self.split(''/'')[8].endsWith(''-'')' + - message: The subnetName should be between 1 and 80 characters, + consisting only of alphanumeric characters, hyphens and + underscores and must start with an alphanumeric character + rule: self.split('/')[10].matches('[a-zA-Z0-9][a-zA-Z0-9-_\\.]{0,79}') + - message: the subnetName in the subnetID must not end with + a period (.) or hyphen (-) character + rule: '!self.split(''/'')[10].endsWith(''.'') && !self.split(''/'')[10].endsWith(''-'')' + - message: SubnetID is immutable + rule: self == oldSelf + subscriptionID: + description: SubscriptionID is a unique identifier for an + Azure subscription used to manage resources. + type: string + x-kubernetes-validations: + - message: SubscriptionID is immutable + rule: self == oldSelf + vnetID: + description: |- + VnetID is the ID of an existing VNET to use in creating VMs. The VNET can exist in a different resource group + other than the one specified in ResourceGroupName, but it must exist under the same subscription as + SubscriptionID. + + In ARO HCP, this will be the ID of the customer provided VNET. + + Example: /subscriptions//resourceGroups//providers/Microsoft.Network/virtualNetworks/ + type: string + x-kubernetes-validations: + - message: VnetID is immutable + rule: self == oldSelf + required: + - credentials + - location + - resourceGroup + - securityGroupID + - subnetID + - subscriptionID + - vnetID + type: object + ibmcloud: + description: IBMCloud defines IBMCloud specific settings for components + properties: + providerType: + description: ProviderType is a specific supported infrastructure + provider within IBM Cloud. + type: string + type: object + kubevirt: + description: KubeVirt defines KubeVirt specific settings for cluster + components. + properties: + baseDomainPassthrough: + description: |- + BaseDomainPassthrough toggles whether or not an automatically + generated base domain for the guest cluster should be used that + is a subdomain of the management cluster's *.apps DNS. + + For the KubeVirt platform, the basedomain can be autogenerated using + the *.apps domain of the management/infra hosting cluster + This makes the guest cluster's base domain a subdomain of the + hypershift infra/mgmt cluster's base domain. + + Example: + Infra/Mgmt cluster's DNS + Base: example.com + Cluster: mgmt-cluster.example.com + Apps: *.apps.mgmt-cluster.example.com + KubeVirt Guest cluster's DNS + Base: apps.mgmt-cluster.example.com + Cluster: guest.apps.mgmt-cluster.example.com + Apps: *.apps.guest.apps.mgmt-cluster.example.com + + This is possible using OCP wildcard routes + type: boolean + x-kubernetes-validations: + - message: baseDomainPassthrough is immutable + rule: self == oldSelf + credentials: + description: |- + Credentials defines the client credentials used when creating KubeVirt virtual machines. + Defining credentials is only necessary when the KubeVirt virtual machines are being placed + on a cluster separate from the one hosting the Hosted Control Plane components. + + The default behavior when Credentials is not defined is for the KubeVirt VMs to be placed on + the same cluster and namespace as the Hosted Control Plane. + properties: + infraKubeConfigSecret: + description: |- + InfraKubeConfigSecret is a reference to a secret that contains the kubeconfig for the external infra cluster + that will be used to host the KubeVirt virtual machines for this cluster. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + x-kubernetes-validations: + - message: infraKubeConfigSecret is immutable + rule: self == oldSelf + infraNamespace: + description: |- + InfraNamespace defines the namespace on the external infra cluster that is used to host the KubeVirt + virtual machines. This namespace must already exist before creating the HostedCluster and the kubeconfig + referenced in the InfraKubeConfigSecret must have access to manage the required resources within this + namespace. + type: string + x-kubernetes-validations: + - message: infraNamespace is immutable + rule: self == oldSelf + required: + - infraKubeConfigSecret + - infraNamespace + type: object + generateID: + description: |- + GenerateID is used to uniquely apply a name suffix to resources associated with + kubevirt infrastructure resources + maxLength: 11 + type: string + x-kubernetes-validations: + - message: Kubevirt GenerateID is immutable once set + rule: self == oldSelf + storageDriver: + description: |- + StorageDriver defines how the KubeVirt CSI driver exposes StorageClasses on + the infra cluster (hosting the VMs) to the guest cluster. + properties: + manual: + description: |- + Manual is used to explicitly define how the infra storageclasses are + mapped to guest storageclasses + properties: + storageClassMapping: + description: |- + StorageClassMapping maps StorageClasses on the infra cluster hosting + the KubeVirt VMs to StorageClasses that are made available within the + Guest Cluster. + + NOTE: It is possible that not all capabilities of an infra cluster's + storageclass will be present for the corresponding guest clusters storageclass. + items: + properties: + group: + description: Group contains which group this + mapping belongs to. + type: string + guestStorageClassName: + description: |- + GuestStorageClassName is the name that the corresponding storageclass will + be called within the guest cluster + type: string + infraStorageClassName: + description: |- + InfraStorageClassName is the name of the infra cluster storage class that + will be exposed to the guest. + type: string + required: + - guestStorageClassName + - infraStorageClassName + type: object + type: array + x-kubernetes-validations: + - message: storageClassMapping is immutable + rule: self == oldSelf + volumeSnapshotClassMapping: + items: + properties: + group: + description: Group contains which group this + mapping belongs to. + type: string + guestVolumeSnapshotClassName: + description: |- + GuestVolumeSnapshotClassName is the name that the corresponding volumeSnapshotClass will + be called within the guest cluster + type: string + infraVolumeSnapshotClassName: + description: |- + InfraStorageClassName is the name of the infra cluster volume snapshot class that + will be exposed to the guest. + type: string + required: + - guestVolumeSnapshotClassName + - infraVolumeSnapshotClassName + type: object + type: array + x-kubernetes-validations: + - message: volumeSnapshotClassMapping is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: storageDriver.Manual is immutable + rule: self == oldSelf + type: + default: Default + description: Type represents the type of kubevirt csi + driver configuration to use + enum: + - None + - Default + - Manual + type: string + x-kubernetes-validations: + - message: storageDriver.Type is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: storageDriver is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: Kubevirt GenerateID is required once set + rule: '!has(oldSelf.generateID) || has(self.generateID)' + powervs: + description: |- + PowerVS specifies configuration for clusters running on IBMCloud Power VS Service. + This field is immutable. Once set, It can't be changed. + properties: + accountID: + description: |- + AccountID is the IBMCloud account id. + This field is immutable. Once set, It can't be changed. + type: string + cisInstanceCRN: + description: |- + CISInstanceCRN is the IBMCloud CIS Service Instance's Cloud Resource Name + This field is immutable. Once set, It can't be changed. + pattern: '^crn:' + type: string + imageRegistryOperatorCloudCreds: + description: |- + ImageRegistryOperatorCloudCreds is a reference to a secret containing ibm cloud + credentials for image registry operator to get authenticated with ibm cloud. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + ingressOperatorCloudCreds: + description: |- + IngressOperatorCloudCreds is a reference to a secret containing ibm cloud + credentials for ingress operator to get authenticated with ibm cloud. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + kubeCloudControllerCreds: + description: | + KubeCloudControllerCreds is a reference to a secret containing cloud + credentials with permissions matching the cloud controller policy. + This field is immutable. Once set, It can't be changed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + nodePoolManagementCreds: + description: | + NodePoolManagementCreds is a reference to a secret containing cloud + credentials with permissions matching the node pool management policy. + This field is immutable. Once set, It can't be changed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + region: + description: |- + Region is the IBMCloud region in which the cluster resides. This configures the + OCP control plane cloud integrations, and is used by NodePool to resolve + the correct boot image for a given release. + This field is immutable. Once set, It can't be changed. + type: string + resourceGroup: + description: |- + ResourceGroup is the IBMCloud Resource Group in which the cluster resides. + This field is immutable. Once set, It can't be changed. + type: string + serviceInstanceID: + description: |- + ServiceInstance is the reference to the Power VS service on which the server instance(VM) will be created. + Power VS service is a container for all Power VS instances at a specific geographic region. + serviceInstance can be created via IBM Cloud catalog or CLI. + ServiceInstanceID is the unique identifier that can be obtained from IBM Cloud UI or IBM Cloud cli. + + More detail about Power VS service instance. + https://cloud.ibm.com/docs/power-iaas?topic=power-iaas-creating-power-virtual-server + + This field is immutable. Once set, It can't be changed. + type: string + storageOperatorCloudCreds: + description: |- + StorageOperatorCloudCreds is a reference to a secret containing ibm cloud + credentials for storage operator to get authenticated with ibm cloud. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + subnet: + description: |- + Subnet is the subnet to use for control plane cloud resources. + This field is immutable. Once set, It can't be changed. + properties: + id: + description: ID of resource + type: string + name: + description: Name of resource + type: string + type: object + vpc: + description: |- + VPC specifies IBM Cloud PowerVS Load Balancing configuration for the control + plane. + This field is immutable. Once set, It can't be changed. + properties: + name: + description: |- + Name for VPC to used for all the service load balancer. + This field is immutable. Once set, It can't be changed. + type: string + region: + description: |- + Region is the IBMCloud region in which VPC gets created, this VPC used for all the ingress traffic + into the OCP cluster. + This field is immutable. Once set, It can't be changed. + type: string + subnet: + description: |- + Subnet is the subnet to use for load balancer. + This field is immutable. Once set, It can't be changed. + type: string + zone: + description: |- + Zone is the availability zone where load balancer cloud resources are + created. + This field is immutable. Once set, It can't be changed. + type: string + required: + - name + - region + type: object + zone: + description: |- + Zone is the availability zone where control plane cloud resources are + created. + This field is immutable. Once set, It can't be changed. + type: string + required: + - accountID + - cisInstanceCRN + - imageRegistryOperatorCloudCreds + - ingressOperatorCloudCreds + - kubeCloudControllerCreds + - nodePoolManagementCreds + - region + - resourceGroup + - serviceInstanceID + - storageOperatorCloudCreds + - subnet + - vpc + - zone + type: object + type: + description: Type is the type of infrastructure provider for the + cluster. + type: string + x-kubernetes-validations: + - message: Type is immutable + rule: self == oldSelf + required: + - type + type: object + pullSecret: + description: |- + pullSecret is a local reference to a Secret that must have a ".dockerconfigjson" key whose content must be a valid Openshift pull secret JSON. + If the reference is set but none of the above requirements are met, the HostedCluster will enter a degraded state. + This pull secret will be part of every payload generated by the controllers for any NodePool of the HostedCluster + and it will be injected into the container runtime of all NodePools. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + Changing the content of the secret inplace will not trigger a rollout and might result in unpredictable behaviour. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + release: + description: |- + release specifies the desired OCP release payload for all the hosted cluster components. + This includes those components running management side like the Kube API Server and the CVO but also the operands which land in the hosted cluster data plane like the ingress controller, ovn agents, etc. + The maximum and minimum supported release versions are determined by the running Hypersfhit Operator. + Attempting to use an unsupported version will result in the HostedCluster being degraded and the validateReleaseImage condition being false. + Attempting to use a release with a skew against a NodePool release bigger than N-2 for the y-stream will result in leaving the NodePool in an unsupported state. + Changing this field will trigger a rollout of the control plane components. + The behavior of the rollout will be driven by the ControllerAvailabilityPolicy and InfrastructureAvailabilityPolicy for PDBs and maxUnavailable and surce policies. + properties: + image: + description: |- + Image is the image pullspec of an OCP release payload image. + See https://quay.io/repository/openshift-release-dev/ocp-release?tab=tags for a list of available images. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: Image must start with a word character (letters, digits, + or underscores) and contain no white spaces + rule: self.matches('^(\\w+\\S+)$') + required: + - image + type: object + secretEncryption: + description: |- + secretEncryption specifies a Kubernetes secret encryption strategy for the + control plane. + properties: + aescbc: + description: AESCBC defines metadata about the AESCBC secret encryption + strategy + properties: + activeKey: + description: ActiveKey defines the active key used to encrypt + new secrets + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + backupKey: + description: |- + BackupKey defines the old key during the rotation process so previously created + secrets can continue to be decrypted until they are all re-encrypted with the active key. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - activeKey + type: object + kms: + description: KMS defines metadata about the kms secret encryption + strategy + properties: + aws: + description: AWS defines metadata about the configuration + of the AWS KMS Secret Encryption provider + properties: + activeKey: + description: ActiveKey defines the active key used to + encrypt new secrets + properties: + arn: + description: ARN is the Amazon Resource Name for the + encryption key + pattern: '^arn:' + type: string + required: + - arn + type: object + auth: + description: Auth defines metadata about the management + of credentials used to interact with AWS KMS + properties: + awsKms: + description: "The referenced role must have a trust + relationship that allows it to be assumed via web + identity.\nhttps://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html.\nExample:\n{\n\t\t\"Version\": + \"2012-10-17\",\n\t\t\"Statement\": [\n\t\t\t{\n\t\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\t\"Principal\": {\n\t\t\t\t\t\"Federated\": + \"{{ .ProviderARN }}\"\n\t\t\t\t},\n\t\t\t\t\t\"Action\": + \"sts:AssumeRoleWithWebIdentity\",\n\t\t\t\t\"Condition\": + {\n\t\t\t\t\t\"StringEquals\": {\n\t\t\t\t\t\t\"{{ + .ProviderName }}:sub\": {{ .ServiceAccounts }}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t]\n\t}\n\nAWSKMSARN + is an ARN value referencing a role appropriate for + managing the auth via the AWS KMS key.\n\nThe following + is an example of a valid policy document:\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n \t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"kms:Encrypt\",\n\t\t\t\t\"kms:Decrypt\",\n\t\t\t\t\"kms:ReEncrypt*\",\n\t\t\t\t\"kms:GenerateDataKey*\",\n\t\t\t\t\"kms:DescribeKey\"\n\t\t\t],\n\t\t\t\"Resource\": + %q\n\t\t}\n\t]\n}" + type: string + required: + - awsKms + type: object + backupKey: + description: |- + BackupKey defines the old key during the rotation process so previously created + secrets can continue to be decrypted until they are all re-encrypted with the active key. + properties: + arn: + description: ARN is the Amazon Resource Name for the + encryption key + pattern: '^arn:' + type: string + required: + - arn + type: object + region: + description: Region contains the AWS region + type: string + required: + - activeKey + - auth + - region + type: object + azure: + description: Azure defines metadata about the configuration + of the Azure KMS Secret Encryption provider using Azure + key vault + properties: + activeKey: + description: ActiveKey defines the active key used to + encrypt new secrets + properties: + keyName: + description: KeyName is the name of the keyvault key + used for encrypt/decrypt + type: string + keyVaultName: + description: |- + KeyVaultName is the name of the keyvault. Must match criteria specified at https://docs.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name + Your Microsoft Entra application used to create the cluster must be authorized to access this keyvault, e.g using the AzureCLI: + `az keyvault set-policy -n $KEYVAULT_NAME --key-permissions decrypt encrypt --spn ` + type: string + keyVersion: + description: KeyVersion contains the version of the + key to use + type: string + required: + - keyName + - keyVaultName + - keyVersion + type: object + backupKey: + description: |- + BackupKey defines the old key during the rotation process so previously created + secrets can continue to be decrypted until they are all re-encrypted with the active key. + properties: + keyName: + description: KeyName is the name of the keyvault key + used for encrypt/decrypt + type: string + keyVaultName: + description: |- + KeyVaultName is the name of the keyvault. Must match criteria specified at https://docs.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name + Your Microsoft Entra application used to create the cluster must be authorized to access this keyvault, e.g using the AzureCLI: + `az keyvault set-policy -n $KEYVAULT_NAME --key-permissions decrypt encrypt --spn ` + type: string + keyVersion: + description: KeyVersion contains the version of the + key to use + type: string + required: + - keyName + - keyVaultName + - keyVersion + type: object + required: + - activeKey + type: object + ibmcloud: + description: IBMCloud defines metadata for the IBM Cloud KMS + encryption strategy + properties: + auth: + description: Auth defines metadata for how authentication + is done with IBM Cloud KMS + properties: + managed: + description: |- + Managed defines metadata around the service to service authentication strategy for the IBM Cloud + KMS system (all provider managed). + type: object + type: + description: Type defines the IBM Cloud KMS authentication + strategy + enum: + - Managed + - Unmanaged + type: string + unmanaged: + description: Unmanaged defines the auth metadata the + customer provides to interact with IBM Cloud KMS + properties: + credentials: + description: |- + Credentials should reference a secret with a key field of IBMCloudIAMAPIKeySecretKey that contains a apikey to + call IBM Cloud KMS APIs + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - credentials + type: object + required: + - type + type: object + keyList: + description: KeyList defines the list of keys used for + data encryption + items: + description: IBMCloudKMSKeyEntry defines metadata for + an IBM Cloud KMS encryption key + properties: + correlationID: + description: CorrelationID is an identifier used + to track all api call usage from hypershift + type: string + crkID: + description: CRKID is the customer rook key id + type: string + instanceID: + description: InstanceID is the id for the key protect + instance + type: string + keyVersion: + description: |- + KeyVersion is a unique number associated with the key. The number increments whenever a new + key is enabled for data encryption. + type: integer + url: + description: URL is the url to call key protect + apis over + pattern: ^https:// + type: string + required: + - correlationID + - crkID + - instanceID + - keyVersion + - url + type: object + type: array + region: + description: Region is the IBM Cloud region + type: string + required: + - auth + - keyList + - region + type: object + provider: + description: Provider defines the KMS provider + enum: + - IBMCloud + - AWS + - Azure + type: string + required: + - provider + type: object + type: + description: Type defines the type of kube secret encryption being + used + enum: + - kms + - aescbc + type: string + required: + - type + type: object + serviceAccountSigningKey: + description: |- + serviceAccountSigningKey is a local reference to a secret that must have a "key" key whose content must be the private key + used by the service account token issuer. + If not specified, a service account signing key will + be generated automatically for the cluster. + When specifying a service account signing key, an IssuerURL must also be specified. + If the reference is set but none of the above requirements are met, the HostedCluster will enter a degraded state. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + services: + description: |- + services specifies how individual control plane services endpoints are published for consumption. + This requires APIServer;OAuthServer;Konnectivity;Ignition. + This field is immutable for all platforms but IBMCloud. + Max is 6 to account for OIDC;OVNSbDb for backward compatibility though they are no-op. + + -kubebuilder:validation:XValidation:rule="self.all(s, !(s.service == 'APIServer' && s.servicePublishingStrategy.type == 'Route') || has(s.servicePublishingStrategy.route.hostname))",message="If serviceType is 'APIServer' and publishing strategy is 'Route', then hostname must be set" + -kubebuilder:validation:XValidation:rule="['APIServer', 'OAuthServer', 'Konnectivity', 'Ignition'].all(requiredType, self.exists(s, s.service == requiredType))",message="Services list must contain at least 'APIServer', 'OAuthServer', 'Konnectivity', and 'Ignition' service types" + -kubebuilder:validation:XValidation:rule="self.filter(s, s.servicePublishingStrategy.type == 'Route' && has(s.servicePublishingStrategy.route) && has(s.servicePublishingStrategy.route.hostname)).all(x, self.filter(y, y.servicePublishingStrategy.type == 'Route' && (has(y.servicePublishingStrategy.route) && has(y.servicePublishingStrategy.route.hostname) && y.servicePublishingStrategy.route.hostname == x.servicePublishingStrategy.route.hostname)).size() <= 1)",message="Each route publishingStrategy 'hostname' must be unique within the Services list." + -kubebuilder:validation:XValidation:rule="self.filter(s, s.servicePublishingStrategy.type == 'NodePort' && has(s.servicePublishingStrategy.nodePort) && has(s.servicePublishingStrategy.nodePort.address) && has(s.servicePublishingStrategy.nodePort.port)).all(x, self.filter(y, y.servicePublishingStrategy.type == 'NodePort' && (has(y.servicePublishingStrategy.nodePort) && has(y.servicePublishingStrategy.nodePort.address) && y.servicePublishingStrategy.nodePort.address == x.servicePublishingStrategy.nodePort.address && has(y.servicePublishingStrategy.nodePort.port) && y.servicePublishingStrategy.nodePort.port == x.servicePublishingStrategy.nodePort.port )).size() <= 1)",message="Each nodePort publishingStrategy 'nodePort' and 'hostname' must be unique within the Services list." + items: + description: |- + ServicePublishingStrategyMapping specifies how individual control plane services endpoints are published for consumption. + This includes APIServer;OAuthServer;Konnectivity;Ignition. + If a given service is not present in this list, it will be exposed publicly by default. + properties: + service: + description: |- + service identifies the type of service being published. + It can be APIServer;OAuthServer;Konnectivity;Ignition + OVNSbDb;OIDC are no-op and kept for backward compatibility. + This field is immutable. + enum: + - APIServer + - OAuthServer + - OIDC + - Konnectivity + - Ignition + - OVNSbDb + type: string + servicePublishingStrategy: + description: servicePublishingStrategy specifies how to publish + a service endpoint. + properties: + loadBalancer: + description: loadBalancer configures exposing a service + using a dedicated LoadBalancer. + properties: + hostname: + description: |- + hostname is the name of the DNS record that will be created pointing to the LoadBalancer and passed through to consumers of the service. + If omitted, the value will be inferred from the corev1.Service Load balancer type .status. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: hostname must be a valid domain name (e.g., + example.com) + rule: self.matches('^(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}$') + type: object + nodePort: + description: nodePort configures exposing a service using + a NodePort. + properties: + address: + description: address is the host/ip that the NodePort + service is exposed over. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: address must be a valid hostname, IPv4, or + IPv6 address + rule: isIP(self) || self.matches('^(([a-zA-Z0-9][-a-zA-Z0-9]*\\.)+[a-zA-Z]{2,}|localhost)$') + || self.matches('^((\\d{1,3}\\.){3}\\d{1,3})$') + || self.matches('^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$') + port: + description: |- + port is the port of the NodePort service. If <=0, the port is dynamically + assigned when the service is created. + format: int32 + type: integer + required: + - address + type: object + route: + description: |- + route configures exposing a service using a Route through and an ingress controller behind a cloud Load Balancer. + The specifics of the setup are platform dependent. + properties: + hostname: + description: |- + Hostname is the name of the DNS record that will be created pointing to the Route and passed through to consumers of the service. + If omitted, the value will be inferred from management ingress.Spec.Domain. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: hostname must be a valid domain name (e.g., + example.com) + rule: self.matches('^(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}$') + type: object + type: + description: |- + type is the publishing strategy used for the service. + It can be LoadBalancer;NodePort;Route;None;S3 + enum: + - LoadBalancer + - NodePort + - Route + - None + - S3 + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: nodePort is required when type is NodePort, and forbidden + otherwise + rule: 'self.type == ''NodePort'' ? has(self.nodePort) : !has(self.nodePort)' + - message: only route is allowed when type is Route, and forbidden + otherwise + rule: 'self.type == ''Route'' ? !has(self.nodePort) && !has(self.loadBalancer) + : !has(self.route)' + - message: only loadBalancer is required when type is LoadBalancer, + and forbidden otherwise + rule: 'self.type == ''LoadBalancer'' ? !has(self.nodePort) + && !has(self.route) : !has(self.loadBalancer)' + - message: None does not allowed any configuration for loadBalancer, + nodePort, or route + rule: 'self.type == ''None'' ? !has(self.nodePort) && !has(self.route) + && !has(self.loadBalancer) : true' + - message: S3 does not allowed any configuration for loadBalancer, + nodePort, or route + rule: 'self.type == ''S3'' ? !has(self.nodePort) && !has(self.route) + && !has(self.loadBalancer) : true' + required: + - service + - servicePublishingStrategy + type: object + maxItems: 6 + minItems: 4 + type: array + sshKey: + description: |- + sshKey is a local reference to a Secret that must have a "id_rsa.pub" key whose content must be the public part of 1..N SSH keys. + If the reference is set but none of the above requirements are met, the HostedCluster will enter a degraded state. + When sshKey is set, the controllers will generate a machineConfig with the sshAuthorizedKeys https://coreos.github.io/ignition/configuration-v3_2/ populated with this value. + This MachineConfig will be part of every payload generated by the controllers for any NodePool of the HostedCluster. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + tolerations: + description: Tolerations when specified, define what custom tolerations + are added to the hcp pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + updateService: + description: |- + updateService may be used to specify the preferred upstream update service. + If omitted we will use the appropriate update service for the cluster and region. + This is used by the control plane operator to determine and signal the appropriate available upgrades in the hostedCluster.status. + type: string + x-kubernetes-validations: + - message: updateService must be a valid absolute URL + rule: isURL(self) + required: + - etcd + - networking + - platform + - pullSecret + - release + - services + type: object + x-kubernetes-validations: + - message: Services is immutable. Changes might result in unpredictable + and disruptive behavior. + rule: 'self.platform.type != "IBMCloud" ? self.services == oldSelf.services + : true' + - message: Azure platform requires APIServer Route service with a hostname + to be defined + rule: 'self.platform.type == "Azure" ? self.services.exists(s, s.service + == "APIServer" && s.servicePublishingStrategy.type == "Route" && s.servicePublishingStrategy.route.hostname + != "") : true' + - message: Azure platform requires OAuthServer Route service with a hostname + to be defined + rule: 'self.platform.type == "Azure" ? self.services.exists(s, s.service + == "OAuthServer" && s.servicePublishingStrategy.type == "Route" && + s.servicePublishingStrategy.route.hostname != "") : true' + - message: Azure platform requires Konnectivity Route service with a hostname + to be defined + rule: 'self.platform.type == "Azure" ? self.services.exists(s, s.service + == "Konnectivity" && s.servicePublishingStrategy.type == "Route" && + s.servicePublishingStrategy.route.hostname != "") : true' + - message: Azure platform requires Ignition Route service with a hostname + to be defined + rule: 'self.platform.type == "Azure" ? self.services.exists(s, s.service + == "Ignition" && s.servicePublishingStrategy.type == "Route" && s.servicePublishingStrategy.route.hostname + != "") : true' + - message: If serviceAccountSigningKey is set, issuerURL must be set + rule: has(self.issuerURL) || !has(self.serviceAccountSigningKey) + - message: CIDR ranges in machineNetwork, clusterNetwork, and serviceNetwork + must be unique and non-overlapping + rule: (self.platform.type == "IBMCloud" || !has(self.networking.machineNetwork) + && self.networking.clusterNetwork.all(c, self.networking.serviceNetwork.all(s, + c.cidr != s.cidr)) || (has(self.networking.machineNetwork) && (self.networking.machineNetwork.all(m, + self.networking.clusterNetwork.all(c, m.cidr != c.cidr)) && self.networking.machineNetwork.all(m, + self.networking.serviceNetwork.all(s, m.cidr != s.cidr)) && self.networking.clusterNetwork.all(c, + self.networking.serviceNetwork.all(s, c.cidr != s.cidr))))) + status: + description: Status is the latest observed status of the HostedCluster. + properties: + conditions: + description: |- + Conditions represents the latest available observations of a control + plane's current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controlPlaneEndpoint: + description: |- + ControlPlaneEndpoint contains the endpoint information by which + external clients can access the control plane. This is populated + after the infrastructure is ready. + properties: + host: + description: Host is the hostname on which the API server is serving. + type: string + port: + description: Port is the port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + ignitionEndpoint: + description: |- + IgnitionEndpoint is the endpoint injected in the ign config userdata. + It exposes the config for instances to become kubernetes nodes. + type: string + kubeadminPassword: + description: |- + KubeadminPassword is a reference to the secret that contains the initial + kubeadmin user password for the guest cluster. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + kubeconfig: + description: |- + KubeConfig is a reference to the secret containing the default kubeconfig + for the cluster. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + oauthCallbackURLTemplate: + description: |- + OAuthCallbackURLTemplate contains a template for the URL to use as a callback + for identity providers. The [identity-provider-name] placeholder must be replaced + with the name of an identity provider defined on the HostedCluster. + This is populated after the infrastructure is ready. + type: string + payloadArch: + description: |- + payloadArch represents the CPU architecture type of the HostedCluster.Spec.Release.Image. The valid values are: + Multi, ARM64, AMD64, S390X, or PPC64LE. + enum: + - Multi + - ARM64 + - AMD64 + - PPC64LE + - S390X + type: string + platform: + description: Platform contains platform-specific status of the HostedCluster + properties: + aws: + description: AWSPlatformStatus contains status specific to the + AWS platform + properties: + defaultWorkerSecurityGroupID: + description: |- + DefaultWorkerSecurityGroupID is the ID of a security group created by + the control plane operator. It is always added to worker machines in + addition to any security groups specified in the NodePool. + type: string + type: object + type: object + version: + description: |- + Version is the status of the release version applied to the + HostedCluster. + properties: + availableUpdates: + description: |- + availableUpdates contains updates recommended for this + cluster. Updates which appear in conditionalUpdates but not in + availableUpdates may expose this cluster to known issues. This list + may be empty if no updates are recommended, if the update service + is unavailable, or if an invalid channel has been specified. + items: + description: Release represents an OpenShift release image and + associated metadata. + properties: + channels: + description: |- + channels is the set of Cincinnati channels to which the release + currently belongs. + items: + type: string + type: array + x-kubernetes-list-type: set + image: + description: |- + image is a container image location that contains the update. When this + field is part of spec, image is optional if version is specified and the + availableUpdates field contains a matching version. + type: string + url: + description: |- + url contains information about this release. This URL is set by + the 'url' metadata property on a release or the metadata returned by + the update API and should be displayed as a link in user + interfaces. The URL field may not be set for test or nightly + releases. + type: string + version: + description: |- + version is a semantic version identifying the update version. When this + field is part of spec, version is optional if image is specified. + type: string + required: + - image + - version + type: object + nullable: true + type: array + conditionalUpdates: + description: |- + conditionalUpdates contains the list of updates that may be + recommended for this cluster if it meets specific required + conditions. Consumers interested in the set of updates that are + actually recommended for this cluster should use + availableUpdates. This list may be empty if no updates are + recommended, if the update service is unavailable, or if an empty + or invalid channel has been specified. + items: + description: |- + ConditionalUpdate represents an update which is recommended to some + clusters on the version the current cluster is reconciling, but which + may not be recommended for the current cluster. + properties: + conditions: + description: |- + conditions represents the observations of the conditional update's + current status. Known types are: + * Recommended, for whether the update is recommended for the current cluster. + items: + description: Condition contains details for one aspect + of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, + False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in + foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + release: + description: release is the target of the update. + properties: + channels: + description: |- + channels is the set of Cincinnati channels to which the release + currently belongs. + items: + type: string + type: array + x-kubernetes-list-type: set + image: + description: |- + image is a container image location that contains the update. When this + field is part of spec, image is optional if version is specified and the + availableUpdates field contains a matching version. + type: string + url: + description: |- + url contains information about this release. This URL is set by + the 'url' metadata property on a release or the metadata returned by + the update API and should be displayed as a link in user + interfaces. The URL field may not be set for test or nightly + releases. + type: string + version: + description: |- + version is a semantic version identifying the update version. When this + field is part of spec, version is optional if image is specified. + type: string + required: + - image + - version + type: object + risks: + description: |- + risks represents the range of issues associated with + updating to the target release. The cluster-version + operator will evaluate all entries, and only recommend the + update if there is at least one entry and all entries + recommend the update. + items: + description: |- + ConditionalUpdateRisk represents a reason and cluster-state + for not recommending a conditional update. + properties: + matchingRules: + description: |- + matchingRules is a slice of conditions for deciding which + clusters match the risk and which do not. The slice is + ordered by decreasing precedence. The cluster-version + operator will walk the slice in order, and stop after the + first it can successfully evaluate. If no condition can be + successfully evaluated, the update will not be recommended. + items: + description: |- + ClusterCondition is a union of typed cluster conditions. The 'type' + property determines which of the type-specific properties are relevant. + When evaluated on a cluster, the condition may match, not match, or + fail to evaluate. + properties: + promql: + description: promQL represents a cluster condition + based on PromQL. + properties: + promql: + description: |- + PromQL is a PromQL query classifying clusters. This query + query should return a 1 in the match case and a 0 in the + does-not-match case. Queries which return no time + series, or which return values besides 0 or 1, are + evaluation failures. + type: string + required: + - promql + type: object + type: + description: |- + type represents the cluster-condition type. This defines + the members and semantics of any additional properties. + enum: + - Always + - PromQL + type: string + required: + - type + type: object + minItems: 1 + type: array + x-kubernetes-list-type: atomic + message: + description: |- + message provides additional information about the risk of + updating, in the event that matchingRules match the cluster + state. This is only to be consumed by humans. It may + contain Line Feed characters (U+000A), which should be + rendered as new lines. + minLength: 1 + type: string + name: + description: |- + name is the CamelCase reason for not recommending a + conditional update, in the event that matchingRules match the + cluster state. + minLength: 1 + type: string + url: + description: url contains information about this risk. + format: uri + minLength: 1 + type: string + required: + - matchingRules + - message + - name + - url + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - release + - risks + type: object + type: array + x-kubernetes-list-type: atomic + desired: + description: |- + desired is the version that the cluster is reconciling towards. + If the cluster is not yet fully initialized desired will be set + with the information available, which may be an image or a tag. + properties: + channels: + description: |- + channels is the set of Cincinnati channels to which the release + currently belongs. + items: + type: string + type: array + x-kubernetes-list-type: set + image: + description: |- + image is a container image location that contains the update. When this + field is part of spec, image is optional if version is specified and the + availableUpdates field contains a matching version. + type: string + url: + description: |- + url contains information about this release. This URL is set by + the 'url' metadata property on a release or the metadata returned by + the update API and should be displayed as a link in user + interfaces. The URL field may not be set for test or nightly + releases. + type: string + version: + description: |- + version is a semantic version identifying the update version. When this + field is part of spec, version is optional if image is specified. + type: string + required: + - image + - version + type: object + history: + description: |- + history contains a list of the most recent versions applied to the cluster. + This value may be empty during cluster startup, and then will be updated + when a new update is being applied. The newest update is first in the + list and it is ordered by recency. Updates in the history have state + Completed if the rollout completed - if an update was failing or halfway + applied the state will be Partial. Only a limited amount of update history + is preserved. + items: + description: UpdateHistory is a single attempted update to the + cluster. + properties: + acceptedRisks: + description: |- + acceptedRisks records risks which were accepted to initiate the update. + For example, it may menition an Upgradeable=False or missing signature + that was overriden via desiredUpdate.force, or an update that was + initiated despite not being in the availableUpdates set of recommended + update targets. + type: string + completionTime: + description: |- + completionTime, if set, is when the update was fully applied. The update + that is currently being applied will have a null completion time. + Completion time will always be set for entries that are not the current + update (usually to the started time of the next update). + format: date-time + nullable: true + type: string + image: + description: |- + image is a container image location that contains the update. This value + is always populated. + type: string + startedTime: + description: startedTime is the time at which the update + was started. + format: date-time + type: string + state: + description: |- + state reflects whether the update was fully applied. The Partial state + indicates the update is not fully applied, while the Completed state + indicates the update was successfully rolled out at least once (all + parts of the update successfully applied). + type: string + verified: + description: |- + verified indicates whether the provided update was properly verified + before it was installed. If this is false the cluster may not be trusted. + Verified does not cover upgradeable checks that depend on the cluster + state at the time when the update target was accepted. + type: boolean + version: + description: |- + version is a semantic version identifying the update version. If the + requested image does not define a version, or if a failure occurs + retrieving the image, this value may be empty. + type: string + required: + - completionTime + - image + - startedTime + - state + - verified + type: object + type: array + observedGeneration: + description: |- + observedGeneration reports which version of the spec is being synced. + If this value is not equal to metadata.generation, then the desired + and conditions fields may represent a previous version. + format: int64 + type: integer + required: + - availableUpdates + - desired + - observedGeneration + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AutoNodeKarpenter.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AutoNodeKarpenter.yaml new file mode 100644 index 0000000000..c9fa1777af --- /dev/null +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AutoNodeKarpenter.yaml @@ -0,0 +1,4516 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + feature-gate.release.openshift.io/AutoNodeKarpenter: "true" + name: hostedcontrolplanes.hypershift.openshift.io +spec: + group: hypershift.openshift.io + names: + categories: + - cluster-api + kind: HostedControlPlane + listKind: HostedControlPlaneList + plural: hostedcontrolplanes + shortNames: + - hcp + - hcps + singular: hostedcontrolplane + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: HostedControlPlane defines the desired state of HostedControlPlane + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: HostedControlPlaneSpec defines the desired state of HostedControlPlane + properties: + additionalTrustBundle: + description: AdditionalTrustBundle references a ConfigMap containing + a PEM-encoded X.509 certificate bundle + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + auditWebhook: + description: |- + AuditWebhook contains metadata for configuring an audit webhook + endpoint for a cluster to process cluster audit events. It references + a secret that contains the webhook information for the audit webhook endpoint. + It is a secret because if the endpoint has MTLS the kubeconfig will contain client + keys. This is currently only supported in IBM Cloud. The kubeconfig needs to be stored + in the secret with a secret key name that corresponds to the constant AuditWebhookKubeconfigKey. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + autoNode: + description: autoNode specifies the configuration for the autoNode + feature. + properties: + provisionerConfig: + description: provisioner is the implementation used for Node auto + provisioning. + properties: + karpenter: + description: karpenter specifies the configuration for the + Karpenter provisioner. + properties: + aws: + description: aws specifies the AWS-specific configuration + for Karpenter. + properties: + roleARN: + description: arn specifies the ARN of the Karpenter + provisioner. + type: string + required: + - roleARN + type: object + platform: + description: platform specifies the platform-specific + configuration for Karpenter. + type: string + required: + - platform + type: object + name: + allOf: + - enum: + - Karpenter + - enum: + - Karpenter + description: name specifies the name of the provisioner to + use. + type: string + required: + - name + type: object + required: + - provisionerConfig + type: object + autoscaling: + description: |- + Autoscaling specifies auto-scaling behavior that applies to all NodePools + associated with the control plane. + properties: + maxNodeProvisionTime: + description: |- + maxNodeProvisionTime is the maximum time to wait for node provisioning + before considering the provisioning to be unsuccessful, expressed as a Go + duration string. The default is 15 minutes. + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + maxNodesTotal: + description: |- + maxNodesTotal is the maximum allowable number of nodes for the Autoscaler scale out to be operational. + The autoscaler will not grow the cluster beyond this number. + If omitted, the autoscaler will not have a maximum limit. + number. + format: int32 + minimum: 0 + type: integer + maxPodGracePeriod: + description: |- + maxPodGracePeriod is the maximum seconds to wait for graceful pod + termination before scaling down a NodePool. The default is 600 seconds. + format: int32 + minimum: 0 + type: integer + podPriorityThreshold: + description: |- + podPriorityThreshold enables users to schedule "best-effort" pods, which + shouldn't trigger autoscaler actions, but only run when there are spare + resources available. The default is -10. + + See the following for more details: + https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#how-does-cluster-autoscaler-work-with-pod-priority-and-preemption + format: int32 + type: integer + type: object + channel: + description: |- + channel is an identifier for explicitly requesting that a non-default + set of updates be applied to this cluster. The default channel will be + contain stable updates that are appropriate for production clusters. + type: string + clusterID: + description: |- + ClusterID is the unique id that identifies the cluster externally. + Making it optional here allows us to keep compatibility with previous + versions of the control-plane-operator that have no knowledge of this + field. + type: string + configuration: + description: |- + Configuration embeds resources that correspond to the openshift configuration API: + https://docs.openshift.com/container-platform/4.7/rest_api/config_apis/config-apis-index.html + properties: + apiServer: + description: |- + APIServer holds configuration (like serving certificates, client CA and CORS domains) + shared by all API servers in the system, among them especially kube-apiserver + and openshift-apiserver. + properties: + additionalCORSAllowedOrigins: + description: |- + additionalCORSAllowedOrigins lists additional, user-defined regular expressions describing hosts for which the + API server allows access using the CORS headers. This may be needed to access the API and the integrated OAuth + server from JavaScript applications. + The values are regular expressions that correspond to the Golang regular expression language. + items: + type: string + type: array + audit: + default: + profile: Default + description: |- + audit specifies the settings for audit configuration to be applied to all OpenShift-provided + API servers in the cluster. + properties: + customRules: + description: |- + customRules specify profiles per group. These profile take precedence over the + top-level profile field if they apply. They are evaluation from top to bottom and + the first one that matches, applies. + items: + description: |- + AuditCustomRule describes a custom rule for an audit profile that takes precedence over + the top-level profile. + properties: + group: + description: group is a name of group a request + user must be member of in order to this profile + to apply. + minLength: 1 + type: string + profile: + description: |- + profile specifies the name of the desired audit policy configuration to be deployed to + all OpenShift-provided API servers in the cluster. + + The following profiles are provided: + - Default: the existing default policy. + - WriteRequestBodies: like 'Default', but logs request and response HTTP payloads for + write requests (create, update, patch). + - AllRequestBodies: like 'WriteRequestBodies', but also logs request and response + HTTP payloads for read requests (get, list). + - None: no requests are logged at all, not even oauthaccesstokens and oauthauthorizetokens. + + If unset, the 'Default' profile is used as the default. + enum: + - Default + - WriteRequestBodies + - AllRequestBodies + - None + type: string + required: + - group + - profile + type: object + type: array + x-kubernetes-list-map-keys: + - group + x-kubernetes-list-type: map + profile: + default: Default + description: |- + profile specifies the name of the desired top-level audit profile to be applied to all requests + sent to any of the OpenShift-provided API servers in the cluster (kube-apiserver, + openshift-apiserver and oauth-apiserver), with the exception of those requests that match + one or more of the customRules. + + The following profiles are provided: + - Default: default policy which means MetaData level logging with the exception of events + (not logged at all), oauthaccesstokens and oauthauthorizetokens (both logged at RequestBody + level). + - WriteRequestBodies: like 'Default', but logs request and response HTTP payloads for + write requests (create, update, patch). + - AllRequestBodies: like 'WriteRequestBodies', but also logs request and response + HTTP payloads for read requests (get, list). + - None: no requests are logged at all, not even oauthaccesstokens and oauthauthorizetokens. + + Warning: It is not recommended to disable audit logging by using the `None` profile unless you + are fully aware of the risks of not logging data that can be beneficial when troubleshooting issues. + If you disable audit logging and a support situation arises, you might need to enable audit logging + and reproduce the issue in order to troubleshoot properly. + + If unset, the 'Default' profile is used as the default. + enum: + - Default + - WriteRequestBodies + - AllRequestBodies + - None + type: string + type: object + clientCA: + description: |- + clientCA references a ConfigMap containing a certificate bundle for the signers that will be recognized for + incoming client certificates in addition to the operator managed signers. If this is empty, then only operator managed signers are valid. + You usually only have to set this if you have your own PKI you wish to honor client certificates from. + The ConfigMap must exist in the openshift-config namespace and contain the following required fields: + - ConfigMap.Data["ca-bundle.crt"] - CA bundle. + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + required: + - name + type: object + encryption: + description: encryption allows the configuration of encryption + of resources at the datastore layer. + properties: + type: + description: |- + type defines what encryption type should be used to encrypt resources at the datastore layer. + When this field is unset (i.e. when it is set to the empty string), identity is implied. + The behavior of unset can and will change over time. Even if encryption is enabled by default, + the meaning of unset may change to a different encryption type based on changes in best practices. + + When encryption is enabled, all sensitive resources shipped with the platform are encrypted. + This list of sensitive resources can and will change over time. The current authoritative list is: + + 1. secrets + 2. configmaps + 3. routes.route.openshift.io + 4. oauthaccesstokens.oauth.openshift.io + 5. oauthauthorizetokens.oauth.openshift.io + enum: + - "" + - identity + - aescbc + - aesgcm + type: string + type: object + servingCerts: + description: |- + servingCert is the TLS cert info for serving secure traffic. If not specified, operator managed certificates + will be used for serving secure traffic. + properties: + namedCertificates: + description: |- + namedCertificates references secrets containing the TLS cert info for serving secure traffic to specific hostnames. + If no named certificates are provided, or no named certificates match the server name as understood by a client, + the defaultServingCertificate will be used. + items: + description: APIServerNamedServingCert maps a server + DNS name, as understood by a client, to a certificate. + properties: + names: + description: |- + names is a optional list of explicit DNS names (leading wildcards allowed) that should use this certificate to + serve secure traffic. If no names are provided, the implicit names will be extracted from the certificates. + Exact names trump over wildcard names. Explicit names defined here trump over extracted implicit names. + items: + type: string + type: array + servingCertificate: + description: |- + servingCertificate references a kubernetes.io/tls type secret containing the TLS cert info for serving secure traffic. + The secret must exist in the openshift-config namespace and contain the following required fields: + - Secret.Data["tls.key"] - TLS private key. + - Secret.Data["tls.crt"] - TLS certificate. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + type: object + type: array + type: object + tlsSecurityProfile: + description: |- + tlsSecurityProfile specifies settings for TLS connections for externally exposed servers. + + If unset, a default (which may change between releases) is chosen. Note that only Old, + Intermediate and Custom profiles are currently supported, and the maximum available + minTLSVersion is VersionTLS12. + properties: + custom: + description: |- + custom is a user-defined TLS security profile. Be extremely careful using a custom + profile as invalid configurations can be catastrophic. An example custom profile + looks like this: + + ciphers: + + - ECDHE-ECDSA-CHACHA20-POLY1305 + + - ECDHE-RSA-CHACHA20-POLY1305 + + - ECDHE-RSA-AES128-GCM-SHA256 + + - ECDHE-ECDSA-AES128-GCM-SHA256 + + minTLSVersion: VersionTLS11 + nullable: true + properties: + ciphers: + description: |- + ciphers is used to specify the cipher algorithms that are negotiated + during the TLS handshake. Operators may remove entries their operands + do not support. For example, to use DES-CBC3-SHA (yaml): + + ciphers: + - DES-CBC3-SHA + items: + type: string + type: array + minTLSVersion: + description: |- + minTLSVersion is used to specify the minimal version of the TLS protocol + that is negotiated during the TLS handshake. For example, to use TLS + versions 1.1, 1.2 and 1.3 (yaml): + + minTLSVersion: VersionTLS11 + + NOTE: currently the highest minTLSVersion allowed is VersionTLS12 + enum: + - VersionTLS10 + - VersionTLS11 + - VersionTLS12 + - VersionTLS13 + type: string + type: object + intermediate: + description: |- + intermediate is a TLS security profile based on: + + https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29 + + and looks like this (yaml): + + ciphers: + + - TLS_AES_128_GCM_SHA256 + + - TLS_AES_256_GCM_SHA384 + + - TLS_CHACHA20_POLY1305_SHA256 + + - ECDHE-ECDSA-AES128-GCM-SHA256 + + - ECDHE-RSA-AES128-GCM-SHA256 + + - ECDHE-ECDSA-AES256-GCM-SHA384 + + - ECDHE-RSA-AES256-GCM-SHA384 + + - ECDHE-ECDSA-CHACHA20-POLY1305 + + - ECDHE-RSA-CHACHA20-POLY1305 + + - DHE-RSA-AES128-GCM-SHA256 + + - DHE-RSA-AES256-GCM-SHA384 + + minTLSVersion: VersionTLS12 + nullable: true + type: object + modern: + description: |- + modern is a TLS security profile based on: + + https://wiki.mozilla.org/Security/Server_Side_TLS#Modern_compatibility + + and looks like this (yaml): + + ciphers: + + - TLS_AES_128_GCM_SHA256 + + - TLS_AES_256_GCM_SHA384 + + - TLS_CHACHA20_POLY1305_SHA256 + + minTLSVersion: VersionTLS13 + nullable: true + type: object + old: + description: |- + old is a TLS security profile based on: + + https://wiki.mozilla.org/Security/Server_Side_TLS#Old_backward_compatibility + + and looks like this (yaml): + + ciphers: + + - TLS_AES_128_GCM_SHA256 + + - TLS_AES_256_GCM_SHA384 + + - TLS_CHACHA20_POLY1305_SHA256 + + - ECDHE-ECDSA-AES128-GCM-SHA256 + + - ECDHE-RSA-AES128-GCM-SHA256 + + - ECDHE-ECDSA-AES256-GCM-SHA384 + + - ECDHE-RSA-AES256-GCM-SHA384 + + - ECDHE-ECDSA-CHACHA20-POLY1305 + + - ECDHE-RSA-CHACHA20-POLY1305 + + - DHE-RSA-AES128-GCM-SHA256 + + - DHE-RSA-AES256-GCM-SHA384 + + - DHE-RSA-CHACHA20-POLY1305 + + - ECDHE-ECDSA-AES128-SHA256 + + - ECDHE-RSA-AES128-SHA256 + + - ECDHE-ECDSA-AES128-SHA + + - ECDHE-RSA-AES128-SHA + + - ECDHE-ECDSA-AES256-SHA384 + + - ECDHE-RSA-AES256-SHA384 + + - ECDHE-ECDSA-AES256-SHA + + - ECDHE-RSA-AES256-SHA + + - DHE-RSA-AES128-SHA256 + + - DHE-RSA-AES256-SHA256 + + - AES128-GCM-SHA256 + + - AES256-GCM-SHA384 + + - AES128-SHA256 + + - AES256-SHA256 + + - AES128-SHA + + - AES256-SHA + + - DES-CBC3-SHA + + minTLSVersion: VersionTLS10 + nullable: true + type: object + type: + description: |- + type is one of Old, Intermediate, Modern or Custom. Custom provides + the ability to specify individual TLS security profile parameters. + Old, Intermediate and Modern are TLS security profiles based on: + + https://wiki.mozilla.org/Security/Server_Side_TLS#Recommended_configurations + + The profiles are intent based, so they may change over time as new ciphers are developed and existing ciphers + are found to be insecure. Depending on precisely which ciphers are available to a process, the list may be + reduced. + + Note that the Modern profile is currently not supported because it is not + yet well adopted by common software libraries. + enum: + - Old + - Intermediate + - Modern + - Custom + type: string + type: object + type: object + authentication: + description: |- + Authentication specifies cluster-wide settings for authentication (like OAuth and + webhook token authenticators). + properties: + oauthMetadata: + description: |- + oauthMetadata contains the discovery endpoint data for OAuth 2.0 + Authorization Server Metadata for an external OAuth server. + This discovery document can be viewed from its served location: + oc get --raw '/.well-known/oauth-authorization-server' + For further details, see the IETF Draft: + https://tools.ietf.org/html/draft-ietf-oauth-discovery-04#section-2 + If oauthMetadata.name is non-empty, this value has precedence + over any metadata reference stored in status. + The key "oauthMetadata" is used to locate the data. + If specified and the config map or expected key is not found, no metadata is served. + If the specified metadata is not valid, no metadata is served. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + required: + - name + type: object + serviceAccountIssuer: + description: |- + serviceAccountIssuer is the identifier of the bound service account token + issuer. + The default is https://kubernetes.default.svc + WARNING: Updating this field will not result in immediate invalidation of all bound tokens with the + previous issuer value. Instead, the tokens issued by previous service account issuer will continue to + be trusted for a time period chosen by the platform (currently set to 24h). + This time period is subject to change over time. + This allows internal components to transition to use new service account issuer without service distruption. + type: string + type: + description: |- + type identifies the cluster managed, user facing authentication mode in use. + Specifically, it manages the component that responds to login attempts. + The default is IntegratedOAuth. + type: string + webhookTokenAuthenticator: + description: |- + webhookTokenAuthenticator configures a remote token reviewer. + These remote authentication webhooks can be used to verify bearer tokens + via the tokenreviews.authentication.k8s.io REST API. This is required to + honor bearer tokens that are provisioned by an external authentication service. + + Can only be set if "Type" is set to "None". + properties: + kubeConfig: + description: |- + kubeConfig references a secret that contains kube config file data which + describes how to access the remote webhook service. + The namespace for the referenced secret is openshift-config. + + For further details, see: + + https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication + + The key "kubeConfig" is used to locate the data. + If the secret or expected key is not found, the webhook is not honored. + If the specified kube config data is not valid, the webhook is not honored. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + required: + - kubeConfig + type: object + webhookTokenAuthenticators: + description: webhookTokenAuthenticators is DEPRECATED, setting + it has no effect. + items: + description: |- + deprecatedWebhookTokenAuthenticator holds the necessary configuration options for a remote token authenticator. + It's the same as WebhookTokenAuthenticator but it's missing the 'required' validation on KubeConfig field. + properties: + kubeConfig: + description: |- + kubeConfig contains kube config file data which describes how to access the remote webhook service. + For further details, see: + https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication + The key "kubeConfig" is used to locate the data. + If the secret or expected key is not found, the webhook is not honored. + If the specified kube config data is not valid, the webhook is not honored. + The namespace for this secret is determined by the point of use. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + featureGate: + description: FeatureGate holds cluster-wide information about + feature gates. + properties: + customNoUpgrade: + description: |- + customNoUpgrade allows the enabling or disabling of any feature. Turning this feature set on IS NOT SUPPORTED, CANNOT BE UNDONE, and PREVENTS UPGRADES. + Because of its nature, this setting cannot be validated. If you have any typos or accidentally apply invalid combinations + your cluster may fail in an unrecoverable way. featureSet must equal "CustomNoUpgrade" must be set to use this field. + nullable: true + properties: + disabled: + description: disabled is a list of all feature gates that + you want to force off + items: + description: FeatureGateName is a string to enforce + patterns on the name of a FeatureGate + pattern: ^([A-Za-z0-9-]+\.)*[A-Za-z0-9-]+\.?$ + type: string + type: array + enabled: + description: enabled is a list of all feature gates that + you want to force on + items: + description: FeatureGateName is a string to enforce + patterns on the name of a FeatureGate + pattern: ^([A-Za-z0-9-]+\.)*[A-Za-z0-9-]+\.?$ + type: string + type: array + type: object + featureSet: + description: |- + featureSet changes the list of features in the cluster. The default is empty. Be very careful adjusting this setting. + Turning on or off features may cause irreversible changes in your cluster which cannot be undone. + type: string + x-kubernetes-validations: + - message: CustomNoUpgrade may not be changed + rule: 'oldSelf == ''CustomNoUpgrade'' ? self == ''CustomNoUpgrade'' + : true' + - message: TechPreviewNoUpgrade may not be changed + rule: 'oldSelf == ''TechPreviewNoUpgrade'' ? self == ''TechPreviewNoUpgrade'' + : true' + - message: DevPreviewNoUpgrade may not be changed + rule: 'oldSelf == ''DevPreviewNoUpgrade'' ? self == ''DevPreviewNoUpgrade'' + : true' + type: object + image: + description: |- + Image governs policies related to imagestream imports and runtime configuration + for external registries. It allows cluster admins to configure which registries + OpenShift is allowed to import images from, extra CA trust bundles for external + registries, and policies to block or allow registry hostnames. + When exposing OpenShift's image registry to the public, this also lets cluster + admins specify the external hostname. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + properties: + additionalTrustedCA: + description: |- + additionalTrustedCA is a reference to a ConfigMap containing additional CAs that + should be trusted during imagestream import, pod image pull, build image pull, and + imageregistry pullthrough. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + required: + - name + type: object + allowedRegistriesForImport: + description: |- + allowedRegistriesForImport limits the container image registries that normal users may import + images from. Set this list to the registries that you trust to contain valid Docker + images and that you want applications to be able to import from. Users with + permission to create Images or ImageStreamMappings via the API are not affected by + this policy - typically only administrators or system integrations will have those + permissions. + items: + description: |- + RegistryLocation contains a location of the registry specified by the registry domain + name. The domain name might include wildcards, like '*' or '??'. + properties: + domainName: + description: |- + domainName specifies a domain name for the registry + In case the registry use non-standard (80 or 443) port, the port should be included + in the domain name as well. + type: string + insecure: + description: |- + insecure indicates whether the registry is secure (https) or insecure (http) + By default (if not specified) the registry is assumed as secure. + type: boolean + type: object + type: array + externalRegistryHostnames: + description: |- + externalRegistryHostnames provides the hostnames for the default external image + registry. The external hostname should be set only when the image registry + is exposed externally. The first value is used in 'publicDockerImageRepository' + field in ImageStreams. The value must be in "hostname[:port]" format. + items: + type: string + type: array + registrySources: + description: |- + registrySources contains configuration that determines how the container runtime + should treat individual registries when accessing images for builds+pods. (e.g. + whether or not to allow insecure access). It does not contain configuration for the + internal cluster registry. + properties: + allowedRegistries: + description: |- + allowedRegistries are the only registries permitted for image pull and push actions. All other registries are denied. + + Only one of BlockedRegistries or AllowedRegistries may be set. + items: + type: string + type: array + blockedRegistries: + description: |- + blockedRegistries cannot be used for image pull and push actions. All other registries are permitted. + + Only one of BlockedRegistries or AllowedRegistries may be set. + items: + type: string + type: array + containerRuntimeSearchRegistries: + description: |- + containerRuntimeSearchRegistries are registries that will be searched when pulling images that do not have fully qualified + domains in their pull specs. Registries will be searched in the order provided in the list. + Note: this search list only works with the container runtime, i.e CRI-O. Will NOT work with builds or imagestream imports. + format: hostname + items: + type: string + minItems: 1 + type: array + x-kubernetes-list-type: set + insecureRegistries: + description: insecureRegistries are registries which do + not have a valid TLS certificates or only support HTTP + connections. + items: + type: string + type: array + type: object + type: object + ingress: + description: |- + Ingress holds cluster-wide information about ingress, including the default ingress domain + used for routes. + properties: + appsDomain: + description: |- + appsDomain is an optional domain to use instead of the one specified + in the domain field when a Route is created without specifying an explicit + host. If appsDomain is nonempty, this value is used to generate default + host values for Route. Unlike domain, appsDomain may be modified after + installation. + This assumes a new ingresscontroller has been setup with a wildcard + certificate. + type: string + componentRoutes: + description: |- + componentRoutes is an optional list of routes that are managed by OpenShift components + that a cluster-admin is able to configure the hostname and serving certificate for. + The namespace and name of each route in this list should match an existing entry in the + status.componentRoutes list. + + To determine the set of configurable Routes, look at namespace and name of entries in the + .status.componentRoutes list, where participating operators write the status of + configurable routes. + items: + description: ComponentRouteSpec allows for configuration + of a route's hostname and serving certificate. + properties: + hostname: + description: hostname is the hostname that should be + used by the route. + pattern: ^([a-zA-Z0-9\p{S}\p{L}]((-?[a-zA-Z0-9\p{S}\p{L}]{0,62})?)|([a-zA-Z0-9\p{S}\p{L}](([a-zA-Z0-9-\p{S}\p{L}]{0,61}[a-zA-Z0-9\p{S}\p{L}])?)(\.)){1,}([a-zA-Z\p{L}]){2,63})$|^(([a-z0-9][-a-z0-9]{0,61}[a-z0-9]|[a-z0-9]{1,63})[\.]){0,}([a-z0-9][-a-z0-9]{0,61}[a-z0-9]|[a-z0-9]{1,63})$ + type: string + name: + description: |- + name is the logical name of the route to customize. + + The namespace and name of this componentRoute must match a corresponding + entry in the list of status.componentRoutes if the route is to be customized. + maxLength: 256 + minLength: 1 + type: string + namespace: + description: |- + namespace is the namespace of the route to customize. + + The namespace and name of this componentRoute must match a corresponding + entry in the list of status.componentRoutes if the route is to be customized. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + servingCertKeyPairSecret: + description: |- + servingCertKeyPairSecret is a reference to a secret of type `kubernetes.io/tls` in the openshift-config namespace. + The serving cert/key pair must match and will be used by the operator to fulfill the intent of serving with this name. + If the custom hostname uses the default routing suffix of the cluster, + the Secret specification for a serving certificate will not be needed. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + required: + - hostname + - name + - namespace + type: object + type: array + x-kubernetes-list-map-keys: + - namespace + - name + x-kubernetes-list-type: map + domain: + description: |- + domain is used to generate a default host name for a route when the + route's host name is empty. The generated host name will follow this + pattern: "..". + + It is also used as the default wildcard domain suffix for ingress. The + default ingresscontroller domain will follow this pattern: "*.". + + Once set, changing domain is not currently supported. + type: string + loadBalancer: + description: |- + loadBalancer contains the load balancer details in general which are not only specific to the underlying infrastructure + provider of the current cluster and are required for Ingress Controller to work on OpenShift. + properties: + platform: + description: |- + platform holds configuration specific to the underlying + infrastructure provider for the ingress load balancers. + When omitted, this means the user has no opinion and the platform is left + to choose reasonable defaults. These defaults are subject to change over time. + properties: + aws: + description: aws contains settings specific to the + Amazon Web Services infrastructure provider. + properties: + type: + description: |- + type allows user to set a load balancer type. + When this field is set the default ingresscontroller will get created using the specified LBType. + If this field is not set then the default ingress controller of LBType Classic will be created. + Valid values are: + + * "Classic": A Classic Load Balancer that makes routing decisions at either + the transport layer (TCP/SSL) or the application layer (HTTP/HTTPS). See + the following for additional details: + + https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#clb + + * "NLB": A Network Load Balancer that makes routing decisions at the + transport layer (TCP/SSL). See the following for additional details: + + https://docs.aws.amazon.com/AmazonECS/latest/developerguide/load-balancer-types.html#nlb + enum: + - NLB + - Classic + type: string + required: + - type + type: object + type: + description: |- + type is the underlying infrastructure provider for the cluster. + Allowed values are "AWS", "Azure", "BareMetal", "GCP", "Libvirt", + "OpenStack", "VSphere", "oVirt", "KubeVirt", "EquinixMetal", "PowerVS", + "AlibabaCloud", "Nutanix" and "None". Individual components may not support all platforms, + and must handle unrecognized platforms as None if they do not support that platform. + enum: + - "" + - AWS + - Azure + - BareMetal + - GCP + - Libvirt + - OpenStack + - None + - VSphere + - oVirt + - IBMCloud + - KubeVirt + - EquinixMetal + - PowerVS + - AlibabaCloud + - Nutanix + - External + type: string + type: object + type: object + requiredHSTSPolicies: + description: |- + requiredHSTSPolicies specifies HSTS policies that are required to be set on newly created or updated routes + matching the domainPattern/s and namespaceSelector/s that are specified in the policy. + Each requiredHSTSPolicy must have at least a domainPattern and a maxAge to validate a route HSTS Policy route + annotation, and affect route admission. + + A candidate route is checked for HSTS Policies if it has the HSTS Policy route annotation: + "haproxy.router.openshift.io/hsts_header" + E.g. haproxy.router.openshift.io/hsts_header: max-age=31536000;preload;includeSubDomains + + - For each candidate route, if it matches a requiredHSTSPolicy domainPattern and optional namespaceSelector, + then the maxAge, preloadPolicy, and includeSubdomainsPolicy must be valid to be admitted. Otherwise, the route + is rejected. + - The first match, by domainPattern and optional namespaceSelector, in the ordering of the RequiredHSTSPolicies + determines the route's admission status. + - If the candidate route doesn't match any requiredHSTSPolicy domainPattern and optional namespaceSelector, + then it may use any HSTS Policy annotation. + + The HSTS policy configuration may be changed after routes have already been created. An update to a previously + admitted route may then fail if the updated route does not conform to the updated HSTS policy configuration. + However, changing the HSTS policy configuration will not cause a route that is already admitted to stop working. + + Note that if there are no RequiredHSTSPolicies, any HSTS Policy annotation on the route is valid. + items: + properties: + domainPatterns: + description: |- + domainPatterns is a list of domains for which the desired HSTS annotations are required. + If domainPatterns is specified and a route is created with a spec.host matching one of the domains, + the route must specify the HSTS Policy components described in the matching RequiredHSTSPolicy. + + The use of wildcards is allowed like this: *.foo.com matches everything under foo.com. + foo.com only matches foo.com, so to cover foo.com and everything under it, you must specify *both*. + items: + type: string + minItems: 1 + type: array + includeSubDomainsPolicy: + description: |- + includeSubDomainsPolicy means the HSTS Policy should apply to any subdomains of the host's + domain name. Thus, for the host bar.foo.com, if includeSubDomainsPolicy was set to RequireIncludeSubDomains: + - the host app.bar.foo.com would inherit the HSTS Policy of bar.foo.com + - the host bar.foo.com would inherit the HSTS Policy of bar.foo.com + - the host foo.com would NOT inherit the HSTS Policy of bar.foo.com + - the host def.foo.com would NOT inherit the HSTS Policy of bar.foo.com + enum: + - RequireIncludeSubDomains + - RequireNoIncludeSubDomains + - NoOpinion + type: string + maxAge: + description: |- + maxAge is the delta time range in seconds during which hosts are regarded as HSTS hosts. + If set to 0, it negates the effect, and hosts are removed as HSTS hosts. + If set to 0 and includeSubdomains is specified, all subdomains of the host are also removed as HSTS hosts. + maxAge is a time-to-live value, and if this policy is not refreshed on a client, the HSTS + policy will eventually expire on that client. + properties: + largestMaxAge: + description: |- + The largest allowed value (in seconds) of the RequiredHSTSPolicy max-age + This value can be left unspecified, in which case no upper limit is enforced. + format: int32 + maximum: 2147483647 + minimum: 0 + type: integer + smallestMaxAge: + description: |- + The smallest allowed value (in seconds) of the RequiredHSTSPolicy max-age + Setting max-age=0 allows the deletion of an existing HSTS header from a host. This is a necessary + tool for administrators to quickly correct mistakes. + This value can be left unspecified, in which case no lower limit is enforced. + format: int32 + maximum: 2147483647 + minimum: 0 + type: integer + type: object + namespaceSelector: + description: |- + namespaceSelector specifies a label selector such that the policy applies only to those routes that + are in namespaces with labels that match the selector, and are in one of the DomainPatterns. + Defaults to the empty LabelSelector, which matches everything. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + preloadPolicy: + description: |- + preloadPolicy directs the client to include hosts in its host preload list so that + it never needs to do an initial load to get the HSTS header (note that this is not defined + in RFC 6797 and is therefore client implementation-dependent). + enum: + - RequirePreload + - RequireNoPreload + - NoOpinion + type: string + required: + - domainPatterns + type: object + type: array + type: object + network: + description: |- + Network holds cluster-wide information about the network. It is used to configure the desired network configuration, such as: IP address pools for services/pod IPs, network plugin, etc. + Please view network.spec for an explanation on what applies when configuring this resource. + properties: + clusterNetwork: + description: |- + IP address pool to use for pod IPs. + This field is immutable after installation. + items: + description: |- + ClusterNetworkEntry is a contiguous block of IP addresses from which pod IPs + are allocated. + properties: + cidr: + description: The complete block for pod IPs. + type: string + hostPrefix: + description: |- + The size (prefix) of block to allocate to each node. If this + field is not used by the plugin, it can be left unset. + format: int32 + minimum: 0 + type: integer + type: object + type: array + x-kubernetes-list-type: atomic + externalIP: + description: |- + externalIP defines configuration for controllers that + affect Service.ExternalIP. If nil, then ExternalIP is + not allowed to be set. + properties: + autoAssignCIDRs: + description: |- + autoAssignCIDRs is a list of CIDRs from which to automatically assign + Service.ExternalIP. These are assigned when the service is of type + LoadBalancer. In general, this is only useful for bare-metal clusters. + In Openshift 3.x, this was misleadingly called "IngressIPs". + Automatically assigned External IPs are not affected by any + ExternalIPPolicy rules. + Currently, only one entry may be provided. + items: + type: string + type: array + x-kubernetes-list-type: atomic + policy: + description: |- + policy is a set of restrictions applied to the ExternalIP field. + If nil or empty, then ExternalIP is not allowed to be set. + properties: + allowedCIDRs: + description: allowedCIDRs is the list of allowed CIDRs. + items: + type: string + type: array + x-kubernetes-list-type: atomic + rejectedCIDRs: + description: |- + rejectedCIDRs is the list of disallowed CIDRs. These take precedence + over allowedCIDRs. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + type: object + networkType: + description: |- + NetworkType is the plugin that is to be deployed (e.g. OpenShiftSDN). + This should match a value that the cluster-network-operator understands, + or else no networking will be installed. + Currently supported values are: + - OpenShiftSDN + This field is immutable after installation. + type: string + serviceNetwork: + description: |- + IP address pool for services. + Currently, we only support a single entry here. + This field is immutable after installation. + items: + type: string + type: array + x-kubernetes-list-type: atomic + serviceNodePortRange: + description: |- + The port range allowed for Services of type NodePort. + If not specified, the default of 30000-32767 will be used. + Such Services without a NodePort specified will have one + automatically allocated from this range. + This parameter can be updated after the cluster is + installed. + pattern: ^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$ + type: string + type: object + oauth: + description: |- + OAuth holds cluster-wide information about OAuth. + It is used to configure the integrated OAuth server. + This configuration is only honored when the top level Authentication config has type set to IntegratedOAuth. + properties: + identityProviders: + description: |- + identityProviders is an ordered list of ways for a user to identify themselves. + When this list is empty, no identities are provisioned for users. + items: + description: IdentityProvider provides identities for users + authenticating using credentials + properties: + basicAuth: + description: basicAuth contains configuration options + for the BasicAuth IdP + properties: + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + tlsClientCert: + description: |- + tlsClientCert is an optional reference to a secret by name that contains the + PEM-encoded TLS client certificate to present when connecting to the server. + The key "tls.crt" is used to locate the data. + If specified and the secret or expected key is not found, the identity provider is not honored. + If the specified certificate data is not valid, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + tlsClientKey: + description: |- + tlsClientKey is an optional reference to a secret by name that contains the + PEM-encoded TLS private key for the client certificate referenced in tlsClientCert. + The key "tls.key" is used to locate the data. + If specified and the secret or expected key is not found, the identity provider is not honored. + If the specified certificate data is not valid, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + url: + description: url is the remote URL to connect to + type: string + type: object + github: + description: github enables user authentication using + GitHub credentials + properties: + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + This can only be configured when hostname is set to a non-empty value. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + clientID: + description: clientID is the oauth client ID + type: string + clientSecret: + description: |- + clientSecret is a required reference to the secret by name containing the oauth client secret. + The key "clientSecret" is used to locate the data. + If the secret or expected key is not found, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + hostname: + description: |- + hostname is the optional domain (e.g. "mycompany.com") for use with a hosted instance of + GitHub Enterprise. + It must match the GitHub Enterprise settings value configured at /setup/settings#hostname. + type: string + organizations: + description: organizations optionally restricts + which organizations are allowed to log in + items: + type: string + type: array + teams: + description: teams optionally restricts which teams + are allowed to log in. Format is /. + items: + type: string + type: array + type: object + gitlab: + description: gitlab enables user authentication using + GitLab credentials + properties: + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + clientID: + description: clientID is the oauth client ID + type: string + clientSecret: + description: |- + clientSecret is a required reference to the secret by name containing the oauth client secret. + The key "clientSecret" is used to locate the data. + If the secret or expected key is not found, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + url: + description: url is the oauth server base URL + type: string + type: object + google: + description: google enables user authentication using + Google credentials + properties: + clientID: + description: clientID is the oauth client ID + type: string + clientSecret: + description: |- + clientSecret is a required reference to the secret by name containing the oauth client secret. + The key "clientSecret" is used to locate the data. + If the secret or expected key is not found, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + hostedDomain: + description: hostedDomain is the optional Google + App domain (e.g. "mycompany.com") to restrict + logins to + type: string + type: object + htpasswd: + description: htpasswd enables user authentication using + an HTPasswd file to validate credentials + properties: + fileData: + description: |- + fileData is a required reference to a secret by name containing the data to use as the htpasswd file. + The key "htpasswd" is used to locate the data. + If the secret or expected key is not found, the identity provider is not honored. + If the specified htpasswd data is not valid, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + type: object + keystone: + description: keystone enables user authentication using + keystone password credentials + properties: + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + domainName: + description: domainName is required for keystone + v3 + type: string + tlsClientCert: + description: |- + tlsClientCert is an optional reference to a secret by name that contains the + PEM-encoded TLS client certificate to present when connecting to the server. + The key "tls.crt" is used to locate the data. + If specified and the secret or expected key is not found, the identity provider is not honored. + If the specified certificate data is not valid, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + tlsClientKey: + description: |- + tlsClientKey is an optional reference to a secret by name that contains the + PEM-encoded TLS private key for the client certificate referenced in tlsClientCert. + The key "tls.key" is used to locate the data. + If specified and the secret or expected key is not found, the identity provider is not honored. + If the specified certificate data is not valid, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + url: + description: url is the remote URL to connect to + type: string + type: object + ldap: + description: ldap enables user authentication using + LDAP credentials + properties: + attributes: + description: attributes maps LDAP attributes to + identities + properties: + email: + description: |- + email is the list of attributes whose values should be used as the email address. Optional. + If unspecified, no email is set for the identity + items: + type: string + type: array + id: + description: |- + id is the list of attributes whose values should be used as the user ID. Required. + First non-empty attribute is used. At least one attribute is required. If none of the listed + attribute have a value, authentication fails. + LDAP standard identity attribute is "dn" + items: + type: string + type: array + name: + description: |- + name is the list of attributes whose values should be used as the display name. Optional. + If unspecified, no display name is set for the identity + LDAP standard display name attribute is "cn" + items: + type: string + type: array + preferredUsername: + description: |- + preferredUsername is the list of attributes whose values should be used as the preferred username. + LDAP standard login attribute is "uid" + items: + type: string + type: array + type: object + bindDN: + description: bindDN is an optional DN to bind with + during the search phase. + type: string + bindPassword: + description: |- + bindPassword is an optional reference to a secret by name + containing a password to bind with during the search phase. + The key "bindPassword" is used to locate the data. + If specified and the secret or expected key is not found, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + insecure: + description: |- + insecure, if true, indicates the connection should not use TLS + WARNING: Should not be set to `true` with the URL scheme "ldaps://" as "ldaps://" URLs always + attempt to connect using TLS, even when `insecure` is set to `true` + When `true`, "ldap://" URLS connect insecurely. When `false`, "ldap://" URLs are upgraded to + a TLS connection using StartTLS as specified in https://tools.ietf.org/html/rfc2830. + type: boolean + url: + description: |- + url is an RFC 2255 URL which specifies the LDAP search parameters to use. + The syntax of the URL is: + ldap://host:port/basedn?attribute?scope?filter + type: string + type: object + mappingMethod: + description: |- + mappingMethod determines how identities from this provider are mapped to users + Defaults to "claim" + type: string + name: + description: |- + name is used to qualify the identities returned by this provider. + - It MUST be unique and not shared by any other identity provider used + - It MUST be a valid path segment: name cannot equal "." or ".." or contain "/" or "%" or ":" + Ref: https://godoc.org/github.com/openshift/origin/pkg/user/apis/user/validation#ValidateIdentityProviderName + type: string + openID: + description: openID enables user authentication using + OpenID credentials + properties: + ca: + description: |- + ca is an optional reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + The key "ca.crt" is used to locate the data. + If specified and the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + If empty, the default system roots are used. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + claims: + description: claims mappings + properties: + email: + description: |- + email is the list of claims whose values should be used as the email address. Optional. + If unspecified, no email is set for the identity + items: + type: string + type: array + x-kubernetes-list-type: atomic + groups: + description: |- + groups is the list of claims value of which should be used to synchronize groups + from the OIDC provider to OpenShift for the user. + If multiple claims are specified, the first one with a non-empty value is used. + items: + description: |- + OpenIDClaim represents a claim retrieved from an OpenID provider's tokens or userInfo + responses + minLength: 1 + type: string + type: array + x-kubernetes-list-type: atomic + name: + description: |- + name is the list of claims whose values should be used as the display name. Optional. + If unspecified, no display name is set for the identity + items: + type: string + type: array + x-kubernetes-list-type: atomic + preferredUsername: + description: |- + preferredUsername is the list of claims whose values should be used as the preferred username. + If unspecified, the preferred username is determined from the value of the sub claim + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + clientID: + description: clientID is the oauth client ID + type: string + clientSecret: + description: |- + clientSecret is a required reference to the secret by name containing the oauth client secret. + The key "clientSecret" is used to locate the data. + If the secret or expected key is not found, the identity provider is not honored. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced secret + type: string + required: + - name + type: object + extraAuthorizeParameters: + additionalProperties: + type: string + description: extraAuthorizeParameters are any custom + parameters to add to the authorize request. + type: object + extraScopes: + description: extraScopes are any scopes to request + in addition to the standard "openid" scope. + items: + type: string + type: array + issuer: + description: |- + issuer is the URL that the OpenID Provider asserts as its Issuer Identifier. + It must use the https scheme with no query or fragment component. + type: string + type: object + requestHeader: + description: requestHeader enables user authentication + using request header credentials + properties: + ca: + description: |- + ca is a required reference to a config map by name containing the PEM-encoded CA bundle. + It is used as a trust anchor to validate the TLS certificate presented by the remote server. + Specifically, it allows verification of incoming requests to prevent header spoofing. + The key "ca.crt" is used to locate the data. + If the config map or expected key is not found, the identity provider is not honored. + If the specified ca data is not valid, the identity provider is not honored. + The namespace for this config map is openshift-config. + properties: + name: + description: name is the metadata.name of the + referenced config map + type: string + required: + - name + type: object + challengeURL: + description: |- + challengeURL is a URL to redirect unauthenticated /authorize requests to + Unauthenticated requests from OAuth clients which expect WWW-Authenticate challenges will be + redirected here. + ${url} is replaced with the current URL, escaped to be safe in a query parameter + https://www.example.com/sso-login?then=${url} + ${query} is replaced with the current query string + https://www.example.com/auth-proxy/oauth/authorize?${query} + Required when challenge is set to true. + type: string + clientCommonNames: + description: |- + clientCommonNames is an optional list of common names to require a match from. If empty, any + client certificate validated against the clientCA bundle is considered authoritative. + items: + type: string + type: array + emailHeaders: + description: emailHeaders is the set of headers + to check for the email address + items: + type: string + type: array + headers: + description: headers is the set of headers to check + for identity information + items: + type: string + type: array + loginURL: + description: |- + loginURL is a URL to redirect unauthenticated /authorize requests to + Unauthenticated requests from OAuth clients which expect interactive logins will be redirected here + ${url} is replaced with the current URL, escaped to be safe in a query parameter + https://www.example.com/sso-login?then=${url} + ${query} is replaced with the current query string + https://www.example.com/auth-proxy/oauth/authorize?${query} + Required when login is set to true. + type: string + nameHeaders: + description: nameHeaders is the set of headers to + check for the display name + items: + type: string + type: array + preferredUsernameHeaders: + description: preferredUsernameHeaders is the set + of headers to check for the preferred username + items: + type: string + type: array + type: object + type: + description: type identifies the identity provider type + for this entry. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + templates: + description: templates allow you to customize pages like the + login page. + properties: + error: + description: |- + error is the name of a secret that specifies a go template to use to render error pages + during the authentication or grant flow. + The key "errors.html" is used to locate the template data. + If specified and the secret or expected key is not found, the default error page is used. + If the specified template is not valid, the default error page is used. + If unspecified, the default error page is used. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + login: + description: |- + login is the name of a secret that specifies a go template to use to render the login page. + The key "login.html" is used to locate the template data. + If specified and the secret or expected key is not found, the default login page is used. + If the specified template is not valid, the default login page is used. + If unspecified, the default login page is used. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + providerSelection: + description: |- + providerSelection is the name of a secret that specifies a go template to use to render + the provider selection page. + The key "providers.html" is used to locate the template data. + If specified and the secret or expected key is not found, the default provider selection page is used. + If the specified template is not valid, the default provider selection page is used. + If unspecified, the default provider selection page is used. + The namespace for this secret is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + secret + type: string + required: + - name + type: object + type: object + tokenConfig: + description: tokenConfig contains options for authorization + and access tokens + properties: + accessTokenInactivityTimeout: + description: |- + accessTokenInactivityTimeout defines the token inactivity timeout + for tokens granted by any client. + The value represents the maximum amount of time that can occur between + consecutive uses of the token. Tokens become invalid if they are not + used within this temporal window. The user will need to acquire a new + token to regain access once a token times out. Takes valid time + duration string such as "5m", "1.5h" or "2h45m". The minimum allowed + value for duration is 300s (5 minutes). If the timeout is configured + per client, then that value takes precedence. If the timeout value is + not specified and the client does not override the value, then tokens + are valid until their lifetime. + + WARNING: existing tokens' timeout will not be affected (lowered) by changing this value + type: string + accessTokenInactivityTimeoutSeconds: + description: 'accessTokenInactivityTimeoutSeconds - DEPRECATED: + setting this field has no effect.' + format: int32 + type: integer + accessTokenMaxAgeSeconds: + description: accessTokenMaxAgeSeconds defines the maximum + age of access tokens + format: int32 + type: integer + type: object + type: object + x-kubernetes-validations: + - message: spec.configuration.oauth.tokenConfig.accessTokenInactivityTimeout + minimum acceptable token timeout value is 300 seconds + rule: '!has(self.tokenConfig) || !has(self.tokenConfig.accessTokenInactivityTimeout) + || duration(self.tokenConfig.accessTokenInactivityTimeout).getSeconds() + >= 300' + operatorhub: + description: |- + OperatorHub specifies the configuration for the Operator Lifecycle Manager in the HostedCluster. This is only configured at deployment time but the controller are not reconcilling over it. + The OperatorHub configuration will be constantly reconciled if catalog placement is management, but only on cluster creation otherwise. + properties: + disableAllDefaultSources: + description: |- + disableAllDefaultSources allows you to disable all the default hub + sources. If this is true, a specific entry in sources can be used to + enable a default source. If this is false, a specific entry in + sources can be used to disable or enable a default source. + type: boolean + sources: + description: |- + sources is the list of default hub sources and their configuration. + If the list is empty, it implies that the default hub sources are + enabled on the cluster unless disableAllDefaultSources is true. + If disableAllDefaultSources is true and sources is not empty, + the configuration present in sources will take precedence. The list of + default hub sources and their current state will always be reflected in + the status block. + items: + description: HubSource is used to specify the hub source + and its configuration + properties: + disabled: + description: disabled is used to disable a default hub + source on cluster + type: boolean + name: + description: name is the name of one of the default + hub sources + maxLength: 253 + minLength: 1 + type: string + type: object + type: array + type: object + proxy: + description: ProxySpec contains cluster proxy creation configuration. + properties: + httpProxy: + description: httpProxy is the URL of the proxy for HTTP requests. Empty + means unset and will not result in an env var. + type: string + httpsProxy: + description: httpsProxy is the URL of the proxy for HTTPS + requests. Empty means unset and will not result in an env + var. + type: string + noProxy: + description: |- + noProxy is a comma-separated list of hostnames and/or CIDRs and/or IPs for which the proxy should not be used. + Empty means unset and will not result in an env var. + type: string + readinessEndpoints: + description: readinessEndpoints is a list of endpoints used + to verify readiness of the proxy. + items: + type: string + type: array + trustedCA: + description: |- + trustedCA is a reference to a ConfigMap containing a CA certificate bundle. + The trustedCA field should only be consumed by a proxy validator. The + validator is responsible for reading the certificate bundle from the required + key "ca-bundle.crt", merging it with the system default trust bundle, + and writing the merged trust bundle to a ConfigMap named "trusted-ca-bundle" + in the "openshift-config-managed" namespace. Clients that expect to make + proxy connections must use the trusted-ca-bundle for all HTTPS requests to + the proxy, and may use the trusted-ca-bundle for non-proxy HTTPS requests as + well. + + The namespace for the ConfigMap referenced by trustedCA is + "openshift-config". Here is an example ConfigMap (in yaml): + + apiVersion: v1 + kind: ConfigMap + metadata: + name: user-ca-bundle + namespace: openshift-config + data: + ca-bundle.crt: | + -----BEGIN CERTIFICATE----- + Custom CA certificate bundle. + -----END CERTIFICATE----- + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + required: + - name + type: object + type: object + scheduler: + description: |- + Scheduler holds cluster-wide config information to run the Kubernetes Scheduler + and influence its placement decisions. The canonical name for this config is `cluster`. + properties: + defaultNodeSelector: + description: |- + defaultNodeSelector helps set the cluster-wide default node selector to + restrict pod placement to specific nodes. This is applied to the pods + created in all namespaces and creates an intersection with any existing + nodeSelectors already set on a pod, additionally constraining that pod's selector. + For example, + defaultNodeSelector: "type=user-node,region=east" would set nodeSelector + field in pod spec to "type=user-node,region=east" to all pods created + in all namespaces. Namespaces having project-wide node selectors won't be + impacted even if this field is set. This adds an annotation section to + the namespace. + For example, if a new namespace is created with + node-selector='type=user-node,region=east', + the annotation openshift.io/node-selector: type=user-node,region=east + gets added to the project. When the openshift.io/node-selector annotation + is set on the project the value is used in preference to the value we are setting + for defaultNodeSelector field. + For instance, + openshift.io/node-selector: "type=user-node,region=west" means + that the default of "type=user-node,region=east" set in defaultNodeSelector + would not be applied. + type: string + mastersSchedulable: + description: |- + MastersSchedulable allows masters nodes to be schedulable. When this flag is + turned on, all the master nodes in the cluster will be made schedulable, + so that workload pods can run on them. The default value for this field is false, + meaning none of the master nodes are schedulable. + Important Note: Once the workload pods start running on the master nodes, + extreme care must be taken to ensure that cluster-critical control plane components + are not impacted. + Please turn on this field after doing due diligence. + type: boolean + policy: + description: |- + DEPRECATED: the scheduler Policy API has been deprecated and will be removed in a future release. + policy is a reference to a ConfigMap containing scheduler policy which has + user specified predicates and priorities. If this ConfigMap is not available + scheduler will default to use DefaultAlgorithmProvider. + The namespace for this configmap is openshift-config. + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + required: + - name + type: object + profile: + description: |- + profile sets which scheduling profile should be set in order to configure scheduling + decisions for new pods. + + Valid values are "LowNodeUtilization", "HighNodeUtilization", "NoScoring" + Defaults to "LowNodeUtilization" + enum: + - "" + - LowNodeUtilization + - HighNodeUtilization + - NoScoring + type: string + type: object + type: object + controlPlaneReleaseImage: + description: |- + ControlPlaneReleaseImage specifies the desired OCP release payload for + control plane components running on the management cluster. + If not defined, ReleaseImage is used + type: string + controllerAvailabilityPolicy: + default: SingleReplica + description: |- + ControllerAvailabilityPolicy specifies the availability policy applied to + critical control plane components. The default value is SingleReplica. + enum: + - HighlyAvailable + - SingleReplica + type: string + x-kubernetes-validations: + - message: ControllerAvailabilityPolicy is immutable + rule: self == oldSelf + dns: + description: DNSSpec specifies the DNS configuration for the hosted + cluster ingress. + properties: + baseDomain: + description: |- + baseDomain is the base domain of the hosted cluster. + It will be used to configure ingress in the hosted cluster through the subdomain baseDomainPrefix.baseDomain. + If baseDomainPrefix is omitted, the hostedCluster.name will be used as the subdomain. + Once set, this field is immutable. + When the value is the empty string "", the controller might default to a value depending on the platform. + maxLength: 253 + type: string + x-kubernetes-validations: + - message: baseDomain must be a valid domain name (e.g., example, + example.com, sub.example.com) + rule: self == "" || self.matches('^(?:(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}|[a-zA-Z0-9-]+)$') + - message: baseDomain is immutable + rule: oldSelf == "" || self == oldSelf + baseDomainPrefix: + description: |- + baseDomainPrefix is the base domain prefix for the hosted cluster ingress. + It will be used to configure ingress in the hosted cluster through the subdomain baseDomainPrefix.baseDomain. + If baseDomainPrefix is omitted, the hostedCluster.name will be used as the subdomain. + Set baseDomainPrefix to an empty string "", if you don't want a prefix at all (not even hostedCluster.name) to be prepended to baseDomain. + This field is immutable. + maxLength: 253 + type: string + x-kubernetes-validations: + - message: baseDomainPrefix must be a valid domain name (e.g., + example, example.com, sub.example.com) + rule: self == "" || self.matches('^(?:(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}|[a-zA-Z0-9-]+)$') + - message: baseDomainPrefix is immutable + rule: self == oldSelf + privateZoneID: + description: |- + privateZoneID is the Hosted Zone ID where all the DNS records that are only available internally to the cluster exist. + This field is optional and mainly leveraged in cloud environments where the DNS records for the .baseDomain are created by controllers in this zone. + Once set, this value is immutable. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: privateZoneID is immutable + rule: oldSelf == "" || self == oldSelf + publicZoneID: + description: |- + publicZoneID is the Hosted Zone ID where all the DNS records that are publicly accessible to the internet exist. + This field is optional and mainly leveraged in cloud environments where the DNS records for the .baseDomain are created by controllers in this zone. + Once set, this value is immutable. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: publicZoneID is immutable + rule: oldSelf == "" || self == oldSelf + required: + - baseDomain + type: object + etcd: + description: |- + Etcd contains metadata about the etcd cluster the hypershift managed Openshift control plane components + use to store data. + properties: + managed: + description: managed specifies the behavior of an etcd cluster + managed by HyperShift. + properties: + storage: + description: storage specifies how etcd data is persisted. + properties: + persistentVolume: + description: |- + persistentVolume is the configuration for PersistentVolume etcd storage. + With this implementation, a PersistentVolume will be allocated for every + etcd member (either 1 or 3 depending on the HostedCluster control plane + availability configuration). + properties: + size: + anyOf: + - type: integer + - type: string + default: 8Gi + description: |- + size is the minimum size of the data volume for each etcd member. + Default is 8Gi. + This field is immutable + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + x-kubernetes-validations: + - message: Etcd PV storage size is immutable + rule: self == oldSelf + storageClassName: + description: |- + storageClassName is the StorageClass of the data volume for each etcd member. + See https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1. + type: string + x-kubernetes-validations: + - message: storageClassName is immutable + rule: self == oldSelf + type: object + restoreSnapshotURL: + description: |- + restoreSnapshotURL allows an optional URL to be provided where + an etcd snapshot can be downloaded, for example a pre-signed URL + referencing a storage service. + This snapshot will be restored on initial startup, only when the etcd PV + is empty. + items: + type: string + type: array + x-kubernetes-validations: + - message: RestoreSnapshotURL shouldn't contain more than + 1 entry + rule: self.size() <= 1 + type: + description: |- + type is the kind of persistent storage implementation to use for etcd. + Only PersistentVolume is supported at the moment. + enum: + - PersistentVolume + type: string + required: + - type + type: object + required: + - storage + type: object + managementType: + description: |- + managementType defines how the etcd cluster is managed. + This can be either Managed or Unmanaged. + This field is immutable. + enum: + - Managed + - Unmanaged + type: string + x-kubernetes-validations: + - message: managementType is immutable + rule: self == oldSelf + unmanaged: + description: |- + unmanaged specifies configuration which enables the control plane to + integrate with an externally managed etcd cluster. + properties: + endpoint: + description: |- + endpoint is the full etcd cluster client endpoint URL. For example: + + https://etcd-client:2379 + + If the URL uses an HTTPS scheme, the TLS field is required. + pattern: ^https:// + type: string + tls: + description: tls specifies TLS configuration for HTTPS etcd + client endpoints. + properties: + clientSecret: + description: |- + ClientSecret refers to a secret for client mTLS authentication with the etcd cluster. It + may have the following key/value pairs: + + etcd-client-ca.crt: Certificate Authority value + etcd-client.crt: Client certificate value + etcd-client.key: Client certificate key value + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - clientSecret + type: object + required: + - endpoint + - tls + type: object + required: + - managementType + type: object + x-kubernetes-validations: + - message: Only managed configuration must be set when managementType + is Managed + rule: 'self.managementType == ''Managed'' ? has(self.managed) : + !has(self.managed)' + - message: Only unmanaged configuration must be set when managementType + is Unmanaged + rule: 'self.managementType == ''Unmanaged'' ? has(self.unmanaged) + : !has(self.unmanaged)' + fips: + description: FIPS specifies if the nodes for the cluster will be running + in FIPS mode + type: boolean + imageContentSources: + description: ImageContentSources lists sources/repositories for the + release-image content. + items: + description: |- + ImageContentSource specifies image mirrors that can be used by cluster nodes + to pull content. For cluster workloads, if a container image registry host of + the pullspec matches Source then one of the Mirrors are substituted as hosts + in the pullspec and tried in order to fetch the image. + properties: + mirrors: + description: Mirrors are one or more repositories that may also + contain the same images. + items: + type: string + type: array + source: + description: |- + Source is the repository that users refer to, e.g. in image pull + specifications. + type: string + required: + - source + type: object + type: array + infraID: + type: string + infrastructureAvailabilityPolicy: + default: SingleReplica + description: |- + InfrastructureAvailabilityPolicy specifies the availability policy applied + to infrastructure services which run on cluster nodes. The default value is + SingleReplica. + enum: + - HighlyAvailable + - SingleReplica + type: string + issuerURL: + description: |- + IssuerURL is an OIDC issuer URL which is used as the issuer in all + ServiceAccount tokens generated by the control plane API server. The + default value is kubernetes.default.svc, which only works for in-cluster + validation. + type: string + kubeconfig: + description: KubeConfig specifies the name and key for the kubeconfig + secret + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + labels: + additionalProperties: + type: string + description: |- + labels when specified, define what custom labels are added to the hcp pods. + Changing this day 2 will cause a rollout of all hcp pods. + Duplicate keys are not supported. If duplicate keys are defined, only the last key/value pair is preserved. + Valid values are those in https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + + -kubebuilder:validation:XValidation:rule=`self.all(key, size(key) <= 317 && key.matches('^(([A-Za-z0-9]+(\\.[A-Za-z0-9]+)?)*[A-Za-z0-9]\\/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$'))`, message="label key must have two segments: an optional prefix and name, separated by a slash (/). The name segment is required and must be 63 characters or less, beginning and ending with an alphanumeric character ([a-z0-9A-Z]) with dashes (-), underscores (_), dots (.), and alphanumerics between. The prefix is optional. If specified, the prefix must be a DNS subdomain: a series of DNS labels separated by dots (.), not longer than 253 characters in total, followed by a slash (/)" + -kubebuilder:validation:XValidation:rule=`self.all(key, size(self[key]) <= 63 && self[key].matches('^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$'))`, message="label value must be 63 characters or less (can be empty), consist of alphanumeric characters, dashes (-), underscores (_) or dots (.), and begin and end with an alphanumeric character" + maxProperties: 20 + type: object + networking: + description: |- + Networking specifies network configuration for the cluster. + Temporarily optional for backward compatibility, required in future releases. + properties: + apiServer: + description: |- + apiServer contains advanced network settings for the API server that affect + how the APIServer is exposed inside a hosted cluster node. + properties: + advertiseAddress: + description: |- + advertiseAddress is the address that pods within the nodes will use to talk to the API + server. This is an address associated with the loopback adapter of each + node. If not specified, the controller will take default values. + The default values will be set as 172.20.0.1 or fd00::1. + This value is immutable. + type: string + x-kubernetes-validations: + - message: advertiseAddress is immutable + rule: self == oldSelf + allowedCIDRBlocks: + description: |- + allowedCIDRBlocks is an allow list of CIDR blocks that can access the APIServer + If not specified, traffic is allowed from all addresses. + This depends on underlying support by the cloud provider for Service LoadBalancerSourceRanges + items: + pattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(3[0-2]|[1-2][0-9]|[0-9]))$ + type: string + type: array + port: + description: |- + port is the port at which the APIServer is exposed inside a node. Other + pods using host networking cannot listen on this port. + If omitted 6443 is used. + This is useful to choose a port other than the default one which might interfere with customer environments e.g. https://github.com/openshift/hypershift/pull/356. + Setting this to 443 is possible only for backward compatibility reasons and it's discouraged. + Doing so, it would result in the controller overriding the KAS endpoint in the guest cluster having a discrepancy with the KAS Pod and potentially causing temporarily network failures. + This value is immutable. + format: int32 + type: integer + x-kubernetes-validations: + - message: port is immutable + rule: self == oldSelf + type: object + clusterNetwork: + default: + - cidr: 10.132.0.0/14 + description: |- + clusterNetwork is the list of IP address pools for pods. + Defaults to cidr: "10.132.0.0/14". + Currently only one entry is supported. + This field is immutable. + items: + description: |- + ClusterNetworkEntry is a single IP address block for pod IP blocks. IP blocks + are allocated with size 2^HostSubnetLength. + properties: + cidr: + description: cidr is the IP block address pool. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') + hostPrefix: + description: |- + hostPrefix is the prefix size to allocate to each node from the CIDR. + For example, 24 would allocate 2^(32-24)=2^8=256 addresses to each node. If this + field is not used by the plugin, it can be left unset. + format: int32 + type: integer + required: + - cidr + type: object + maxItems: 2 + minItems: 1 + type: array + x-kubernetes-validations: + - message: clusterNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf + machineNetwork: + description: |- + machineNetwork is the list of IP address pools for machines. + This might be used among other things to generate appropriate networking security groups in some clouds providers. + Currently only one entry or two for dual stack is supported. + This field is immutable. + items: + description: MachineNetworkEntry is a single IP address block + for node IP blocks. + properties: + cidr: + description: CIDR is the IP block address pool for machines + within the cluster. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') + required: + - cidr + type: object + maxItems: 2 + minItems: 1 + type: array + x-kubernetes-validations: + - message: machineNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf + networkType: + default: OVNKubernetes + description: |- + networkType specifies the SDN provider used for cluster networking. + Defaults to OVNKubernetes. + This field is required and immutable. + kubebuilder:validation:XValidation:rule="self == oldSelf", message="networkType is immutable" + enum: + - OpenShiftSDN + - Calico + - OVNKubernetes + - Other + type: string + serviceNetwork: + default: + - cidr: 172.31.0.0/16 + description: |- + serviceNetwork is the list of IP address pools for services. + Defaults to cidr: "172.31.0.0/16". + Currently only one entry is supported. + This field is immutable. + items: + description: ServiceNetworkEntry is a single IP address block + for the service network. + properties: + cidr: + description: cidr is the IP block address pool for services + within the cluster in CIDR format (e.g., 192.168.1.0/24 + or 2001:0db8::/64) + maxLength: 43 + type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') + required: + - cidr + type: object + maxItems: 2 + minItems: 1 + type: array + x-kubernetes-validations: + - message: serviceNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector when specified, must be true for the pods + managed by the HostedCluster to be scheduled. + type: object + olmCatalogPlacement: + default: management + description: |- + OLMCatalogPlacement specifies the placement of OLM catalog components. By default, + this is set to management and OLM catalog components are deployed onto the management + cluster. If set to guest, the OLM catalog components will be deployed onto the guest + cluster. + enum: + - management + - guest + type: string + pausedUntil: + description: |- + PausedUntil is a field that can be used to pause reconciliation on a resource. + Either a date can be provided in RFC3339 format or a boolean. If a date is + provided: reconciliation is paused on the resource until that date. If the boolean true is + provided: reconciliation is paused on the resource until the field is removed. + type: string + platform: + description: |- + PlatformSpec specifies the underlying infrastructure provider for the cluster + and is used to configure platform specific behavior. + properties: + agent: + description: Agent specifies configuration for agent-based installations. + properties: + agentNamespace: + description: AgentNamespace is the namespace where to search + for Agents for this cluster + type: string + required: + - agentNamespace + type: object + aws: + description: AWS specifies configuration for clusters running + on Amazon Web Services. + properties: + additionalAllowedPrincipals: + description: |- + AdditionalAllowedPrincipals specifies a list of additional allowed principal ARNs + to be added to the hosted control plane's VPC Endpoint Service to enable additional + VPC Endpoint connection requests to be automatically accepted. + See https://docs.aws.amazon.com/vpc/latest/privatelink/configure-endpoint-service.html + for more details around VPC Endpoint Service allowed principals. + items: + type: string + type: array + cloudProviderConfig: + description: |- + CloudProviderConfig specifies AWS networking configuration for the control + plane. + This is mainly used for cloud provider controller config: + https://github.com/kubernetes/kubernetes/blob/f5be5052e3d0808abb904aebd3218fe4a5c2dd82/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go#L1347-L1364 + properties: + subnet: + description: Subnet is the subnet to use for control plane + cloud resources. + properties: + filters: + description: |- + Filters is a set of key/value pairs used to identify a resource + They are applied according to the rules defined by the AWS API: + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html + items: + description: Filter is a filter used to identify + an AWS resource + properties: + name: + description: Name of the filter. Filter names + are case-sensitive. + type: string + values: + description: Values includes one or more filter + values. Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + vpc: + description: VPC is the VPC to use for control plane cloud + resources. + type: string + zone: + description: |- + Zone is the availability zone where control plane cloud resources are + created. + type: string + required: + - vpc + type: object + endpointAccess: + default: Public + description: |- + EndpointAccess specifies the publishing scope of cluster endpoints. The + default is Public. + enum: + - Public + - PublicAndPrivate + - Private + type: string + multiArch: + default: false + description: |- + MultiArch specifies whether the Hosted Cluster will be expected to support NodePools with different + CPU architectures, i.e., supporting arm64 NodePools and supporting amd64 NodePools on the same Hosted Cluster. + Deprecated: This field is no longer used. The HyperShift Operator now performs multi-arch validations + automatically despite the platform type. The HyperShift Operator will set HostedCluster.Status.PayloadArch based + on the HostedCluster release image. This field is used by the NodePool controller to validate the + NodePool.Spec.Arch is supported. + type: boolean + region: + description: |- + Region is the AWS region in which the cluster resides. This configures the + OCP control plane cloud integrations, and is used by NodePool to resolve + the correct boot AMI for a given release. + type: string + resourceTags: + description: |- + ResourceTags is a list of additional tags to apply to AWS resources created + for the cluster. See + https://docs.aws.amazon.com/general/latest/gr/aws_tagging.html for + information on tagging AWS resources. AWS supports a maximum of 50 tags per + resource. OpenShift reserves 25 tags for its use, leaving 25 tags available + for the user. + items: + description: AWSResourceTag is a tag to apply to AWS resources + created for the cluster. + properties: + key: + description: Key is the key of the tag. + maxLength: 128 + minLength: 1 + pattern: ^[0-9A-Za-z_.:/=+-@]+$ + type: string + value: + description: |- + Value is the value of the tag. + + Some AWS service do not support empty values. Since tags are added to + resources in many services, the length of the tag value must meet the + requirements of all services. + maxLength: 256 + minLength: 1 + pattern: ^[0-9A-Za-z_.:/=+-@]+$ + type: string + required: + - key + - value + type: object + maxItems: 25 + type: array + rolesRef: + description: |- + RolesRef contains references to various AWS IAM roles required to enable + integrations such as OIDC. + properties: + controlPlaneOperatorARN: + description: "ControlPlaneOperatorARN is an ARN value + referencing a role appropriate for the Control Plane + Operator.\n\nThe following is an example of a valid + policy document:\n\n{\n\t\"Version\": \"2012-10-17\",\n\t\"Statement\": + [\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": + [\n\t\t\t\t\"ec2:CreateVpcEndpoint\",\n\t\t\t\t\"ec2:DescribeVpcEndpoints\",\n\t\t\t\t\"ec2:ModifyVpcEndpoint\",\n\t\t\t\t\"ec2:DeleteVpcEndpoints\",\n\t\t\t\t\"ec2:CreateTags\",\n\t\t\t\t\"route53:ListHostedZones\",\n\t\t\t\t\"ec2:CreateSecurityGroup\",\n\t\t\t\t\"ec2:AuthorizeSecurityGroupIngress\",\n\t\t\t\t\"ec2:AuthorizeSecurityGroupEgress\",\n\t\t\t\t\"ec2:DeleteSecurityGroup\",\n\t\t\t\t\"ec2:RevokeSecurityGroupIngress\",\n\t\t\t\t\"ec2:RevokeSecurityGroupEgress\",\n\t\t\t\t\"ec2:DescribeSecurityGroups\",\n\t\t\t\t\"ec2:DescribeVpcs\",\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t},\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": + [\n\t\t\t\t\"route53:ChangeResourceRecordSets\",\n\t\t\t\t\"route53:ListResourceRecordSets\"\n\t\t\t],\n\t\t\t\"Resource\": + \"arn:aws:route53:::%s\"\n\t\t}\n\t]\n}" + type: string + imageRegistryARN: + description: "ImageRegistryARN is an ARN value referencing + a role appropriate for the Image Registry Operator.\n\nThe + following is an example of a valid policy document:\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"s3:CreateBucket\",\n\t\t\t\t\"s3:DeleteBucket\",\n\t\t\t\t\"s3:PutBucketTagging\",\n\t\t\t\t\"s3:GetBucketTagging\",\n\t\t\t\t\"s3:PutBucketPublicAccessBlock\",\n\t\t\t\t\"s3:GetBucketPublicAccessBlock\",\n\t\t\t\t\"s3:PutEncryptionConfiguration\",\n\t\t\t\t\"s3:GetEncryptionConfiguration\",\n\t\t\t\t\"s3:PutLifecycleConfiguration\",\n\t\t\t\t\"s3:GetLifecycleConfiguration\",\n\t\t\t\t\"s3:GetBucketLocation\",\n\t\t\t\t\"s3:ListBucket\",\n\t\t\t\t\"s3:GetObject\",\n\t\t\t\t\"s3:PutObject\",\n\t\t\t\t\"s3:DeleteObject\",\n\t\t\t\t\"s3:ListBucketMultipartUploads\",\n\t\t\t\t\"s3:AbortMultipartUpload\",\n\t\t\t\t\"s3:ListMultipartUploadParts\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t}\n\t]\n}" + type: string + ingressARN: + description: "The referenced role must have a trust relationship + that allows it to be assumed via web identity.\nhttps://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html.\nExample:\n{\n\t\t\"Version\": + \"2012-10-17\",\n\t\t\"Statement\": [\n\t\t\t{\n\t\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\t\"Principal\": {\n\t\t\t\t\t\"Federated\": + \"{{ .ProviderARN }}\"\n\t\t\t\t},\n\t\t\t\t\t\"Action\": + \"sts:AssumeRoleWithWebIdentity\",\n\t\t\t\t\"Condition\": + {\n\t\t\t\t\t\"StringEquals\": {\n\t\t\t\t\t\t\"{{ .ProviderName + }}:sub\": {{ .ServiceAccounts }}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t]\n\t}\n\nIngressARN + is an ARN value referencing a role appropriate for the + Ingress Operator.\n\nThe following is an example of + a valid policy document:\n\n{\n\t\"Version\": \"2012-10-17\",\n\t\"Statement\": + [\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": + [\n\t\t\t\t\"elasticloadbalancing:DescribeLoadBalancers\",\n\t\t\t\t\"tag:GetResources\",\n\t\t\t\t\"route53:ListHostedZones\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t},\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": + [\n\t\t\t\t\"route53:ChangeResourceRecordSets\"\n\t\t\t],\n\t\t\t\"Resource\": + [\n\t\t\t\t\"arn:aws:route53:::PUBLIC_ZONE_ID\",\n\t\t\t\t\"arn:aws:route53:::PRIVATE_ZONE_ID\"\n\t\t\t]\n\t\t}\n\t]\n}" + type: string + kubeCloudControllerARN: + description: |- + KubeCloudControllerARN is an ARN value referencing a role appropriate for the KCM/KCC. + Source: https://cloud-provider-aws.sigs.k8s.io/prerequisites/#iam-policies + + The following is an example of a valid policy document: + + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstances", + "ec2:DescribeImages", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ], + "Effect": "Allow" + } + ] + } + type: string + networkARN: + description: "NetworkARN is an ARN value referencing a + role appropriate for the Network Operator.\n\nThe following + is an example of a valid policy document:\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"ec2:DescribeInstances\",\n + \ \"ec2:DescribeInstanceStatus\",\n \"ec2:DescribeInstanceTypes\",\n + \ \"ec2:UnassignPrivateIpAddresses\",\n \"ec2:AssignPrivateIpAddresses\",\n + \ \"ec2:UnassignIpv6Addresses\",\n \"ec2:AssignIpv6Addresses\",\n + \ \"ec2:DescribeSubnets\",\n \"ec2:DescribeNetworkInterfaces\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t}\n\t]\n}" + type: string + nodePoolManagementARN: + description: "NodePoolManagementARN is an ARN value referencing + a role appropriate for the CAPI Controller.\n\nThe following + is an example of a valid policy document:\n\n{\n \"Version\": + \"2012-10-17\",\n \"Statement\": [\n {\n \"Action\": + [\n \"ec2:AssociateRouteTable\",\n \"ec2:AttachInternetGateway\",\n + \ \"ec2:AuthorizeSecurityGroupIngress\",\n \"ec2:CreateInternetGateway\",\n + \ \"ec2:CreateNatGateway\",\n \"ec2:CreateRoute\",\n + \ \"ec2:CreateRouteTable\",\n \"ec2:CreateSecurityGroup\",\n + \ \"ec2:CreateSubnet\",\n \"ec2:CreateTags\",\n + \ \"ec2:DeleteInternetGateway\",\n \"ec2:DeleteNatGateway\",\n + \ \"ec2:DeleteRouteTable\",\n \"ec2:DeleteSecurityGroup\",\n + \ \"ec2:DeleteSubnet\",\n \"ec2:DeleteTags\",\n + \ \"ec2:DescribeAccountAttributes\",\n \"ec2:DescribeAddresses\",\n + \ \"ec2:DescribeAvailabilityZones\",\n \"ec2:DescribeImages\",\n + \ \"ec2:DescribeInstances\",\n \"ec2:DescribeInternetGateways\",\n + \ \"ec2:DescribeNatGateways\",\n \"ec2:DescribeNetworkInterfaces\",\n + \ \"ec2:DescribeNetworkInterfaceAttribute\",\n + \ \"ec2:DescribeRouteTables\",\n \"ec2:DescribeSecurityGroups\",\n + \ \"ec2:DescribeSubnets\",\n \"ec2:DescribeVpcs\",\n + \ \"ec2:DescribeVpcAttribute\",\n \"ec2:DescribeVolumes\",\n + \ \"ec2:DetachInternetGateway\",\n \"ec2:DisassociateRouteTable\",\n + \ \"ec2:DisassociateAddress\",\n \"ec2:ModifyInstanceAttribute\",\n + \ \"ec2:ModifyNetworkInterfaceAttribute\",\n \"ec2:ModifySubnetAttribute\",\n + \ \"ec2:RevokeSecurityGroupIngress\",\n \"ec2:RunInstances\",\n + \ \"ec2:TerminateInstances\",\n \"tag:GetResources\",\n + \ \"ec2:CreateLaunchTemplate\",\n \"ec2:CreateLaunchTemplateVersion\",\n + \ \"ec2:DescribeLaunchTemplates\",\n \"ec2:DescribeLaunchTemplateVersions\",\n + \ \"ec2:DeleteLaunchTemplate\",\n \"ec2:DeleteLaunchTemplateVersions\"\n + \ ],\n \"Resource\": [\n \"*\"\n ],\n + \ \"Effect\": \"Allow\"\n },\n {\n \"Condition\": + {\n \"StringLike\": {\n \"iam:AWSServiceName\": + \"elasticloadbalancing.amazonaws.com\"\n }\n },\n + \ \"Action\": [\n \"iam:CreateServiceLinkedRole\"\n + \ ],\n \"Resource\": [\n \"arn:*:iam::*:role/aws-service-role/elasticloadbalancing.amazonaws.com/AWSServiceRoleForElasticLoadBalancing\"\n + \ ],\n \"Effect\": \"Allow\"\n },\n {\n \"Action\": + [\n \"iam:PassRole\"\n ],\n \"Resource\": + [\n \"arn:*:iam::*:role/*-worker-role\"\n ],\n + \ \"Effect\": \"Allow\"\n },\n\t {\n\t \t\"Effect\": + \"Allow\",\n\t \t\"Action\": [\n\t \t\t\"kms:Decrypt\",\n\t + \ \t\t\"kms:ReEncrypt\",\n\t \t\t\"kms:GenerateDataKeyWithoutPlainText\",\n\t + \ \t\t\"kms:DescribeKey\"\n\t \t],\n\t \t\"Resource\": + \"*\"\n\t },\n\t {\n\t \t\"Effect\": \"Allow\",\n\t + \ \t\"Action\": [\n\t \t\t\"kms:CreateGrant\"\n\t \t],\n\t + \ \t\"Resource\": \"*\",\n\t \t\"Condition\": {\n\t + \ \t\t\"Bool\": {\n\t \t\t\t\"kms:GrantIsForAWSResource\": + true\n\t \t\t}\n\t \t}\n\t }\n ]\n}" + type: string + storageARN: + description: "StorageARN is an ARN value referencing a + role appropriate for the Storage Operator.\n\nThe following + is an example of a valid policy document:\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"ec2:AttachVolume\",\n\t\t\t\t\"ec2:CreateSnapshot\",\n\t\t\t\t\"ec2:CreateTags\",\n\t\t\t\t\"ec2:CreateVolume\",\n\t\t\t\t\"ec2:DeleteSnapshot\",\n\t\t\t\t\"ec2:DeleteTags\",\n\t\t\t\t\"ec2:DeleteVolume\",\n\t\t\t\t\"ec2:DescribeInstances\",\n\t\t\t\t\"ec2:DescribeSnapshots\",\n\t\t\t\t\"ec2:DescribeTags\",\n\t\t\t\t\"ec2:DescribeVolumes\",\n\t\t\t\t\"ec2:DescribeVolumesModifications\",\n\t\t\t\t\"ec2:DetachVolume\",\n\t\t\t\t\"ec2:ModifyVolume\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t}\n\t]\n}" + type: string + required: + - controlPlaneOperatorARN + - imageRegistryARN + - ingressARN + - kubeCloudControllerARN + - networkARN + - nodePoolManagementARN + - storageARN + type: object + serviceEndpoints: + description: |- + ServiceEndpoints specifies optional custom endpoints which will override + the default service endpoint of specific AWS Services. + + There must be only one ServiceEndpoint for a given service name. + items: + description: |- + AWSServiceEndpoint stores the configuration for services to + override existing defaults of AWS Services. + properties: + name: + description: |- + Name is the name of the AWS service. + This must be provided and cannot be empty. + type: string + url: + description: |- + URL is fully qualified URI with scheme https, that overrides the default generated + endpoint for a client. + This must be provided and cannot be empty. + pattern: ^https:// + type: string + required: + - name + - url + type: object + type: array + sharedVPC: + description: |- + SharedVPC contains fields that must be specified if the HostedCluster must use a VPC that is + created in a different AWS account and is shared with the AWS account where the HostedCluster + will be created. + properties: + localZoneID: + description: |- + LocalZoneID is the ID of the route53 hosted zone for [cluster-name].hypershift.local that is + associated with the HostedCluster's VPC and exists in the VPC owner account. + maxLength: 32 + type: string + rolesRef: + description: |- + RolesRef contains references to roles in the VPC owner account that enable a + HostedCluster on a shared VPC. + properties: + controlPlaneARN: + description: "ControlPlaneARN is an ARN value referencing + the role in the VPC owner account that allows\nthe + control plane operator in the cluster account to + create and manage a VPC endpoint, its\ncorresponding + Security Group, and DNS records in the hypershift + local hosted zone.\n\nThe referenced role must have + a trust relationship that allows it to be assumed + by the\ncontrol plane operator role in the VPC creator + account.\nExample:\n{\n\t \"Version\": \"2012-10-17\",\n\t + \"Statement\": [\n\t \t{\n\t \t\t\"Sid\": \"Statement1\",\n\t + \t\t\"Effect\": \"Allow\",\n\t \t\t\"Principal\": + {\n\t \t\t\t\"AWS\": \"arn:aws:iam::[cluster-creator-account-id]:role/[infra-id]-control-plane-operator\"\n\t + \t\t},\n\t \t\t\"Action\": \"sts:AssumeRole\"\n\t + \t}\n\t ]\n}\n\nThe following is an example of the + policy document for this role.\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"ec2:CreateVpcEndpoint\",\n\t\t\t\t\"ec2:DescribeVpcEndpoints\",\n\t\t\t\t\"ec2:ModifyVpcEndpoint\",\n\t\t\t\t\"ec2:DeleteVpcEndpoints\",\n\t\t\t\t\"ec2:CreateTags\",\n\t\t\t\t\"route53:ListHostedZones\",\n\t\t\t\t\"ec2:CreateSecurityGroup\",\n\t\t\t\t\"ec2:AuthorizeSecurityGroupIngress\",\n\t\t\t\t\"ec2:AuthorizeSecurityGroupEgress\",\n\t\t\t\t\"ec2:DeleteSecurityGroup\",\n\t\t\t\t\"ec2:RevokeSecurityGroupIngress\",\n\t\t\t\t\"ec2:RevokeSecurityGroupEgress\",\n\t\t\t\t\"ec2:DescribeSecurityGroups\",\n\t\t\t\t\"ec2:DescribeVpcs\",\n\t\t\t\t\"route53:ChangeResourceRecordSets\",\n\t\t\t\t\"route53:ListResourceRecordSets\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t}\n\t]\n}" + pattern: ^arn:(aws|aws-cn|aws-us-gov):iam::[0-9]{12}:role\/.*$ + type: string + ingressARN: + description: "IngressARN is an ARN value referencing + the role in the VPC owner account that allows the\ningress + operator in the cluster account to create and manage + records in the private DNS\nhosted zone.\n\nThe + referenced role must have a trust relationship that + allows it to be assumed by the\ningress operator + role in the VPC creator account.\nExample:\n{\n\t + \"Version\": \"2012-10-17\",\n\t \"Statement\": + [\n\t \t{\n\t \t\t\"Sid\": \"Statement1\",\n\t \t\t\"Effect\": + \"Allow\",\n\t \t\t\"Principal\": {\n\t \t\t\t\"AWS\": + \"arn:aws:iam::[cluster-creator-account-id]:role/[infra-id]-openshift-ingress\"\n\t + \t\t},\n\t \t\t\"Action\": \"sts:AssumeRole\"\n\t + \t}\n\t ]\n}\n\nThe following is an example of the + policy document for this role.\n(Based on https://docs.openshift.com/rosa/rosa_install_access_delete_clusters/rosa-shared-vpc-config.html#rosa-sharing-vpc-dns-and-roles_rosa-shared-vpc-config)\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"route53:ListHostedZones\",\n\t\t\t\t\"route53:ListHostedZonesByName\",\n\t\t\t\t\"route53:ChangeTagsForResource\",\n\t\t\t\t\"route53:GetAccountLimit\",\n\t\t\t\t\"route53:GetChange\",\n\t\t\t\t\"route53:GetHostedZone\",\n\t\t\t\t\"route53:ListTagsForResource\",\n\t\t\t\t\"route53:UpdateHostedZoneComment\",\n\t\t\t\t\"tag:GetResources\",\n\t\t\t\t\"tag:UntagResources\"\n\t\t\t\t\"route53:ChangeResourceRecordSets\",\n\t\t\t\t\"route53:ListResourceRecordSets\"\n\t\t\t],\n\t\t\t\"Resource\": + \"*\"\n\t\t},\n\t]\n}" + pattern: ^arn:(aws|aws-cn|aws-us-gov):iam::[0-9]{12}:role\/.*$ + type: string + required: + - controlPlaneARN + - ingressARN + type: object + required: + - localZoneID + - rolesRef + type: object + required: + - region + - rolesRef + type: object + azure: + description: Azure defines azure specific settings + properties: + cloud: + default: AzurePublicCloud + description: 'Cloud is the cloud environment identifier, valid + values could be found here: https://github.com/Azure/go-autorest/blob/4c0e21ca2bbb3251fe7853e6f9df6397f53dd419/autorest/azure/environments.go#L33' + enum: + - AzurePublicCloud + - AzureUSGovernmentCloud + - AzureChinaCloud + - AzureGermanCloud + - AzureStackCloud + type: string + credentials: + description: |- + Credentials is the object containing existing Azure credentials needed for creating and managing cloud + infrastructure resources. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + location: + description: |- + Location is the Azure region in where all the cloud infrastructure resources will be created. + + Example: eastus + type: string + x-kubernetes-validations: + - message: Location is immutable + rule: self == oldSelf + resourceGroup: + default: default + description: |- + ResourceGroupName is the name of an existing resource group where all cloud resources created by the Hosted + Cluster are to be placed. The resource group is expected to exist under the same subscription as SubscriptionID. + + In ARO HCP, this will be the managed resource group where customer cloud resources will be created. + + Resource group naming requirements can be found here: https://azure.github.io/PSRule.Rules.Azure/en/rules/Azure.ResourceGroup.Name/. + + Example: if your resource group ID is /subscriptions//resourceGroups/, your + ResourceGroupName is . + pattern: ^[a-zA-Z0-9_()\-\.]{1,89}[a-zA-Z0-9_()\-]$ + type: string + x-kubernetes-validations: + - message: ResourceGroupName is immutable + rule: self == oldSelf + securityGroupID: + description: |- + SecurityGroupID is the ID of an existing security group on the SubnetID. This field is provided as part of the + configuration for the Azure cloud provider, aka Azure cloud controller manager (CCM). This security group is + expected to exist under the same subscription as SubscriptionID. + type: string + x-kubernetes-validations: + - message: SecurityGroupID is immutable + rule: self == oldSelf + subnetID: + description: |- + subnetID is the subnet ID of an existing subnet where the nodes in the nodepool will be created. This can be a + different subnet than the one listed in the HostedCluster, HostedCluster.Spec.Platform.Azure.SubnetID, but must + exist in the same network, HostedCluster.Spec.Platform.Azure.VnetID, and must exist under the same subscription ID, + HostedCluster.Spec.Platform.Azure.SubscriptionID. + subnetID is immutable once set. + The subnetID should be in the format `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}`. + The subscriptionId in the encryptionSetID must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + The resourceGroupName should be between 1 and 90 characters, consisting only of alphanumeric characters, hyphens, underscores, periods and parenthesis and must not end with a period (.) character. + The vnetName should be between 2 and 64 characters, consisting only of alphanumeric characters, hyphens, underscores and periods and must not end with either a period (.) or hyphen (-) character. + The subnetName should be between 1 and 80 characters, consisting only of alphanumeric characters, hyphens and underscores and must start with an alphanumeric character and must not end with a period (.) or hyphen (-) character. + maxLength: 355 + minLength: 1 + type: string + x-kubernetes-validations: + - message: encryptionSetID must be in the format `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}` + rule: size(self.split('/')) == 11 && self.matches('^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Network/virtualNetworks/.*/subnets/.*$') + - message: The resourceGroupName should be between 1 and 90 + characters, consisting only of alphanumeric characters, + hyphens, underscores, periods and parenthesis + rule: self.split('/')[4].matches('[a-zA-Z0-9-_\\(\\)\\.]{1,90}') + - message: the resourceGroupName in the subnetID must not + end with a period (.) character + rule: '!self.split(''/'')[4].endsWith(''.'')' + - message: The vnetName should be between 2 and 64 characters, + consisting only of alphanumeric characters, hyphens, underscores + and periods + rule: self.split('/')[8].matches('[a-zA-Z0-9-_\\.]{2,64}') + - message: the vnetName in the subnetID must not end with + either a period (.) or hyphen (-) character + rule: '!self.split(''/'')[8].endsWith(''.'') && !self.split(''/'')[8].endsWith(''-'')' + - message: The subnetName should be between 1 and 80 characters, + consisting only of alphanumeric characters, hyphens and + underscores and must start with an alphanumeric character + rule: self.split('/')[10].matches('[a-zA-Z0-9][a-zA-Z0-9-_\\.]{0,79}') + - message: the subnetName in the subnetID must not end with + a period (.) or hyphen (-) character + rule: '!self.split(''/'')[10].endsWith(''.'') && !self.split(''/'')[10].endsWith(''-'')' + - message: SubnetID is immutable + rule: self == oldSelf + subscriptionID: + description: SubscriptionID is a unique identifier for an + Azure subscription used to manage resources. + type: string + x-kubernetes-validations: + - message: SubscriptionID is immutable + rule: self == oldSelf + vnetID: + description: |- + VnetID is the ID of an existing VNET to use in creating VMs. The VNET can exist in a different resource group + other than the one specified in ResourceGroupName, but it must exist under the same subscription as + SubscriptionID. + + In ARO HCP, this will be the ID of the customer provided VNET. + + Example: /subscriptions//resourceGroups//providers/Microsoft.Network/virtualNetworks/ + type: string + x-kubernetes-validations: + - message: VnetID is immutable + rule: self == oldSelf + required: + - credentials + - location + - resourceGroup + - securityGroupID + - subnetID + - subscriptionID + - vnetID + type: object + ibmcloud: + description: IBMCloud defines IBMCloud specific settings for components + properties: + providerType: + description: ProviderType is a specific supported infrastructure + provider within IBM Cloud. + type: string + type: object + kubevirt: + description: KubeVirt defines KubeVirt specific settings for cluster + components. + properties: + baseDomainPassthrough: + description: |- + BaseDomainPassthrough toggles whether or not an automatically + generated base domain for the guest cluster should be used that + is a subdomain of the management cluster's *.apps DNS. + + For the KubeVirt platform, the basedomain can be autogenerated using + the *.apps domain of the management/infra hosting cluster + This makes the guest cluster's base domain a subdomain of the + hypershift infra/mgmt cluster's base domain. + + Example: + Infra/Mgmt cluster's DNS + Base: example.com + Cluster: mgmt-cluster.example.com + Apps: *.apps.mgmt-cluster.example.com + KubeVirt Guest cluster's DNS + Base: apps.mgmt-cluster.example.com + Cluster: guest.apps.mgmt-cluster.example.com + Apps: *.apps.guest.apps.mgmt-cluster.example.com + + This is possible using OCP wildcard routes + type: boolean + x-kubernetes-validations: + - message: baseDomainPassthrough is immutable + rule: self == oldSelf + credentials: + description: |- + Credentials defines the client credentials used when creating KubeVirt virtual machines. + Defining credentials is only necessary when the KubeVirt virtual machines are being placed + on a cluster separate from the one hosting the Hosted Control Plane components. + + The default behavior when Credentials is not defined is for the KubeVirt VMs to be placed on + the same cluster and namespace as the Hosted Control Plane. + properties: + infraKubeConfigSecret: + description: |- + InfraKubeConfigSecret is a reference to a secret that contains the kubeconfig for the external infra cluster + that will be used to host the KubeVirt virtual machines for this cluster. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + x-kubernetes-validations: + - message: infraKubeConfigSecret is immutable + rule: self == oldSelf + infraNamespace: + description: |- + InfraNamespace defines the namespace on the external infra cluster that is used to host the KubeVirt + virtual machines. This namespace must already exist before creating the HostedCluster and the kubeconfig + referenced in the InfraKubeConfigSecret must have access to manage the required resources within this + namespace. + type: string + x-kubernetes-validations: + - message: infraNamespace is immutable + rule: self == oldSelf + required: + - infraKubeConfigSecret + - infraNamespace + type: object + generateID: + description: |- + GenerateID is used to uniquely apply a name suffix to resources associated with + kubevirt infrastructure resources + maxLength: 11 + type: string + x-kubernetes-validations: + - message: Kubevirt GenerateID is immutable once set + rule: self == oldSelf + storageDriver: + description: |- + StorageDriver defines how the KubeVirt CSI driver exposes StorageClasses on + the infra cluster (hosting the VMs) to the guest cluster. + properties: + manual: + description: |- + Manual is used to explicitly define how the infra storageclasses are + mapped to guest storageclasses + properties: + storageClassMapping: + description: |- + StorageClassMapping maps StorageClasses on the infra cluster hosting + the KubeVirt VMs to StorageClasses that are made available within the + Guest Cluster. + + NOTE: It is possible that not all capabilities of an infra cluster's + storageclass will be present for the corresponding guest clusters storageclass. + items: + properties: + group: + description: Group contains which group this + mapping belongs to. + type: string + guestStorageClassName: + description: |- + GuestStorageClassName is the name that the corresponding storageclass will + be called within the guest cluster + type: string + infraStorageClassName: + description: |- + InfraStorageClassName is the name of the infra cluster storage class that + will be exposed to the guest. + type: string + required: + - guestStorageClassName + - infraStorageClassName + type: object + type: array + x-kubernetes-validations: + - message: storageClassMapping is immutable + rule: self == oldSelf + volumeSnapshotClassMapping: + items: + properties: + group: + description: Group contains which group this + mapping belongs to. + type: string + guestVolumeSnapshotClassName: + description: |- + GuestVolumeSnapshotClassName is the name that the corresponding volumeSnapshotClass will + be called within the guest cluster + type: string + infraVolumeSnapshotClassName: + description: |- + InfraStorageClassName is the name of the infra cluster volume snapshot class that + will be exposed to the guest. + type: string + required: + - guestVolumeSnapshotClassName + - infraVolumeSnapshotClassName + type: object + type: array + x-kubernetes-validations: + - message: volumeSnapshotClassMapping is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: storageDriver.Manual is immutable + rule: self == oldSelf + type: + default: Default + description: Type represents the type of kubevirt csi + driver configuration to use + enum: + - None + - Default + - Manual + type: string + x-kubernetes-validations: + - message: storageDriver.Type is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: storageDriver is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: Kubevirt GenerateID is required once set + rule: '!has(oldSelf.generateID) || has(self.generateID)' + powervs: + description: |- + PowerVS specifies configuration for clusters running on IBMCloud Power VS Service. + This field is immutable. Once set, It can't be changed. + properties: + accountID: + description: |- + AccountID is the IBMCloud account id. + This field is immutable. Once set, It can't be changed. + type: string + cisInstanceCRN: + description: |- + CISInstanceCRN is the IBMCloud CIS Service Instance's Cloud Resource Name + This field is immutable. Once set, It can't be changed. + pattern: '^crn:' + type: string + imageRegistryOperatorCloudCreds: + description: |- + ImageRegistryOperatorCloudCreds is a reference to a secret containing ibm cloud + credentials for image registry operator to get authenticated with ibm cloud. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + ingressOperatorCloudCreds: + description: |- + IngressOperatorCloudCreds is a reference to a secret containing ibm cloud + credentials for ingress operator to get authenticated with ibm cloud. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + kubeCloudControllerCreds: + description: | + KubeCloudControllerCreds is a reference to a secret containing cloud + credentials with permissions matching the cloud controller policy. + This field is immutable. Once set, It can't be changed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + nodePoolManagementCreds: + description: | + NodePoolManagementCreds is a reference to a secret containing cloud + credentials with permissions matching the node pool management policy. + This field is immutable. Once set, It can't be changed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + region: + description: |- + Region is the IBMCloud region in which the cluster resides. This configures the + OCP control plane cloud integrations, and is used by NodePool to resolve + the correct boot image for a given release. + This field is immutable. Once set, It can't be changed. + type: string + resourceGroup: + description: |- + ResourceGroup is the IBMCloud Resource Group in which the cluster resides. + This field is immutable. Once set, It can't be changed. + type: string + serviceInstanceID: + description: |- + ServiceInstance is the reference to the Power VS service on which the server instance(VM) will be created. + Power VS service is a container for all Power VS instances at a specific geographic region. + serviceInstance can be created via IBM Cloud catalog or CLI. + ServiceInstanceID is the unique identifier that can be obtained from IBM Cloud UI or IBM Cloud cli. + + More detail about Power VS service instance. + https://cloud.ibm.com/docs/power-iaas?topic=power-iaas-creating-power-virtual-server + + This field is immutable. Once set, It can't be changed. + type: string + storageOperatorCloudCreds: + description: |- + StorageOperatorCloudCreds is a reference to a secret containing ibm cloud + credentials for storage operator to get authenticated with ibm cloud. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + subnet: + description: |- + Subnet is the subnet to use for control plane cloud resources. + This field is immutable. Once set, It can't be changed. + properties: + id: + description: ID of resource + type: string + name: + description: Name of resource + type: string + type: object + vpc: + description: |- + VPC specifies IBM Cloud PowerVS Load Balancing configuration for the control + plane. + This field is immutable. Once set, It can't be changed. + properties: + name: + description: |- + Name for VPC to used for all the service load balancer. + This field is immutable. Once set, It can't be changed. + type: string + region: + description: |- + Region is the IBMCloud region in which VPC gets created, this VPC used for all the ingress traffic + into the OCP cluster. + This field is immutable. Once set, It can't be changed. + type: string + subnet: + description: |- + Subnet is the subnet to use for load balancer. + This field is immutable. Once set, It can't be changed. + type: string + zone: + description: |- + Zone is the availability zone where load balancer cloud resources are + created. + This field is immutable. Once set, It can't be changed. + type: string + required: + - name + - region + type: object + zone: + description: |- + Zone is the availability zone where control plane cloud resources are + created. + This field is immutable. Once set, It can't be changed. + type: string + required: + - accountID + - cisInstanceCRN + - imageRegistryOperatorCloudCreds + - ingressOperatorCloudCreds + - kubeCloudControllerCreds + - nodePoolManagementCreds + - region + - resourceGroup + - serviceInstanceID + - storageOperatorCloudCreds + - subnet + - vpc + - zone + type: object + type: + description: Type is the type of infrastructure provider for the + cluster. + type: string + x-kubernetes-validations: + - message: Type is immutable + rule: self == oldSelf + required: + - type + type: object + pullSecret: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + releaseImage: + description: ReleaseImage is the release image applied to the hosted + control plane. + type: string + secretEncryption: + description: |- + SecretEncryption contains metadata about the kubernetes secret encryption strategy being used for the + cluster when applicable. + properties: + aescbc: + description: AESCBC defines metadata about the AESCBC secret encryption + strategy + properties: + activeKey: + description: ActiveKey defines the active key used to encrypt + new secrets + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + backupKey: + description: |- + BackupKey defines the old key during the rotation process so previously created + secrets can continue to be decrypted until they are all re-encrypted with the active key. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - activeKey + type: object + kms: + description: KMS defines metadata about the kms secret encryption + strategy + properties: + aws: + description: AWS defines metadata about the configuration + of the AWS KMS Secret Encryption provider + properties: + activeKey: + description: ActiveKey defines the active key used to + encrypt new secrets + properties: + arn: + description: ARN is the Amazon Resource Name for the + encryption key + pattern: '^arn:' + type: string + required: + - arn + type: object + auth: + description: Auth defines metadata about the management + of credentials used to interact with AWS KMS + properties: + awsKms: + description: "The referenced role must have a trust + relationship that allows it to be assumed via web + identity.\nhttps://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html.\nExample:\n{\n\t\t\"Version\": + \"2012-10-17\",\n\t\t\"Statement\": [\n\t\t\t{\n\t\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\t\"Principal\": {\n\t\t\t\t\t\"Federated\": + \"{{ .ProviderARN }}\"\n\t\t\t\t},\n\t\t\t\t\t\"Action\": + \"sts:AssumeRoleWithWebIdentity\",\n\t\t\t\t\"Condition\": + {\n\t\t\t\t\t\"StringEquals\": {\n\t\t\t\t\t\t\"{{ + .ProviderName }}:sub\": {{ .ServiceAccounts }}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t]\n\t}\n\nAWSKMSARN + is an ARN value referencing a role appropriate for + managing the auth via the AWS KMS key.\n\nThe following + is an example of a valid policy document:\n\n{\n\t\"Version\": + \"2012-10-17\",\n\t\"Statement\": [\n \t{\n\t\t\t\"Effect\": + \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"kms:Encrypt\",\n\t\t\t\t\"kms:Decrypt\",\n\t\t\t\t\"kms:ReEncrypt*\",\n\t\t\t\t\"kms:GenerateDataKey*\",\n\t\t\t\t\"kms:DescribeKey\"\n\t\t\t],\n\t\t\t\"Resource\": + %q\n\t\t}\n\t]\n}" + type: string + required: + - awsKms + type: object + backupKey: + description: |- + BackupKey defines the old key during the rotation process so previously created + secrets can continue to be decrypted until they are all re-encrypted with the active key. + properties: + arn: + description: ARN is the Amazon Resource Name for the + encryption key + pattern: '^arn:' + type: string + required: + - arn + type: object + region: + description: Region contains the AWS region + type: string + required: + - activeKey + - auth + - region + type: object + azure: + description: Azure defines metadata about the configuration + of the Azure KMS Secret Encryption provider using Azure + key vault + properties: + activeKey: + description: ActiveKey defines the active key used to + encrypt new secrets + properties: + keyName: + description: KeyName is the name of the keyvault key + used for encrypt/decrypt + type: string + keyVaultName: + description: |- + KeyVaultName is the name of the keyvault. Must match criteria specified at https://docs.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name + Your Microsoft Entra application used to create the cluster must be authorized to access this keyvault, e.g using the AzureCLI: + `az keyvault set-policy -n $KEYVAULT_NAME --key-permissions decrypt encrypt --spn ` + type: string + keyVersion: + description: KeyVersion contains the version of the + key to use + type: string + required: + - keyName + - keyVaultName + - keyVersion + type: object + backupKey: + description: |- + BackupKey defines the old key during the rotation process so previously created + secrets can continue to be decrypted until they are all re-encrypted with the active key. + properties: + keyName: + description: KeyName is the name of the keyvault key + used for encrypt/decrypt + type: string + keyVaultName: + description: |- + KeyVaultName is the name of the keyvault. Must match criteria specified at https://docs.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name + Your Microsoft Entra application used to create the cluster must be authorized to access this keyvault, e.g using the AzureCLI: + `az keyvault set-policy -n $KEYVAULT_NAME --key-permissions decrypt encrypt --spn ` + type: string + keyVersion: + description: KeyVersion contains the version of the + key to use + type: string + required: + - keyName + - keyVaultName + - keyVersion + type: object + required: + - activeKey + type: object + ibmcloud: + description: IBMCloud defines metadata for the IBM Cloud KMS + encryption strategy + properties: + auth: + description: Auth defines metadata for how authentication + is done with IBM Cloud KMS + properties: + managed: + description: |- + Managed defines metadata around the service to service authentication strategy for the IBM Cloud + KMS system (all provider managed). + type: object + type: + description: Type defines the IBM Cloud KMS authentication + strategy + enum: + - Managed + - Unmanaged + type: string + unmanaged: + description: Unmanaged defines the auth metadata the + customer provides to interact with IBM Cloud KMS + properties: + credentials: + description: |- + Credentials should reference a secret with a key field of IBMCloudIAMAPIKeySecretKey that contains a apikey to + call IBM Cloud KMS APIs + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - credentials + type: object + required: + - type + type: object + keyList: + description: KeyList defines the list of keys used for + data encryption + items: + description: IBMCloudKMSKeyEntry defines metadata for + an IBM Cloud KMS encryption key + properties: + correlationID: + description: CorrelationID is an identifier used + to track all api call usage from hypershift + type: string + crkID: + description: CRKID is the customer rook key id + type: string + instanceID: + description: InstanceID is the id for the key protect + instance + type: string + keyVersion: + description: |- + KeyVersion is a unique number associated with the key. The number increments whenever a new + key is enabled for data encryption. + type: integer + url: + description: URL is the url to call key protect + apis over + pattern: ^https:// + type: string + required: + - correlationID + - crkID + - instanceID + - keyVersion + - url + type: object + type: array + region: + description: Region is the IBM Cloud region + type: string + required: + - auth + - keyList + - region + type: object + provider: + description: Provider defines the KMS provider + enum: + - IBMCloud + - AWS + - Azure + type: string + required: + - provider + type: object + type: + description: Type defines the type of kube secret encryption being + used + enum: + - kms + - aescbc + type: string + required: + - type + type: object + serviceAccountSigningKey: + description: |- + ServiceAccountSigningKey is a reference to a secret containing the private key + used by the service account token issuer. The secret is expected to contain + a single key named "key". If not specified, a service account signing key will + be generated automatically for the cluster. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + services: + description: |- + Services defines metadata about how control plane services are published + in the management cluster. + items: + description: |- + ServicePublishingStrategyMapping specifies how individual control plane services endpoints are published for consumption. + This includes APIServer;OAuthServer;Konnectivity;Ignition. + If a given service is not present in this list, it will be exposed publicly by default. + properties: + service: + description: |- + service identifies the type of service being published. + It can be APIServer;OAuthServer;Konnectivity;Ignition + OVNSbDb;OIDC are no-op and kept for backward compatibility. + This field is immutable. + enum: + - APIServer + - OAuthServer + - OIDC + - Konnectivity + - Ignition + - OVNSbDb + type: string + servicePublishingStrategy: + description: servicePublishingStrategy specifies how to publish + a service endpoint. + properties: + loadBalancer: + description: loadBalancer configures exposing a service + using a dedicated LoadBalancer. + properties: + hostname: + description: |- + hostname is the name of the DNS record that will be created pointing to the LoadBalancer and passed through to consumers of the service. + If omitted, the value will be inferred from the corev1.Service Load balancer type .status. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: hostname must be a valid domain name (e.g., + example.com) + rule: self.matches('^(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}$') + type: object + nodePort: + description: nodePort configures exposing a service using + a NodePort. + properties: + address: + description: address is the host/ip that the NodePort + service is exposed over. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: address must be a valid hostname, IPv4, or + IPv6 address + rule: isIP(self) || self.matches('^(([a-zA-Z0-9][-a-zA-Z0-9]*\\.)+[a-zA-Z]{2,}|localhost)$') + || self.matches('^((\\d{1,3}\\.){3}\\d{1,3})$') + || self.matches('^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$') + port: + description: |- + port is the port of the NodePort service. If <=0, the port is dynamically + assigned when the service is created. + format: int32 + type: integer + required: + - address + type: object + route: + description: |- + route configures exposing a service using a Route through and an ingress controller behind a cloud Load Balancer. + The specifics of the setup are platform dependent. + properties: + hostname: + description: |- + Hostname is the name of the DNS record that will be created pointing to the Route and passed through to consumers of the service. + If omitted, the value will be inferred from management ingress.Spec.Domain. + maxLength: 253 + minLength: 1 + type: string + x-kubernetes-validations: + - message: hostname must be a valid domain name (e.g., + example.com) + rule: self.matches('^(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}$') + type: object + type: + description: |- + type is the publishing strategy used for the service. + It can be LoadBalancer;NodePort;Route;None;S3 + enum: + - LoadBalancer + - NodePort + - Route + - None + - S3 + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: nodePort is required when type is NodePort, and forbidden + otherwise + rule: 'self.type == ''NodePort'' ? has(self.nodePort) : !has(self.nodePort)' + - message: only route is allowed when type is Route, and forbidden + otherwise + rule: 'self.type == ''Route'' ? !has(self.nodePort) && !has(self.loadBalancer) + : !has(self.route)' + - message: only loadBalancer is required when type is LoadBalancer, + and forbidden otherwise + rule: 'self.type == ''LoadBalancer'' ? !has(self.nodePort) + && !has(self.route) : !has(self.loadBalancer)' + - message: None does not allowed any configuration for loadBalancer, + nodePort, or route + rule: 'self.type == ''None'' ? !has(self.nodePort) && !has(self.route) + && !has(self.loadBalancer) : true' + - message: S3 does not allowed any configuration for loadBalancer, + nodePort, or route + rule: 'self.type == ''S3'' ? !has(self.nodePort) && !has(self.route) + && !has(self.loadBalancer) : true' + required: + - service + - servicePublishingStrategy + type: object + maxItems: 6 + minItems: 4 + type: array + sshKey: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + tolerations: + description: Tolerations when specified, define what custom tolerations + are added to the hcp pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + updateService: + description: |- + updateService may be used to specify the preferred upstream update service. + By default it will use the appropriate update service for the cluster and region. + type: string + required: + - dns + - etcd + - infraID + - issuerURL + - platform + - pullSecret + - releaseImage + - services + - sshKey + type: object + status: + description: HostedControlPlaneStatus defines the observed state of HostedControlPlane + properties: + conditions: + description: |- + Condition contains details for one aspect of the current state of the HostedControlPlane. + Current condition types are: "Available" + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controlPlaneEndpoint: + description: |- + ControlPlaneEndpoint contains the endpoint information by which + external clients can access the control plane. This is populated + after the infrastructure is ready. + properties: + host: + description: Host is the hostname on which the API server is serving. + type: string + port: + description: Port is the port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + externalManagedControlPlane: + default: true + description: |- + ExternalManagedControlPlane indicates to cluster-api that the control plane + is managed by an external service. + https://github.com/kubernetes-sigs/cluster-api/blob/65e5385bffd71bf4aad3cf34a537f11b217c7fab/controllers/machine_controller.go#L468 + type: boolean + initialized: + default: false + description: |- + Initialized denotes whether or not the control plane has + provided a kubeadm-config. + Once this condition is marked true, its value is never changed. See the Ready condition for an indication of + the current readiness of the cluster's control plane. + This satisfies CAPI contract https://github.com/kubernetes-sigs/cluster-api/blob/cd3a694deac89d5ebeb888307deaa61487207aa0/controllers/cluster_controller_phases.go#L238-L252 + type: boolean + kubeConfig: + description: |- + KubeConfig is a reference to the secret containing the default kubeconfig + for this control plane. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + kubeadminPassword: + description: |- + KubeadminPassword is a reference to the secret containing the initial kubeadmin password + for the guest cluster. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + lastReleaseImageTransitionTime: + description: |- + lastReleaseImageTransitionTime is the time of the last update to the current + releaseImage property. + + Deprecated: Use versionStatus.history[0].startedTime instead. + format: date-time + type: string + nodeCount: + description: NodeCount tracks the number of nodes in the HostedControlPlane. + type: integer + oauthCallbackURLTemplate: + description: |- + OAuthCallbackURLTemplate contains a template for the URL to use as a callback + for identity providers. The [identity-provider-name] placeholder must be replaced + with the name of an identity provider defined on the HostedCluster. + This is populated after the infrastructure is ready. + type: string + platform: + description: Platform contains platform-specific status of the HostedCluster + properties: + aws: + description: AWSPlatformStatus contains status specific to the + AWS platform + properties: + defaultWorkerSecurityGroupID: + description: |- + DefaultWorkerSecurityGroupID is the ID of a security group created by + the control plane operator. It is always added to worker machines in + addition to any security groups specified in the NodePool. + type: string + type: object + type: object + ready: + default: false + description: |- + Ready denotes that the HostedControlPlane API Server is ready to + receive requests + This satisfies CAPI contract https://github.com/kubernetes-sigs/cluster-api/blob/cd3a694deac89d5ebeb888307deaa61487207aa0/controllers/cluster_controller_phases.go#L226-L230 + type: boolean + releaseImage: + description: |- + ReleaseImage is the release image applied to the hosted control plane. + + Deprecated: Use versionStatus.desired.image instead. + type: string + version: + description: |- + Version is the semantic version of the release applied by + the hosted control plane operator + + Deprecated: Use versionStatus.desired.version instead. + type: string + versionStatus: + description: |- + versionStatus is the status of the release version applied by the + hosted control plane operator. + properties: + availableUpdates: + description: |- + availableUpdates contains updates recommended for this + cluster. Updates which appear in conditionalUpdates but not in + availableUpdates may expose this cluster to known issues. This list + may be empty if no updates are recommended, if the update service + is unavailable, or if an invalid channel has been specified. + items: + description: Release represents an OpenShift release image and + associated metadata. + properties: + channels: + description: |- + channels is the set of Cincinnati channels to which the release + currently belongs. + items: + type: string + type: array + x-kubernetes-list-type: set + image: + description: |- + image is a container image location that contains the update. When this + field is part of spec, image is optional if version is specified and the + availableUpdates field contains a matching version. + type: string + url: + description: |- + url contains information about this release. This URL is set by + the 'url' metadata property on a release or the metadata returned by + the update API and should be displayed as a link in user + interfaces. The URL field may not be set for test or nightly + releases. + type: string + version: + description: |- + version is a semantic version identifying the update version. When this + field is part of spec, version is optional if image is specified. + type: string + required: + - image + - version + type: object + nullable: true + type: array + conditionalUpdates: + description: |- + conditionalUpdates contains the list of updates that may be + recommended for this cluster if it meets specific required + conditions. Consumers interested in the set of updates that are + actually recommended for this cluster should use + availableUpdates. This list may be empty if no updates are + recommended, if the update service is unavailable, or if an empty + or invalid channel has been specified. + items: + description: |- + ConditionalUpdate represents an update which is recommended to some + clusters on the version the current cluster is reconciling, but which + may not be recommended for the current cluster. + properties: + conditions: + description: |- + conditions represents the observations of the conditional update's + current status. Known types are: + * Recommended, for whether the update is recommended for the current cluster. + items: + description: Condition contains details for one aspect + of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, + False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in + foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + release: + description: release is the target of the update. + properties: + channels: + description: |- + channels is the set of Cincinnati channels to which the release + currently belongs. + items: + type: string + type: array + x-kubernetes-list-type: set + image: + description: |- + image is a container image location that contains the update. When this + field is part of spec, image is optional if version is specified and the + availableUpdates field contains a matching version. + type: string + url: + description: |- + url contains information about this release. This URL is set by + the 'url' metadata property on a release or the metadata returned by + the update API and should be displayed as a link in user + interfaces. The URL field may not be set for test or nightly + releases. + type: string + version: + description: |- + version is a semantic version identifying the update version. When this + field is part of spec, version is optional if image is specified. + type: string + required: + - image + - version + type: object + risks: + description: |- + risks represents the range of issues associated with + updating to the target release. The cluster-version + operator will evaluate all entries, and only recommend the + update if there is at least one entry and all entries + recommend the update. + items: + description: |- + ConditionalUpdateRisk represents a reason and cluster-state + for not recommending a conditional update. + properties: + matchingRules: + description: |- + matchingRules is a slice of conditions for deciding which + clusters match the risk and which do not. The slice is + ordered by decreasing precedence. The cluster-version + operator will walk the slice in order, and stop after the + first it can successfully evaluate. If no condition can be + successfully evaluated, the update will not be recommended. + items: + description: |- + ClusterCondition is a union of typed cluster conditions. The 'type' + property determines which of the type-specific properties are relevant. + When evaluated on a cluster, the condition may match, not match, or + fail to evaluate. + properties: + promql: + description: promQL represents a cluster condition + based on PromQL. + properties: + promql: + description: |- + PromQL is a PromQL query classifying clusters. This query + query should return a 1 in the match case and a 0 in the + does-not-match case. Queries which return no time + series, or which return values besides 0 or 1, are + evaluation failures. + type: string + required: + - promql + type: object + type: + description: |- + type represents the cluster-condition type. This defines + the members and semantics of any additional properties. + enum: + - Always + - PromQL + type: string + required: + - type + type: object + minItems: 1 + type: array + x-kubernetes-list-type: atomic + message: + description: |- + message provides additional information about the risk of + updating, in the event that matchingRules match the cluster + state. This is only to be consumed by humans. It may + contain Line Feed characters (U+000A), which should be + rendered as new lines. + minLength: 1 + type: string + name: + description: |- + name is the CamelCase reason for not recommending a + conditional update, in the event that matchingRules match the + cluster state. + minLength: 1 + type: string + url: + description: url contains information about this risk. + format: uri + minLength: 1 + type: string + required: + - matchingRules + - message + - name + - url + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - release + - risks + type: object + type: array + x-kubernetes-list-type: atomic + desired: + description: |- + desired is the version that the cluster is reconciling towards. + If the cluster is not yet fully initialized desired will be set + with the information available, which may be an image or a tag. + properties: + channels: + description: |- + channels is the set of Cincinnati channels to which the release + currently belongs. + items: + type: string + type: array + x-kubernetes-list-type: set + image: + description: |- + image is a container image location that contains the update. When this + field is part of spec, image is optional if version is specified and the + availableUpdates field contains a matching version. + type: string + url: + description: |- + url contains information about this release. This URL is set by + the 'url' metadata property on a release or the metadata returned by + the update API and should be displayed as a link in user + interfaces. The URL field may not be set for test or nightly + releases. + type: string + version: + description: |- + version is a semantic version identifying the update version. When this + field is part of spec, version is optional if image is specified. + type: string + required: + - image + - version + type: object + history: + description: |- + history contains a list of the most recent versions applied to the cluster. + This value may be empty during cluster startup, and then will be updated + when a new update is being applied. The newest update is first in the + list and it is ordered by recency. Updates in the history have state + Completed if the rollout completed - if an update was failing or halfway + applied the state will be Partial. Only a limited amount of update history + is preserved. + items: + description: UpdateHistory is a single attempted update to the + cluster. + properties: + acceptedRisks: + description: |- + acceptedRisks records risks which were accepted to initiate the update. + For example, it may menition an Upgradeable=False or missing signature + that was overriden via desiredUpdate.force, or an update that was + initiated despite not being in the availableUpdates set of recommended + update targets. + type: string + completionTime: + description: |- + completionTime, if set, is when the update was fully applied. The update + that is currently being applied will have a null completion time. + Completion time will always be set for entries that are not the current + update (usually to the started time of the next update). + format: date-time + nullable: true + type: string + image: + description: |- + image is a container image location that contains the update. This value + is always populated. + type: string + startedTime: + description: startedTime is the time at which the update + was started. + format: date-time + type: string + state: + description: |- + state reflects whether the update was fully applied. The Partial state + indicates the update is not fully applied, while the Completed state + indicates the update was successfully rolled out at least once (all + parts of the update successfully applied). + type: string + verified: + description: |- + verified indicates whether the provided update was properly verified + before it was installed. If this is false the cluster may not be trusted. + Verified does not cover upgradeable checks that depend on the cluster + state at the time when the update target was accepted. + type: boolean + version: + description: |- + version is a semantic version identifying the update version. If the + requested image does not define a version, or if a failure occurs + retrieving the image, this value may be empty. + type: string + required: + - completionTime + - image + - startedTime + - state + - verified + type: object + type: array + observedGeneration: + description: |- + observedGeneration reports which version of the spec is being synced. + If this value is not equal to metadata.generation, then the desired + and conditions fields may represent a previous version. + format: int64 + type: integer + required: + - availableUpdates + - desired + - observedGeneration + type: object + required: + - initialized + - ready + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/client/applyconfiguration/hypershift/v1beta1/autonode.go b/client/applyconfiguration/hypershift/v1beta1/autonode.go new file mode 100644 index 0000000000..374b390290 --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/autonode.go @@ -0,0 +1,38 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// AutoNodeApplyConfiguration represents an declarative configuration of the AutoNode type for use +// with apply. +type AutoNodeApplyConfiguration struct { + Provisioner *ProvisionerConfigApplyConfiguration `json:"provisionerConfig,omitempty"` +} + +// AutoNodeApplyConfiguration constructs an declarative configuration of the AutoNode type for use with +// apply. +func AutoNode() *AutoNodeApplyConfiguration { + return &AutoNodeApplyConfiguration{} +} + +// WithProvisioner sets the Provisioner field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Provisioner field is set to the value of the last call. +func (b *AutoNodeApplyConfiguration) WithProvisioner(value *ProvisionerConfigApplyConfiguration) *AutoNodeApplyConfiguration { + b.Provisioner = value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/hostedclusterspec.go b/client/applyconfiguration/hypershift/v1beta1/hostedclusterspec.go index 3abc109f56..75b1cbfe45 100644 --- a/client/applyconfiguration/hypershift/v1beta1/hostedclusterspec.go +++ b/client/applyconfiguration/hypershift/v1beta1/hostedclusterspec.go @@ -38,6 +38,7 @@ type HostedClusterSpecApplyConfiguration struct { DNS *DNSSpecApplyConfiguration `json:"dns,omitempty"` Networking *ClusterNetworkingApplyConfiguration `json:"networking,omitempty"` Autoscaling *ClusterAutoscalingApplyConfiguration `json:"autoscaling,omitempty"` + AutoNode *AutoNodeApplyConfiguration `json:"autoNode,omitempty"` Etcd *EtcdSpecApplyConfiguration `json:"etcd,omitempty"` Services []ServicePublishingStrategyMappingApplyConfiguration `json:"services,omitempty"` PullSecret *corev1.LocalObjectReference `json:"pullSecret,omitempty"` @@ -159,6 +160,14 @@ func (b *HostedClusterSpecApplyConfiguration) WithAutoscaling(value *ClusterAuto return b } +// WithAutoNode sets the AutoNode field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AutoNode field is set to the value of the last call. +func (b *HostedClusterSpecApplyConfiguration) WithAutoNode(value *AutoNodeApplyConfiguration) *HostedClusterSpecApplyConfiguration { + b.AutoNode = value + return b +} + // WithEtcd sets the Etcd field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the Etcd field is set to the value of the last call. diff --git a/client/applyconfiguration/hypershift/v1beta1/hostedcontrolplanespec.go b/client/applyconfiguration/hypershift/v1beta1/hostedcontrolplanespec.go index e2a39cd276..091a5679b7 100644 --- a/client/applyconfiguration/hypershift/v1beta1/hostedcontrolplanespec.go +++ b/client/applyconfiguration/hypershift/v1beta1/hostedcontrolplanespec.go @@ -53,6 +53,7 @@ type HostedControlPlaneSpecApplyConfiguration struct { PausedUntil *string `json:"pausedUntil,omitempty"` OLMCatalogPlacement *hypershiftv1beta1.OLMCatalogPlacement `json:"olmCatalogPlacement,omitempty"` Autoscaling *ClusterAutoscalingApplyConfiguration `json:"autoscaling,omitempty"` + AutoNode *AutoNodeApplyConfiguration `json:"autoNode,omitempty"` NodeSelector map[string]string `json:"nodeSelector,omitempty"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` Labels map[string]string `json:"labels,omitempty"` @@ -290,6 +291,14 @@ func (b *HostedControlPlaneSpecApplyConfiguration) WithAutoscaling(value *Cluste return b } +// WithAutoNode sets the AutoNode field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AutoNode field is set to the value of the last call. +func (b *HostedControlPlaneSpecApplyConfiguration) WithAutoNode(value *AutoNodeApplyConfiguration) *HostedControlPlaneSpecApplyConfiguration { + b.AutoNode = value + return b +} + // WithNodeSelector puts the entries into the NodeSelector field in the declarative configuration // and returns the receiver, so that objects can be build by chaining "With" function invocations. // If called multiple times, the entries provided by each call will be put on the NodeSelector field, diff --git a/client/applyconfiguration/hypershift/v1beta1/karpenterawsconfig.go b/client/applyconfiguration/hypershift/v1beta1/karpenterawsconfig.go new file mode 100644 index 0000000000..8d8cb1291d --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/karpenterawsconfig.go @@ -0,0 +1,38 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// KarpenterAWSConfigApplyConfiguration represents an declarative configuration of the KarpenterAWSConfig type for use +// with apply. +type KarpenterAWSConfigApplyConfiguration struct { + RoleARN *string `json:"roleARN,omitempty"` +} + +// KarpenterAWSConfigApplyConfiguration constructs an declarative configuration of the KarpenterAWSConfig type for use with +// apply. +func KarpenterAWSConfig() *KarpenterAWSConfigApplyConfiguration { + return &KarpenterAWSConfigApplyConfiguration{} +} + +// WithRoleARN sets the RoleARN field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the RoleARN field is set to the value of the last call. +func (b *KarpenterAWSConfigApplyConfiguration) WithRoleARN(value string) *KarpenterAWSConfigApplyConfiguration { + b.RoleARN = &value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/karpenterconfig.go b/client/applyconfiguration/hypershift/v1beta1/karpenterconfig.go new file mode 100644 index 0000000000..cc2c078a92 --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/karpenterconfig.go @@ -0,0 +1,51 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" +) + +// KarpenterConfigApplyConfiguration represents an declarative configuration of the KarpenterConfig type for use +// with apply. +type KarpenterConfigApplyConfiguration struct { + Platform *v1beta1.PlatformType `json:"platform,omitempty"` + AWS *KarpenterAWSConfigApplyConfiguration `json:"aws,omitempty"` +} + +// KarpenterConfigApplyConfiguration constructs an declarative configuration of the KarpenterConfig type for use with +// apply. +func KarpenterConfig() *KarpenterConfigApplyConfiguration { + return &KarpenterConfigApplyConfiguration{} +} + +// WithPlatform sets the Platform field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Platform field is set to the value of the last call. +func (b *KarpenterConfigApplyConfiguration) WithPlatform(value v1beta1.PlatformType) *KarpenterConfigApplyConfiguration { + b.Platform = &value + return b +} + +// WithAWS sets the AWS field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AWS field is set to the value of the last call. +func (b *KarpenterConfigApplyConfiguration) WithAWS(value *KarpenterAWSConfigApplyConfiguration) *KarpenterConfigApplyConfiguration { + b.AWS = value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/provisionerconfig.go b/client/applyconfiguration/hypershift/v1beta1/provisionerconfig.go new file mode 100644 index 0000000000..7d972793ff --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/provisionerconfig.go @@ -0,0 +1,51 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" +) + +// ProvisionerConfigApplyConfiguration represents an declarative configuration of the ProvisionerConfig type for use +// with apply. +type ProvisionerConfigApplyConfiguration struct { + Name *v1beta1.Provisioner `json:"name,omitempty"` + Karpenter *KarpenterConfigApplyConfiguration `json:"karpenter,omitempty"` +} + +// ProvisionerConfigApplyConfiguration constructs an declarative configuration of the ProvisionerConfig type for use with +// apply. +func ProvisionerConfig() *ProvisionerConfigApplyConfiguration { + return &ProvisionerConfigApplyConfiguration{} +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *ProvisionerConfigApplyConfiguration) WithName(value v1beta1.Provisioner) *ProvisionerConfigApplyConfiguration { + b.Name = &value + return b +} + +// WithKarpenter sets the Karpenter field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Karpenter field is set to the value of the last call. +func (b *ProvisionerConfigApplyConfiguration) WithKarpenter(value *KarpenterConfigApplyConfiguration) *ProvisionerConfigApplyConfiguration { + b.Karpenter = value + return b +} diff --git a/client/applyconfiguration/utils.go b/client/applyconfiguration/utils.go index 62d2b51099..d982c3a874 100644 --- a/client/applyconfiguration/utils.go +++ b/client/applyconfiguration/utils.go @@ -56,6 +56,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &hypershiftv1beta1.APIEndpointApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("APIServerNetworking"): return &hypershiftv1beta1.APIServerNetworkingApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("AutoNode"): + return &hypershiftv1beta1.AutoNodeApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("AWSCloudProviderConfig"): return &hypershiftv1beta1.AWSCloudProviderConfigApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("AWSKMSAuthSpec"): @@ -150,6 +152,10 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &hypershiftv1beta1.ImageContentSourceApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("InPlaceUpgrade"): return &hypershiftv1beta1.InPlaceUpgradeApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("KarpenterAWSConfig"): + return &hypershiftv1beta1.KarpenterAWSConfigApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("KarpenterConfig"): + return &hypershiftv1beta1.KarpenterConfigApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("KMSSpec"): return &hypershiftv1beta1.KMSSpecApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("KubeconfigSecretRef"): @@ -244,6 +250,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &hypershiftv1beta1.PowerVSResourceReferenceApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("PowerVSVPC"): return &hypershiftv1beta1.PowerVSVPCApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("ProvisionerConfig"): + return &hypershiftv1beta1.ProvisionerConfigApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("Release"): return &hypershiftv1beta1.ReleaseApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("ReplaceUpgrade"): diff --git a/cmd/cluster/aws/create.go b/cmd/cluster/aws/create.go index 877ee72f6a..1b0386f915 100644 --- a/cmd/cluster/aws/create.go +++ b/cmd/cluster/aws/create.go @@ -47,6 +47,7 @@ type RawCreateOptions struct { VPCOwnerCredentials awsutil.AWSCredentialsOptions PrivateZonesInClusterAccount bool PublicOnly bool + AutoNode bool } // validatedCreateOptions is a private wrapper that enforces a call of Validate() before Complete() can be invoked. @@ -239,6 +240,19 @@ func (o *CreateOptions) ApplyPlatformSpecifics(cluster *hyperv1.HostedCluster) e EndpointAccess: endpointAccess, }, } + if o.AutoNode { + cluster.Spec.AutoNode = &hyperv1.AutoNode{ + Provisioner: &hyperv1.ProvisionerConfig{ + Name: hyperv1.ProvisionerKarpeneter, + Karpenter: &hyperv1.KarpenterConfig{ + Platform: hyperv1.AWSPlatform, + AWS: &hyperv1.KarpenterAWSConfig{ + RoleARN: o.iamInfo.KarpenterRoleARN, + }, + }, + }, + } + } if o.iamInfo.SharedIngressRoleARN != "" && o.iamInfo.SharedControlPlaneRoleARN != "" { cluster.Spec.Platform.AWS.SharedVPC = &hyperv1.AWSSharedVPC{ @@ -389,6 +403,7 @@ func bindCoreOptions(opts *RawCreateOptions, flags *flag.FlagSet) { flags.StringVar(&opts.CredentialSecretName, "secret-creds", opts.CredentialSecretName, "A Kubernetes secret with needed AWS platform credentials: sts-creds, pull-secret, and a base-domain value. The secret must exist in the supplied \"--namespace\". If a value is provided through the flag '--pull-secret', that value will override the pull-secret value in 'secret-creds'.") flags.StringVar(&opts.IssuerURL, "oidc-issuer-url", "", "The OIDC provider issuer URL") flags.BoolVar(&opts.MultiArch, "multi-arch", opts.MultiArch, "If true, this flag indicates the Hosted Cluster will support multi-arch NodePools and will perform additional validation checks to ensure a multi-arch release image or stream was used.") + flags.BoolVar(&opts.AutoNode, "auto-node", opts.AutoNode, "If true, this flag indicates the Hosted Cluster will support AutoNode feature.") flags.StringVar(&opts.VPCCIDR, "vpc-cidr", opts.VPCCIDR, "The CIDR to use for the cluster VPC (mask must be 16)") flags.BoolVar(&opts.PrivateZonesInClusterAccount, "private-zones-in-cluster-account", opts.PrivateZonesInClusterAccount, "In shared VPC infrastructure, create private hosted zones in cluster account") flags.BoolVar(&opts.PublicOnly, "public-only", opts.PublicOnly, "If true, creates a cluster that does not have private subnets or NAT gateway and assigns public IPs to all instances.") @@ -467,6 +482,7 @@ func CreateIAMOptions(awsOpts *ValidatedCreateOptions, infra *awsinfra.CreateInf KMSKeyARN: awsOpts.EtcdKMSKeyARN, VPCOwnerCredentialsOpts: awsOpts.VPCOwnerCredentials, PrivateZonesInClusterAccount: awsOpts.PrivateZonesInClusterAccount, + CreateKarpenterRoleARN: awsOpts.AutoNode, } } diff --git a/cmd/infra/aws/create_iam.go b/cmd/infra/aws/create_iam.go index 590c5f1ecd..9d54bd6680 100644 --- a/cmd/infra/aws/create_iam.go +++ b/cmd/infra/aws/create_iam.go @@ -41,7 +41,8 @@ type CreateIAMOptions struct { CredentialsSecretData *util.CredentialsSecretData - additionalIAMTags []*iam.Tag + additionalIAMTags []*iam.Tag + CreateKarpenterRoleARN bool } type CreateIAMOutput struct { @@ -55,6 +56,8 @@ type CreateIAMOutput struct { SharedIngressRoleARN string `json:"sharedIngressRoleARN,omitempty"` SharedControlPlaneRoleARN string `json:"sharedControlPlaneRoleARN,omitempty"` + + KarpenterRoleARN string `json:"karpenterRoleARN,omitempty"` } func NewCreateIAMCommand() *cobra.Command { diff --git a/cmd/infra/aws/ec2.go b/cmd/infra/aws/ec2.go index 24e2d598fa..b3315290f0 100644 --- a/cmd/infra/aws/ec2.go +++ b/cmd/infra/aws/ec2.go @@ -11,9 +11,9 @@ import ( "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/go-logr/logr" "github.com/openshift/hypershift/cmd/util" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" ) const ( @@ -215,14 +215,20 @@ func (o *CreateInfraOptions) existingDHCPOptions(client ec2iface.EC2API) (string } func (o *CreateInfraOptions) CreatePrivateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID string, zone string, cidr string) (string, error) { - return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-private-%s", o.InfraID, zone), tagNameSubnetInternalELB) + karpenterDiscoveryTag := []*ec2.Tag{ + { + Key: ptr.To("karpenter.sh/discovery"), + Value: ptr.To(o.InfraID), + }, + } + return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-private-%s", o.InfraID, zone), tagNameSubnetInternalELB, karpenterDiscoveryTag) } func (o *CreateInfraOptions) CreatePublicSubnet(l logr.Logger, client ec2iface.EC2API, vpcID string, zone string, cidr string) (string, error) { - return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-public-%s", o.InfraID, zone), tagNameSubnetPublicELB) + return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-public-%s", o.InfraID, zone), tagNameSubnetPublicELB, nil) } -func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID, zone, cidr, name, scopeTag string) (string, error) { +func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID, zone, cidr, name, scopeTag string, additionalTags []*ec2.Tag) (string, error) { subnetID, err := o.existingSubnet(client, name) if err != nil { return "", err @@ -236,6 +242,9 @@ func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API, Key: aws.String(scopeTag), Value: aws.String("1"), }) + if additionalTags != nil { + tagSpec[0].Tags = append(tagSpec[0].Tags, additionalTags...) + } result, err := client.CreateSubnet(&ec2.CreateSubnetInput{ AvailabilityZone: aws.String(zone), diff --git a/cmd/infra/aws/iam.go b/cmd/infra/aws/iam.go index 6de3f23ee5..da410da7d0 100644 --- a/cmd/infra/aws/iam.go +++ b/cmd/infra/aws/iam.go @@ -209,6 +209,222 @@ var ( }`, } + // { + // "Action": [ + // "*" + // ], + // "Resource": [ + // "*" + // ], + // "Effect": "Allow" + // } + karpenterPolicy = policyBinding{ + name: "karpenter", + serviceAccounts: []string{"system:serviceaccount:kube-system:karpenter"}, + policy: `{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowScopedEC2InstanceAccessActions", + "Effect": "Allow", + "Resource": [ + "arn:*:ec2:*::image/*", + "arn:*:ec2:*::snapshot/*", + "arn:*:ec2:*:*:security-group/*", + "arn:*:ec2:*:*:subnet/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + }, + { + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:*:ec2:*:*:launch-template/*", + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + }, + { + "Sid": "AllowScopedEC2InstanceActionsWithTags", + "Effect": "Allow", + "Resource": [ + "arn:*:ec2:*:*:fleet/*", + "arn:*:ec2:*:*:instance/*", + "arn:*:ec2:*:*:volume/*", + "arn:*:ec2:*:*:network-interface/*", + "arn:*:ec2:*:*:launch-template/*", + "arn:*:ec2:*:*:spot-instances-request/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate" + ], + "Condition": { + "StringLike": { + "aws:RequestTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowScopedResourceCreationTagging", + "Effect": "Allow", + "Resource": [ + "arn:*:ec2:*:*:fleet/*", + "arn:*:ec2:*:*:instance/*", + "arn:*:ec2:*:*:volume/*", + "arn:*:ec2:*:*:network-interface/*", + "arn:*:ec2:*:*:launch-template/*", + "arn:*:ec2:*:*:spot-instances-request/*" + ], + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "ec2:CreateAction": [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate" + ] + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowScopedResourceTagging", + "Effect": "Allow", + "Resource": "arn:*:ec2:*:*:instance/*", + "Action": "ec2:CreateTags", + "Condition": { + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowScopedDeletion", + "Effect": "Allow", + "Resource": [ + "arn:*:ec2:*:*:instance/*", + "arn:*:ec2:*:*:launch-template/*" + ], + "Action": [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" + ], + "Condition": { + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowRegionalReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ] + }, + { + "Sid": "AllowSSMReadActions", + "Effect": "Allow", + "Resource": "arn:*:ssm:*::parameter/aws/service/*", + "Action": "ssm:GetParameter" + }, + { + "Sid": "AllowPricingReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "pricing:GetProducts" + }, + { + "Sid": "AllowInterruptionQueueActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] + }, + { + "Sid": "AllowPassingInstanceRole", + "Effect": "Allow", + "Resource": "arn:*:iam::*:role/*", + "Action": "iam:PassRole", + "Condition": { + "StringEquals": { + "iam:PassedToService": [ + "ec2.amazonaws.com", + "ec2.amazonaws.com.cn" + ] + } + } + }, + { + "Sid": "AllowScopedInstanceProfileCreationActions", + "Effect": "Allow", + "Resource": "arn:*:iam::*:instance-profile/*", + "Action": [ + "iam:CreateInstanceProfile" + ], + "Condition": { + "StringLike": { + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileTagActions", + "Effect": "Allow", + "Resource": "arn:*:iam::*:instance-profile/*", + "Action": [ + "iam:TagInstanceProfile" + ], + "Condition": { + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*", + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileActions", + "Effect": "Allow", + "Resource": "arn:*:iam::*:instance-profile/*", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ], + "Condition": { + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowInstanceProfileReadActions", + "Effect": "Allow", + "Resource": "arn:*:iam::*:instance-profile/*", + "Action": "iam:GetInstanceProfile" + } + ] + }`, + } + nodePoolPolicy = policyBinding{ name: "node-pool", serviceAccounts: []string{"system:serviceaccount:kube-system:capa-controller-manager"}, @@ -628,6 +844,12 @@ func (o *CreateIAMOptions) CreateOIDCResources(iamClient iamiface.IAMAPI, logger &output.Roles.ControlPlaneOperatorARN: controlPlaneOperatorPolicy(o.LocalZoneID, sharedVPC), &output.Roles.NetworkARN: cloudNetworkConfigControllerPolicy, } + + if o.CreateKarpenterRoleARN { + bindings[&output.KarpenterRoleARN] = karpenterPolicy + + } + if len(o.KMSKeyARN) > 0 { bindings[&output.KMSProviderRoleARN] = kmsProviderPolicy(o.KMSKeyARN) } diff --git a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-CustomNoUpgrade.crd.yaml b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-CustomNoUpgrade.crd.yaml index 22bdd43f2d..d51b376b68 100644 --- a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-CustomNoUpgrade.crd.yaml @@ -113,6 +113,51 @@ spec: type: string type: object x-kubernetes-map-type: atomic + autoNode: + description: autoNode specifies the configuration for the autoNode + feature. + properties: + provisionerConfig: + description: provisioner is the implementation used for Node auto + provisioning. + properties: + karpenter: + description: karpenter specifies the configuration for the + Karpenter provisioner. + properties: + aws: + description: aws specifies the AWS-specific configuration + for Karpenter. + properties: + roleARN: + description: arn specifies the ARN of the Karpenter + provisioner. + type: string + required: + - roleARN + type: object + platform: + description: platform specifies the platform-specific + configuration for Karpenter. + type: string + required: + - platform + type: object + name: + allOf: + - enum: + - Karpenter + - enum: + - Karpenter + description: name specifies the name of the provisioner to + use. + type: string + required: + - name + type: object + required: + - provisionerConfig + type: object autoscaling: description: |- autoscaling specifies auto-scaling behavior that applies to all NodePools diff --git a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-TechPreviewNoUpgrade.crd.yaml index b0625a4e76..b93a60db3c 100644 --- a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-TechPreviewNoUpgrade.crd.yaml @@ -113,6 +113,51 @@ spec: type: string type: object x-kubernetes-map-type: atomic + autoNode: + description: autoNode specifies the configuration for the autoNode + feature. + properties: + provisionerConfig: + description: provisioner is the implementation used for Node auto + provisioning. + properties: + karpenter: + description: karpenter specifies the configuration for the + Karpenter provisioner. + properties: + aws: + description: aws specifies the AWS-specific configuration + for Karpenter. + properties: + roleARN: + description: arn specifies the ARN of the Karpenter + provisioner. + type: string + required: + - roleARN + type: object + platform: + description: platform specifies the platform-specific + configuration for Karpenter. + type: string + required: + - platform + type: object + name: + allOf: + - enum: + - Karpenter + - enum: + - Karpenter + description: name specifies the name of the provisioner to + use. + type: string + required: + - name + type: object + required: + - provisionerConfig + type: object autoscaling: description: |- autoscaling specifies auto-scaling behavior that applies to all NodePools diff --git a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-CustomNoUpgrade.crd.yaml b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-CustomNoUpgrade.crd.yaml index 16b74ef7de..1333aa12a9 100644 --- a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-CustomNoUpgrade.crd.yaml @@ -81,6 +81,51 @@ spec: type: string type: object x-kubernetes-map-type: atomic + autoNode: + description: autoNode specifies the configuration for the autoNode + feature. + properties: + provisionerConfig: + description: provisioner is the implementation used for Node auto + provisioning. + properties: + karpenter: + description: karpenter specifies the configuration for the + Karpenter provisioner. + properties: + aws: + description: aws specifies the AWS-specific configuration + for Karpenter. + properties: + roleARN: + description: arn specifies the ARN of the Karpenter + provisioner. + type: string + required: + - roleARN + type: object + platform: + description: platform specifies the platform-specific + configuration for Karpenter. + type: string + required: + - platform + type: object + name: + allOf: + - enum: + - Karpenter + - enum: + - Karpenter + description: name specifies the name of the provisioner to + use. + type: string + required: + - name + type: object + required: + - provisionerConfig + type: object autoscaling: description: |- Autoscaling specifies auto-scaling behavior that applies to all NodePools diff --git a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-TechPreviewNoUpgrade.crd.yaml index cfad27a75e..766cc96b04 100644 --- a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-TechPreviewNoUpgrade.crd.yaml @@ -81,6 +81,51 @@ spec: type: string type: object x-kubernetes-map-type: atomic + autoNode: + description: autoNode specifies the configuration for the autoNode + feature. + properties: + provisionerConfig: + description: provisioner is the implementation used for Node auto + provisioning. + properties: + karpenter: + description: karpenter specifies the configuration for the + Karpenter provisioner. + properties: + aws: + description: aws specifies the AWS-specific configuration + for Karpenter. + properties: + roleARN: + description: arn specifies the ARN of the Karpenter + provisioner. + type: string + required: + - roleARN + type: object + platform: + description: platform specifies the platform-specific + configuration for Karpenter. + type: string + required: + - platform + type: object + name: + allOf: + - enum: + - Karpenter + - enum: + - Karpenter + description: name specifies the name of the provisioner to + use. + type: string + required: + - name + type: object + required: + - provisionerConfig + type: object autoscaling: description: |- Autoscaling specifies auto-scaling behavior that applies to all NodePools diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 285879f0d8..be209077d4 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -5335,6 +5335,17 @@ func createAWSDefaultSecurityGroup(ctx context.Context, ec2Client ec2iface.EC2AP Value: awssdk.String(awsSecurityGroupName(infraID)), }) } + + if hcp.Spec.AutoNode != nil && hcp.Spec.AutoNode.Provisioner.Name == hyperv1.ProvisionerKarpeneter && + hcp.Spec.AutoNode.Provisioner.Karpenter.Platform == hyperv1.AWSPlatform { + if !tagKeys.Has("karpenter.sh/discovery") { + tags = append(tags, &ec2.Tag{ + Key: awssdk.String("karpenter.sh/discovery"), + Value: awssdk.String(infraID), + }) + } + } + createSGResult, err := ec2Client.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{ GroupName: awssdk.String(awsSecurityGroupName(infraID)), Description: awssdk.String("default worker security group"), diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index f09d737815..21b2020e9c 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -348,6 +348,19 @@ associated with this HostedCluster.

+autoNode
+ + +AutoNode + + + + +

autoNode specifies the configuration for the autoNode feature.

+ + + + etcd
@@ -2569,6 +2582,38 @@ string +###AutoNode { #hypershift.openshift.io/v1beta1.AutoNode } +

+(Appears on: +HostedClusterSpec, +HostedControlPlaneSpec) +

+

+

We expose here internal configuration knobs that won’t be exposed to the service.

+

+ + + + + + + + + + + + + +
FieldDescription
+provisionerConfig
+ + +ProvisionerConfig + + +		 +

provisioner is the implementation used for Node auto provisioning.

+
###AvailabilityPolicy { #hypershift.openshift.io/v1beta1.AvailabilityPolicy }

(Appears on: @@ -5024,6 +5069,19 @@ associated with this HostedCluster.

+autoNode
+ + +AutoNode + + + + +

autoNode specifies the configuration for the autoNode feature.

+ + + + etcd
@@ -5861,6 +5919,19 @@ associated with the control plane.

+autoNode
+ + +AutoNode + + + + +

autoNode specifies the configuration for the autoNode feature.

+ + + + nodeSelector
map[string]string @@ -6586,6 +6657,78 @@ AzureKMSSpec +###KarpenterAWSConfig { #hypershift.openshift.io/v1beta1.KarpenterAWSConfig } +

+(Appears on: +KarpenterConfig) +

+

+

+ + + + + + + + + + + + + +
FieldDescription
+roleARN
+ +string + +		 +

arn specifies the ARN of the Karpenter provisioner.

+
+###KarpenterConfig { #hypershift.openshift.io/v1beta1.KarpenterConfig } +

+(Appears on: +ProvisionerConfig) +

+

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+platform
+ + +PlatformType + + +		 +

platform specifies the platform-specific configuration for Karpenter.

+
+aws
+ + +KarpenterAWSConfig + + +		 +(Optional) +

aws specifies the AWS-specific configuration for Karpenter.

+
###KubeVirtNodePoolStatus { #hypershift.openshift.io/v1beta1.KubeVirtNodePoolStatus }

(Appears on: @@ -9322,6 +9465,7 @@ AWSPlatformStatus ###PlatformType { #hypershift.openshift.io/v1beta1.PlatformType }

(Appears on: +KarpenterConfig, NodePoolPlatform, PlatformSpec)

@@ -9996,6 +10140,70 @@ This field is immutable. Once set, It can’t be changed.

+###Provisioner { #hypershift.openshift.io/v1beta1.Provisioner } +

+(Appears on: +ProvisionerConfig) +

+

+

provisioner is a enum specifying the strategy for auto managing Nodes.

+

+ + + + + + + + + + +
ValueDescription

"Karpenter"

+###ProvisionerConfig { #hypershift.openshift.io/v1beta1.ProvisionerConfig } +

+(Appears on: +AutoNode) +

+

+

ProvisionerConfig is a enum specifying the strategy for auto managing Nodes.

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+name
+ + +Provisioner + + +		 +

name specifies the name of the provisioner to use.

+
+karpenter
+ + +KarpenterConfig + + +		 +(Optional) +

karpenter specifies the configuration for the Karpenter provisioner.

+
###PublishingStrategyType { #hypershift.openshift.io/v1beta1.PublishingStrategyType }

(Appears on: diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index 484d0e697b..a74b2bfcdb 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -1757,6 +1757,10 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques } } + if err := r.reconcileKarpenterOperator(ctx, createOrUpdate, hcluster, hcp, r.HypershiftOperatorImage, controlPlaneOperatorImage); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile karpenter operator: %w", err) + } + // Reconcile the Ignition server if !controlplaneOperatorManagesIgnitionServer { releaseInfo, err := r.lookupReleaseImage(ctx, hcluster, releaseProvider) @@ -1955,6 +1959,7 @@ func reconcileHostedControlPlane(hcp *hyperv1.HostedControlPlane, hcluster *hype hcp.Spec.PausedUntil = hcluster.Spec.PausedUntil hcp.Spec.OLMCatalogPlacement = hcluster.Spec.OLMCatalogPlacement hcp.Spec.Autoscaling = hcluster.Spec.Autoscaling + hcp.Spec.AutoNode = hcluster.Spec.AutoNode hcp.Spec.NodeSelector = hcluster.Spec.NodeSelector hcp.Spec.Tolerations = hcluster.Spec.Tolerations hcp.Spec.Labels = hcluster.Spec.Labels diff --git a/hypershift-operator/controllers/hostedcluster/karpenter.go b/hypershift-operator/controllers/hostedcluster/karpenter.go new file mode 100644 index 0000000000..e165e831eb --- /dev/null +++ b/hypershift-operator/controllers/hostedcluster/karpenter.go @@ -0,0 +1,116 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hostedcluster + +import ( + "context" + "fmt" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + karpenteroperatormanifest "github.com/openshift/hypershift/karpenter-operator/manifests" + "github.com/openshift/hypershift/support/upsert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" +) + +func (r *HostedClusterReconciler) reconcileKarpenterOperator(ctx context.Context, createOrUpdate upsert.CreateOrUpdateFN, hcluster *hyperv1.HostedCluster, hcp *hyperv1.HostedControlPlane, hypershiftOperatorImage, controlPlaneOperatorImage string) error { + if hcluster.Spec.AutoNode == nil || hcluster.Spec.AutoNode.Provisioner.Name != hyperv1.ProvisionerKarpeneter || + hcluster.Spec.AutoNode.Provisioner.Karpenter.Platform != hyperv1.AWSPlatform { + return nil + } + + // Generate configMap with KubeletConfig to register Nodes with karpenter expected taint. + taintConfigName := "set-karpenter-taint" + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: taintConfigName, + Namespace: hcluster.Namespace, + }, + } + + kubeletConfig := `apiVersion: machineconfiguration.openshift.io/v1 +kind: KubeletConfig +metadata: + name: set-karpenter-taint +spec: + kubeletConfig: + registerWithTaints: + - key: "karpenter.sh/unregistered" + value: "true" + effect: "NoExecute"` + + _, err := createOrUpdate(ctx, r.Client, configMap, func() error { + configMap.Data = map[string]string{ + "config": kubeletConfig, + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to create configmap: %w", err) + } + + // Managed a NodePool to generate userData for Karpenter instances + // TODO(alberto): consider invoking the token library to manage the karpenter userdata programatically, + // instead of via NodePool API. + nodePool := &hyperv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "karpenter", + Namespace: hcluster.Namespace, + }, + } + _, err = createOrUpdate(ctx, r.Client, nodePool, func() error { + nodePool.Spec = hyperv1.NodePoolSpec{ + ClusterName: hcluster.Name, + Replicas: ptr.To(int32(0)), + Release: hcluster.Spec.Release, + Config: []corev1.LocalObjectReference{ + { + Name: taintConfigName, + }, + }, + Management: hyperv1.NodePoolManagement{ + UpgradeType: hyperv1.UpgradeTypeReplace, + Replace: &hyperv1.ReplaceUpgrade{ + Strategy: hyperv1.UpgradeStrategyRollingUpdate, + RollingUpdate: &hyperv1.RollingUpdate{ + MaxUnavailable: ptr.To(intstr.FromInt(0)), + MaxSurge: ptr.To(intstr.FromInt(1)), + }, + }, + AutoRepair: false, + }, + Platform: hyperv1.NodePoolPlatform{ + Type: hyperv1.AWSPlatform, + AWS: &hyperv1.AWSNodePoolPlatform{ + InstanceType: "m5.large", + Subnet: hyperv1.AWSResourceReference{ + ID: ptr.To("subnet-none"), + }, + }, + }, + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to create configmap: %w", err) + } + // TODO(alberto): Ensure deletion if autoNode is disabled. + + // Run karpenter Operator to manage CRs management and guest side. + if err := karpenteroperatormanifest.ReconcileKarpenterOperator(ctx, createOrUpdate, r.Client, hypershiftOperatorImage, controlPlaneOperatorImage, hcp); err != nil { + return err + } + return nil +} diff --git a/hypershift-operator/controllers/nodepool/token.go b/hypershift-operator/controllers/nodepool/token.go index 58f8cfbb3b..d0c4254ea2 100644 --- a/hypershift-operator/controllers/nodepool/token.go +++ b/hypershift-operator/controllers/nodepool/token.go @@ -60,6 +60,7 @@ type userData struct { caCert []byte ignitionServerEndpoint string proxy *configv1.Proxy + ami string } // NewToken is the contract to create a new Token struct. @@ -124,10 +125,19 @@ func NewToken(ctx context.Context, configGenerator *ConfigGenerator, cpoCapabili proxy := globalconfig.ProxyConfig() globalconfig.ReconcileProxyConfigWithStatusFromHostedCluster(proxy, configGenerator.hostedCluster) + ami := "" + if configGenerator.hostedCluster.Spec.Platform.AWS != nil { + ami, err = defaultNodePoolAMI(configGenerator.hostedCluster.Spec.Platform.AWS.Region, configGenerator.nodePool.Spec.Arch, configGenerator.releaseImage) + if err != nil { + return nil, err + } + } + token.userData = &userData{ ignitionServerEndpoint: ignEndpoint, caCert: caCert, proxy: proxy, + ami: ami, } return token, nil @@ -335,6 +345,19 @@ func (t *Token) reconcileUserDataSecret(userDataSecret *corev1.Secret, token str userDataSecret.Annotations = make(map[string]string) } userDataSecret.Annotations[nodePoolAnnotation] = client.ObjectKeyFromObject(t.nodePool).String() + if userDataSecret.Labels == nil { + userDataSecret.Labels = make(map[string]string) + } + + if t.hostedCluster.Spec.AutoNode != nil && t.hostedCluster.Spec.AutoNode.Provisioner.Name == hyperv1.ProvisionerKarpeneter && + t.hostedCluster.Spec.AutoNode.Provisioner.Karpenter.Platform == hyperv1.AWSPlatform { + // TODO(alberto): prevent nodePool name collisions adding prefix to karpenter NodePool. + if t.nodePool.GetName() == "karpenter" { + userDataSecret.Labels[hyperv1.NodePoolLabel] = fmt.Sprintf("%s-%s", t.nodePool.Spec.ClusterName, t.nodePool.GetName()) + userDataSecret.Labels["hypershift.openshift.io/ami"] = t.userData.ami + } + + } encodedCACert := base64.StdEncoding.EncodeToString(t.userData.caCert) encodedToken := base64.StdEncoding.EncodeToString([]byte(token)) diff --git a/karpenter-operator/controllers/karpenter/assets/assets.go b/karpenter-operator/controllers/karpenter/assets/assets.go new file mode 100644 index 0000000000..e49f902d92 --- /dev/null +++ b/karpenter-operator/controllers/karpenter/assets/assets.go @@ -0,0 +1,13 @@ +package assets + +import ( + "embed" +) + +//go:embed *.yaml +var f embed.FS + +// ReadFile reads and returns the content of the named file. +func ReadFile(name string) ([]byte, error) { + return f.ReadFile(name) +} diff --git a/karpenter-operator/controllers/karpenter/assets/karpenter.k8s.aws_ec2nodeclasses.yaml b/karpenter-operator/controllers/karpenter/assets/karpenter.k8s.aws_ec2nodeclasses.yaml new file mode 100644 index 0000000000..ff1bc9da2b --- /dev/null +++ b/karpenter-operator/controllers/karpenter/assets/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -0,0 +1,739 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: ec2nodeclasses.karpenter.k8s.aws +spec: + group: karpenter.k8s.aws + names: + categories: + - karpenter + kind: EC2NodeClass + listKind: EC2NodeClassList + plural: ec2nodeclasses + shortNames: + - ec2nc + - ec2ncs + singular: ec2nodeclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.role + name: Role + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: EC2NodeClass is the Schema for the EC2NodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. + properties: + amiFamily: + description: |- + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 + type: string + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', 'alias'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + The following are the supported values for each volume type: + + * gp3: 3,000-16,000 IOPS + + * io1: 100-64,000 IOPS + + * io2: 100-64,000 IOPS + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + * gp2 and gp3: 1-16,384 + + * io1 and io2: 4-16,384 + + * st1 and sc1: 125-16,384 + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled + type: string + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled + type: string + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string + type: object + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) + - message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2'') : true)' + - message: if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2023'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2023'') : true)' + - message: if set, amiFamily must be 'Bottlerocket' or 'Custom' when using a Bottlerocket alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''bottlerocket'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Bottlerocket'') : true)' + - message: if set, amiFamily must be 'Windows2019' or 'Custom' when using a Windows2019 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2019'') : true)' + - message: if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2022'') : true)' + - message: must specify amiFamily if amiSelectorTerms does not contain an alias + rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)' + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. + items: + description: AMI contains resolved AMI selector values utilized for node launch + properties: + id: + description: ID of the AMI + type: string + name: + description: Name of the AMI + type: string + requirements: + description: Requirements of the AMI to be utilized on an instance type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile for the role + type: string + securityGroups: + description: |- + SecurityGroups contains the current Security Groups values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current Subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + zoneID: + description: The associated availability zone ID + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/karpenter-operator/controllers/karpenter/assets/karpenter.sh_nodeclaims.yaml b/karpenter-operator/controllers/karpenter/assets/karpenter.sh_nodeclaims.yaml new file mode 100644 index 0000000000..7976cfe0f4 --- /dev/null +++ b/karpenter-operator/controllers/karpenter/assets/karpenter.sh_nodeclaims.yaml @@ -0,0 +1,382 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: nodeclaims.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + singular: nodeclaim + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.providerID + name: ID + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/karpenter-operator/controllers/karpenter/assets/karpenter.sh_nodepools.yaml b/karpenter-operator/controllers/karpenter/assets/karpenter.sh_nodepools.yaml new file mode 100644 index 0000000000..f20e73cd0d --- /dev/null +++ b/karpenter-operator/controllers/karpenter/assets/karpenter.sh_nodepools.yaml @@ -0,0 +1,507 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: nodepools.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodePool + listKind: NodePoolList + plural: nodepools + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted and additional CloudProvider-specific reasons. + items: + description: |- + DisruptionReason defines valid reasons for disruption budgets. + CloudProviders will need to append to the list of enums when implementing cloud provider disruption reasons + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.k8s.aws" is restricted + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/karpenter-operator/controllers/karpenter/karpenter_controller.go b/karpenter-operator/controllers/karpenter/karpenter_controller.go new file mode 100644 index 0000000000..682b247bed --- /dev/null +++ b/karpenter-operator/controllers/karpenter/karpenter_controller.go @@ -0,0 +1,327 @@ +package karpenter + +import ( + "context" + "fmt" + "sort" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/karpenter-operator/controllers/karpenter/assets" + supportassets "github.com/openshift/hypershift/support/assets" + "github.com/openshift/hypershift/support/upsert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const ( + karpenterFinalizer = "hypershift.openshift.io/karpenter-finalizer" + // userDataAMILabel is a label set in the userData secret generated for karpenter instances. + userDataAMILabel = "hypershift.openshift.io/ami" +) + +var ( + crdEC2NodeClass = supportassets.MustCRD(assets.ReadFile, "karpenter.k8s.aws_ec2nodeclasses.yaml") + crdNodePool = supportassets.MustCRD(assets.ReadFile, "karpenter.sh_nodepools.yaml") + crdNodeClaim = supportassets.MustCRD(assets.ReadFile, "karpenter.sh_nodeclaims.yaml") +) + +type Reconciler struct { + ManagementClient client.Client + GuestClient client.Client + Namespace string + ControlPlaneOperatorImage string + upsert.CreateOrUpdateProvider +} + +func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, managementCluster cluster.Cluster) error { + r.ManagementClient = managementCluster.GetClient() + r.GuestClient = mgr.GetClient() + r.CreateOrUpdateProvider = upsert.New(false) + + // First install the CRDs so we can create a watch below. + if err := r.reconcileCRDs(ctx, true); err != nil { + return err + } + + c, err := controller.New("karpenter", mgr, controller.Options{Reconciler: r}) + if err != nil { + return fmt.Errorf("failed to construct controller: %w", err) + } + + // Watch CRDs guest side. + if err := c.Watch(source.Kind[client.Object](mgr.GetCache(), &apiextensionsv1.CustomResourceDefinition{}, handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, o client.Object) []ctrl.Request { + // Only watch our Karpenter CRDs + switch o.GetName() { + case "ec2nodeclasses.karpenter.k8s.aws", + "nodepools.karpenter.sh", + "nodeclaims.karpenter.sh": + return []ctrl.Request{{NamespacedName: client.ObjectKey{Namespace: r.Namespace}}} + } + return nil + }))); err != nil { + return fmt.Errorf("failed to watch CRDs: %w", err) + } + + // Watch EC2NodeClass guest side. + if err := c.Watch(source.Kind[client.Object](mgr.GetCache(), &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "karpenter.k8s.aws/v1", + "kind": "EC2NodeClass", + }}, &handler.EnqueueRequestForObject{})); err != nil { + return fmt.Errorf("failed to watch EC2NodeClass: %w", err) + } + + // Watch the karpenter Deployment management side. + if err := c.Watch(source.Kind[client.Object](managementCluster.GetCache(), &appsv1.Deployment{}, handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, o client.Object) []ctrl.Request { + if o.GetNamespace() != r.Namespace || o.GetName() != "karpenter" { + return nil + } + return []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(o)}} + }))); err != nil { + return fmt.Errorf("failed to watch Deployment: %w", err) + } + + // Trigger initial sync. + initialSync := make(chan event.GenericEvent) + if err := c.Watch(source.Channel(initialSync, &handler.EnqueueRequestForObject{})); err != nil { + return fmt.Errorf("failed to watch initial sync channel: %w", err) + } + go func() { + initialSync <- event.GenericEvent{Object: &hyperv1.HostedControlPlane{}} + }() + + return nil +} + +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + log.Info("Reconciling", "req", req) + + hcp, err := r.getHCP(ctx) + if err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + if hcp.DeletionTimestamp != nil { + // TODO(alberto): implement deletion. E.g. loop over nodeClaims delete them, wait and delete karpeneter deployment. + if controllerutil.ContainsFinalizer(hcp, karpenterFinalizer) { + originalHCP := hcp.DeepCopy() + controllerutil.RemoveFinalizer(hcp, karpenterFinalizer) + if err := r.ManagementClient.Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to remove finalizer from cluster: %w", err) + } + } + return ctrl.Result{}, nil + } + if !controllerutil.ContainsFinalizer(hcp, karpenterFinalizer) { + originalHCP := hcp.DeepCopy() + controllerutil.AddFinalizer(hcp, karpenterFinalizer) + if err := r.ManagementClient.Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to add finalizer to hostedControlPlane: %w", err) + } + } + + // TODO(alberto): + // - reconcile validatingAdmissionPolicy to enforce shared ownership. + // - Watch userDataSecret. + // - Solve token rotation causing drift. + // - CSR approval. + + userDataSecret, err := r.getUserDataSecret(ctx, hcp) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.reconcileEC2NodeClassOwnedFields(ctx, userDataSecret); err != nil { + return ctrl.Result{}, err + } + + if err := r.reconcileEC2NodeClassDefault(ctx, userDataSecret, hcp); err != nil { + return ctrl.Result{}, err + } + + if err := r.reconcileKarpenter(ctx, hcp); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile karpenter deployment: %w", err) + } + + if err := r.reconcileCRDs(ctx, false); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +// reconcileCRDs reconcile the Karpenter CRDs, if onlyCreate is true it uses an only write non cached client. +func (r *Reconciler) reconcileCRDs(ctx context.Context, onlyCreate bool) error { + log := ctrl.LoggerFrom(ctx) + + errs := []error{} + var op controllerutil.OperationResult + var err error + for _, crd := range []*apiextensionsv1.CustomResourceDefinition{ + crdEC2NodeClass, + crdNodePool, + crdNodeClaim, + } { + if onlyCreate { + if err := r.GuestClient.Create(ctx, crd); err != nil { + if !apierrors.IsAlreadyExists(err) { + errs = append(errs, err) + } + } + } else { + op, err = r.CreateOrUpdate(ctx, r.GuestClient, crd, func() error { + return nil + }) + if err != nil { + errs = append(errs, err) + } + + } + } + if err := utilerrors.NewAggregate(errs); err != nil { + return fmt.Errorf("failed to reconcile CRDs: %w", err) + } + log.Info("Reconciled CRDs", "op", op) + + return nil +} + +func (r *Reconciler) reconcileEC2NodeClassOwnedFields(ctx context.Context, userDataSecret *corev1.Secret) error { + log := ctrl.LoggerFrom(ctx) + + ec2NodeClassList := &unstructured.UnstructuredList{} + ec2NodeClassList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "karpenter.k8s.aws", + Version: "v1", + Kind: "EC2NodeClassList", + }) + err := r.GuestClient.List(ctx, ec2NodeClassList) + if err != nil { + return fmt.Errorf("failed to get EC2NodeClassList: %w", err) + } + + errs := []error{} + for _, ec2NodeClass := range ec2NodeClassList.Items { + ec2NodeClass.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "karpenter.k8s.aws", + Version: "v1", + Kind: "EC2NodeClass", + }) + op, err := r.CreateOrUpdate(ctx, r.GuestClient, &ec2NodeClass, func() error { + ec2NodeClass.Object["spec"].(map[string]interface{})["userData"] = string(userDataSecret.Data["value"]) + ec2NodeClass.Object["spec"].(map[string]interface{})["amiFamily"] = "Custom" + ec2NodeClass.Object["spec"].(map[string]interface{})["amiSelectorTerms"] = []map[string]interface{}{ + { + "id": string(userDataSecret.Labels[userDataAMILabel]), + }, + } + return nil + }) + if err != nil { + errs = append(errs, err) + } + if err == nil { + log.Info("Set managed fields in ec2NodeClass", "ec2NodeClass", ec2NodeClass.GetName(), "op", op) + } + } + if err := utilerrors.NewAggregate(errs); err != nil { + return fmt.Errorf("failed to update EC2NodeClass: %w", err) + } + return nil +} + +func (r *Reconciler) reconcileEC2NodeClassDefault(ctx context.Context, userDataSecret *corev1.Secret, hcp *hyperv1.HostedControlPlane) error { + log := ctrl.LoggerFrom(ctx) + + // Create an unstructured object for the EC2NodeClass + ec2NodeClass := &unstructured.Unstructured{} + ec2NodeClass.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "karpenter.k8s.aws", + Version: "v1", + Kind: "EC2NodeClass", + }) + ec2NodeClass.SetName("default") + + op, err := r.CreateOrUpdate(ctx, r.GuestClient, ec2NodeClass, func() error { + ec2NodeClass.Object["spec"] = map[string]interface{}{} + ec2NodeClass.Object["spec"].(map[string]interface{})["role"] = "KarpenterNodeRole-agl" // TODO(alberto): set a convention for this e.g. openshift-karpenter-infraID + ec2NodeClass.Object["spec"].(map[string]interface{})["userData"] = string(userDataSecret.Data["value"]) + ec2NodeClass.Object["spec"].(map[string]interface{})["amiFamily"] = "Custom" + ec2NodeClass.Object["spec"].(map[string]interface{})["amiSelectorTerms"] = []map[string]interface{}{ + { + "id": string(userDataSecret.Labels[userDataAMILabel]), + }, + } + ec2NodeClass.Object["spec"].(map[string]interface{})["subnetSelectorTerms"] = []map[string]interface{}{ + { + "tags": map[string]interface{}{ + "karpenter.sh/discovery": hcp.Spec.InfraID, + }, + }, + } + ec2NodeClass.Object["spec"].(map[string]interface{})["securityGroupSelectorTerms"] = []map[string]interface{}{ + { + "tags": map[string]interface{}{ + "karpenter.sh/discovery": hcp.Spec.InfraID, + }, + }, + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to reconcile default EC2NodeClass: %w", err) + } + log.Info("Reconciled default EC2NodeClass", "op", op) + return nil +} + +func (r *Reconciler) getUserDataSecret(ctx context.Context, hcp *hyperv1.HostedControlPlane) (*corev1.Secret, error) { + labelSelector := labels.SelectorFromSet(labels.Set{hyperv1.NodePoolLabel: fmt.Sprintf("%s-karpenter", hcp.GetName())}) + listOptions := &client.ListOptions{ + LabelSelector: labelSelector, + Namespace: r.Namespace, + } + secretList := &corev1.SecretList{} + err := r.ManagementClient.List(ctx, secretList, listOptions) + if err != nil { + return nil, fmt.Errorf("failed to list secrets: %w", err) + } + + sort.Slice(secretList.Items, func(i, j int) bool { + return secretList.Items[i].CreationTimestamp.After(secretList.Items[j].CreationTimestamp.Time) + }) + if len(secretList.Items) < 1 { + return nil, fmt.Errorf("expected 1 secret, got 0") + } + return &secretList.Items[0], err +} + +func (r *Reconciler) getHCP(ctx context.Context) (*hyperv1.HostedControlPlane, error) { + hcpList := &hyperv1.HostedControlPlaneList{} + if err := r.ManagementClient.List(ctx, hcpList, client.InNamespace(r.Namespace)); err != nil { + return nil, err + } + if len(hcpList.Items) == 0 { + return nil, fmt.Errorf("failed to find HostedControlPlane in namespace %s", r.Namespace) + } + + return &hcpList.Items[0], nil +} diff --git a/karpenter-operator/controllers/karpenter/karpenter_controller_test.go b/karpenter-operator/controllers/karpenter/karpenter_controller_test.go new file mode 100644 index 0000000000..6bb74ba742 --- /dev/null +++ b/karpenter-operator/controllers/karpenter/karpenter_controller_test.go @@ -0,0 +1,226 @@ +package karpenter + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/upsert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestReconcileEC2NodeClassDefault(t *testing.T) { + scheme := runtime.NewScheme() + // _ = corev1.AddToScheme(scheme) + _ = hyperv1.AddToScheme(scheme) + + // Register the EC2NodeClass GVK in the scheme + ec2NodeClassGVK := schema.GroupVersionKind{ + Group: "karpenter.k8s.aws", + Version: "v1", + Kind: "EC2NodeClass", + } + scheme.AddKnownTypeWithName(ec2NodeClassGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName( + schema.GroupVersionKind{ + Group: "karpenter.k8s.aws", + Version: "v1", + Kind: "EC2NodeClassList", + }, + &unstructured.UnstructuredList{}, + ) + + testCases := []struct { + name string + userDataSecret *corev1.Secret + hcp *hyperv1.HostedControlPlane + wantErr bool + }{ + { + name: "When no errors it should create the default EC2NodeClass", + userDataSecret: &corev1.Secret{ + Data: map[string][]byte{ + "value": []byte("test-userdata"), + }, + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + userDataAMILabel: "ami-123", + }, + }, + }, + hcp: &hyperv1.HostedControlPlane{ + Spec: hyperv1.HostedControlPlaneSpec{ + InfraID: "test-infra", + }, + }, + wantErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + Build() + + r := &Reconciler{ + GuestClient: fakeClient, + CreateOrUpdateProvider: upsert.New(false), + } + + err := r.reconcileEC2NodeClassDefault(context.Background(), tc.userDataSecret, tc.hcp) + if (err != nil) != tc.wantErr { + t.Errorf("reconcileEC2NodeClassDefault() error = %v, wantErr %v", err, tc.wantErr) + return + } + + // Verify the EC2NodeClass was created. + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(ec2NodeClassGVK) + + err = fakeClient.Get(context.Background(), types.NamespacedName{Name: "default"}, got) + if err != nil { + t.Errorf("failed to get EC2NodeClass: %v", err) + return + } + + spec, ok := got.Object["spec"].(map[string]interface{}) + if !ok { + t.Fatal("spec is not a map") + } + + // Verify basic fields + g.Expect(spec["role"]).To(Equal("KarpenterNodeRole-agl"), "role = %v, want KarpenterNodeRole-agl", spec["role"]) + g.Expect(spec["userData"]).To(Equal("test-userdata"), "userData = %v, want test-userdata", spec["userData"]) + g.Expect(spec["amiFamily"]).To(Equal("Custom"), "amiFamily = %v, want Custom", spec["amiFamily"]) + + // Verify amiSelectorTerms + amiTerms, ok := spec["amiSelectorTerms"].([]interface{}) + g.Expect(ok).To(BeTrue(), "amiSelectorTerms should be a slice") + g.Expect(len(amiTerms)).To(Equal(1), "amiSelectorTerms should have exactly one element") + + amiTerm, ok := amiTerms[0].(map[string]interface{}) + g.Expect(ok).To(BeTrue(), "amiTerm should be a map") + g.Expect(amiTerm["id"]).To(Equal("ami-123"), "unexpected amiSelectorTerms: %v", amiTerms) + + // Verify selector terms have correct tags + expectedTag := map[string]interface{}{ + "karpenter.sh/discovery": "test-infra", + } + + // Helper function to verify selector terms + verifySelectorTerms := func(field string, expectedTags map[string]interface{}) { + terms, ok := spec[field].([]interface{}) + g.Expect(ok).To(BeTrue(), "terms should be a slice for field %s", field) + g.Expect(len(terms)).To(Equal(1), "terms should have exactly one element for field %s", field) + + term, ok := terms[0].(map[string]interface{}) + g.Expect(ok).To(BeTrue(), "term should be a map for field %s", field) + + tags, ok := term["tags"].(map[string]interface{}) + g.Expect(ok).To(BeTrue(), "tags should be a map for field %s", field) + g.Expect(tags).To(Equal(expectedTags), "%s tags = %v, want %v", field, tags, expectedTags) + } + + verifySelectorTerms("subnetSelectorTerms", expectedTag) + verifySelectorTerms("securityGroupSelectorTerms", expectedTag) + }) + } +} + +func TestGetUserDataSecret(t *testing.T) { + g := NewWithT(t) + + scheme := runtime.NewScheme() + g.Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + testCases := []struct { + name string + namespace string + hcp *hyperv1.HostedControlPlane + objects []client.Object + expectedError string + }{ + { + name: "when multiple exist it should return newest secret", + namespace: "test-namespace", + hcp: &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hcp", + }, + }, + objects: []client.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "older-secret", + Namespace: "test-namespace", + CreationTimestamp: metav1.Time{Time: time.Now().Add(-1 * time.Hour)}, + Labels: map[string]string{ + hyperv1.NodePoolLabel: "test-hcp-karpenter", + }, + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "newer-secret", + Namespace: "test-namespace", + CreationTimestamp: metav1.Time{Time: time.Now()}, + Labels: map[string]string{ + hyperv1.NodePoolLabel: "test-hcp-karpenter", + }, + }, + }, + }, + }, + { + name: "when no secrets exist it should return error", + namespace: "test-namespace", + hcp: &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hcp", + }, + }, + objects: []client.Object{}, + expectedError: "expected 1 secret, got 0", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(tc.objects...). + Build() + + r := &Reconciler{ + ManagementClient: fakeClient, + Namespace: tc.namespace, + } + + secret, err := r.getUserDataSecret(context.Background(), tc.hcp) + + if tc.expectedError != "" { + g.Expect(err).To(MatchError(tc.expectedError)) + g.Expect(secret).To(BeNil()) + return + } + + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(secret).NotTo(BeNil()) + + g.Expect(secret.Name).To(Equal("newer-secret")) + }) + } +} diff --git a/karpenter-operator/controllers/karpenter/manifests.go b/karpenter-operator/controllers/karpenter/manifests.go new file mode 100644 index 0000000000..b2d75b171a --- /dev/null +++ b/karpenter-operator/controllers/karpenter/manifests.go @@ -0,0 +1,420 @@ +package karpenter + +import ( + "context" + "fmt" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" + "github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator" + "github.com/openshift/hypershift/support/config" + "github.com/openshift/hypershift/support/util" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + intstr "k8s.io/apimachinery/pkg/util/intstr" + k8sutilspointer "k8s.io/utils/ptr" + client "sigs.k8s.io/controller-runtime/pkg/client" +) + +func KarpenterDeployment(namespace string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "karpenter", + }, + } +} + +func KarpenterServiceAccount(controlPlaneNamespace string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: controlPlaneNamespace, + Name: "karpenter", + }, + } +} + +func KarpenterRole(controlPlaneNamespace string) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: controlPlaneNamespace, + Name: "karpenter", + }, + } +} + +func KarpenterRoleBinding(controlPlaneNamespace string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: controlPlaneNamespace, + Name: "karpenter", + }, + } +} + +func karpenterSelector() map[string]string { + return map[string]string{ + "karpenter": "karpenter", + } +} + +func ReconcileKarpenterDeployment(deployment *appsv1.Deployment, + hcp *hyperv1.HostedControlPlane, + sa *corev1.ServiceAccount, + kubeConfigSecret *corev1.Secret, + availabilityProberImage, tokenMinterImage string, + credentialsSecret *corev1.Secret, + setDefaultSecurityContext bool, + ownerRef config.OwnerRef) error { + + deployment.Spec = appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: karpenterSelector(), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: karpenterSelector(), + }, + Spec: corev1.PodSpec{ + ServiceAccountName: sa.Name, + TerminationGracePeriodSeconds: k8sutilspointer.To(int64(10)), + Tolerations: []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "target-kubeconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: kubeConfigSecret.Name, + DefaultMode: k8sutilspointer.To(int32(0640)), + Items: []corev1.KeyToPath{ + { + Key: "value", + Path: "target-kubeconfig", + }, + }, + }, + }, + }, + { + Name: "serviceaccount-token", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "provider-creds", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: credentialsSecret.Name, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "karpenter", + // TODO(alberto): lifecycle this image. + Image: "public.ecr.aws/karpenter/controller:1.0.7", + ImagePullPolicy: corev1.PullAlways, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "target-kubeconfig", + MountPath: "/mnt/kubeconfig", + }, + { + Name: "provider-creds", + MountPath: "/etc/provider", + }, + { + Name: "serviceaccount-token", + MountPath: "/var/run/secrets/openshift/serviceaccount", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "KUBECONFIG", + Value: "/mnt/kubeconfig/target-kubeconfig", + }, + { + Name: "SYSTEM_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "DISABLE_WEBHOOK", + Value: "true", + }, + { + Name: "DISABLE_LEADER_ELECTION", + Value: "true", + }, + { + Name: "FEATURE_GATES", + Value: "Drift=true", + }, + { + Name: "AWS_REGION", + Value: "us-east-1", + }, + { + Name: "AWS_SHARED_CREDENTIALS_FILE", + Value: "/etc/provider/credentials", + }, + { + Name: "AWS_SDK_LOAD_CONFIG", + Value: "true", + }, + { + Name: "HEALTH_PROBE_PORT", + Value: "8081", + }, + // TODO (alberto): this is to satisfy current karpenter requirements. We should relax the req. + { + Name: "CLUSTER_ENDPOINT", + Value: "https://fake.com", + }, + { + Name: "CLUSTER_NAME", + Value: "none", + }, + }, + // Command: []string{""}, + Args: []string{ + "--log-level=debug", + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 60, + PeriodSeconds: 60, + SuccessThreshold: 1, + FailureThreshold: 5, + TimeoutSeconds: 5, + }, + + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + TimeoutSeconds: 5, + }, + Ports: []corev1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 8000, + }, + { + Name: "http", + ContainerPort: 8081, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + { + Name: "token-minter", + Command: []string{"/usr/bin/control-plane-operator", "token-minter"}, + Args: []string{ + "--service-account-namespace=kube-system", + "--service-account-name=karpenter", + "--token-file=/var/run/secrets/openshift/serviceaccount/token", + "--kubeconfig=/mnt/kubeconfig/target-kubeconfig", + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + }, + Image: tokenMinterImage, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "target-kubeconfig", + MountPath: "/mnt/kubeconfig", + }, + { + Name: "serviceaccount-token", + MountPath: "/var/run/secrets/openshift/serviceaccount", + }, + }, + }, + }, + }, + }, + } + + util.AvailabilityProber(kas.InClusterKASReadyURL(hcp.Spec.Platform.Type), availabilityProberImage, &deployment.Spec.Template.Spec) + deploymentConfig := config.DeploymentConfig{ + AdditionalLabels: map[string]string{ + config.NeedManagementKASAccessLabel: "true", + }, + Scheduling: config.Scheduling{ + PriorityClass: config.DefaultPriorityClass, + }, + SetDefaultSecurityContext: setDefaultSecurityContext, + } + if hcp.Annotations[hyperv1.ControlPlanePriorityClass] != "" { + deploymentConfig.Scheduling.PriorityClass = hcp.Annotations[hyperv1.ControlPlanePriorityClass] + } + + replicas := k8sutilspointer.To(1) + deploymentConfig.SetDefaults(hcp, nil, replicas) + deploymentConfig.SetRestartAnnotation(hcp.ObjectMeta) + deploymentConfig.ApplyTo(deployment) + + return nil +} + +func ReconcileKarpenterRole(role *rbacv1.Role, owner config.OwnerRef) error { + owner.ApplyTo(role) + role.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{ + "leases", + }, + Verbs: []string{ + "get", + "watch", + "create", + }, + }, + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{ + "leases", + }, + Verbs: []string{ + "patch", + "update", + }, + ResourceNames: []string{ + "karpenter-leader-election", + }, + }, + } + return nil +} + +func ReconcileKarpenterRoleBinding(binding *rbacv1.RoleBinding, role *rbacv1.Role, sa *corev1.ServiceAccount, owner config.OwnerRef) error { + owner.ApplyTo(binding) + binding.RoleRef = rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: role.Name, + } + + binding.Subjects = []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: sa.Name, + Namespace: sa.Namespace, + }, + } + + return nil +} + +// ReconcileKarpenter orchestrates reconciliation of karpenter components. +func (r *Reconciler) reconcileKarpenter(ctx context.Context, hcp *hyperv1.HostedControlPlane) error { + createOrUpdate := r.CreateOrUpdate + c := r.ManagementClient + ownerRef := config.OwnerRefFrom(hcp) + setDefaultSecurityContext := false + availabilityProberImage := r.ControlPlaneOperatorImage + tokenMinterImage := r.ControlPlaneOperatorImage + + role := KarpenterRole(hcp.Namespace) + _, err := createOrUpdate(ctx, c, role, func() error { + return ReconcileKarpenterRole(role, ownerRef) + }) + if err != nil { + return fmt.Errorf("failed to reconcile karpenter role: %w", err) + } + + serviceAccount := KarpenterServiceAccount(hcp.Namespace) + _, err = createOrUpdate(ctx, c, serviceAccount, func() error { + util.EnsurePullSecret(serviceAccount, controlplaneoperator.PullSecret("").Name) + ownerRef.ApplyTo(serviceAccount) + return nil + }) + if err != nil { + return fmt.Errorf("failed to reconcile karpenter service account: %w", err) + } + + roleBinding := KarpenterRoleBinding(hcp.Namespace) + _, err = createOrUpdate(ctx, c, roleBinding, func() error { + return ReconcileKarpenterRoleBinding(roleBinding, role, serviceAccount, ownerRef) + }) + if err != nil { + return fmt.Errorf("failed to reconcile karpenter role binding: %w", err) + } + + awsCredentialsTemplate := `[default] + role_arn = %s + web_identity_token_file = /var/run/secrets/openshift/serviceaccount/token + sts_regional_endpoints = regional +` + arn := hcp.Spec.AutoNode.Provisioner.Karpenter.AWS.RoleARN + credentialsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: hcp.Namespace, + Name: "karpenter-credentials", + }, + } + if _, err := createOrUpdate(ctx, c, credentialsSecret, func() error { + credentials := fmt.Sprintf(awsCredentialsTemplate, arn) + credentialsSecret.Data = map[string][]byte{"credentials": []byte(credentials)} + credentialsSecret.Type = corev1.SecretTypeOpaque + return nil + }); err != nil { + return fmt.Errorf("failed to reconcile karpenter credentials secret %s/%s: %w", credentialsSecret.Namespace, credentialsSecret.Name, err) + } + + // The deployment depends on the kubeconfig being reported. + if hcp.Status.KubeConfig != nil { + // Resolve the kubeconfig secret for CAPI which is used for karpeneter for convience + capiKubeConfigSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: hcp.Namespace, + Name: fmt.Sprintf("%s-kubeconfig", hcp.Spec.InfraID), + }, + } + err = c.Get(ctx, client.ObjectKeyFromObject(capiKubeConfigSecret), capiKubeConfigSecret) + if err != nil { + return fmt.Errorf("failed to get hosted controlplane kubeconfig secret %q: %w", capiKubeConfigSecret.Name, err) + } + + deployment := KarpenterDeployment(hcp.Namespace) + _, err = createOrUpdate(ctx, c, deployment, func() error { + return ReconcileKarpenterDeployment(deployment, hcp, serviceAccount, capiKubeConfigSecret, availabilityProberImage, tokenMinterImage, credentialsSecret, setDefaultSecurityContext, ownerRef) + }) + if err != nil { + return fmt.Errorf("failed to reconcile karpenter deployment: %w", err) + } + } + + return nil +} diff --git a/karpenter-operator/main.go b/karpenter-operator/main.go new file mode 100644 index 0000000000..3a949800c2 --- /dev/null +++ b/karpenter-operator/main.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/openshift/hypershift/karpenter-operator/controllers/karpenter" + hyperapi "github.com/openshift/hypershift/support/api" + "github.com/spf13/cobra" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var ( + setupLog = ctrl.Log.WithName("setup") + + targetKubeconfig string + namespace string + controlPlaneOperatorImage string +) + +func main() { + var rootCmd = &cobra.Command{ + Use: "karpenter-operator", + Short: "Karpenter Operator is a Kubernetes operator for managing Karpenter", + Run: func(cmd *cobra.Command, args []string) { + opts := zap.Options{ + Development: true, + } + // opts.BindFlags(flag.CommandLine) + // flag.Parse() + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + if err := run(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + }, + } + + rootCmd.PersistentFlags().StringVar(&targetKubeconfig, "target-kubeconfig", "", "Path to guest side kubeconfig file. Where the karpenter CRs (nodeClaim, nodePool, nodeClass) live") + rootCmd.PersistentFlags().StringVar(&namespace, "namespace", "", "The namespace to infer input for reconciliation, e.g the userData secret") + rootCmd.PersistentFlags().StringVar(&controlPlaneOperatorImage, "control-plane-operator-image", "", "The image to run the tokenMinter and the availability prober") + rootCmd.MarkPersistentFlagRequired("target-kubeconfig") + rootCmd.MarkPersistentFlagRequired("namespace") + rootCmd.MarkPersistentFlagRequired("control-plane-operator-image") + + if err := rootCmd.Execute(); err != nil { + setupLog.Error(err, "problem executing command") + os.Exit(1) + } +} + +func run(ctx context.Context) error { + managementKubeconfig, err := ctrl.GetConfig() + if err != nil { + return err + } + managementCluster, err := cluster.New(managementKubeconfig, func(opt *cluster.Options) { + opt.Cache = cache.Options{ + DefaultNamespaces: map[string]cache.Config{namespace: {}}, + Scheme: hyperapi.Scheme, + } + opt.Scheme = hyperapi.Scheme + }) + if err != nil { + return err + } + + guestKubeconfig, err := kubeconfigFromFile(targetKubeconfig) + if err != nil { + return fmt.Errorf("failed to create guest kubeconfig: %w", err) + } + + mgr, err := ctrl.NewManager(guestKubeconfig, ctrl.Options{ + Scheme: hyperapi.Scheme, + LeaderElection: false, + }) + if err != nil { + return fmt.Errorf("failed to create manager: %w", err) + } + + if err := mgr.Add(managementCluster); err != nil { + return fmt.Errorf("failed to add managementCluster to controller runtime manager: %v", err) + } + + // Add health check endpoints + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return fmt.Errorf("failed to setup healthz check: %w", err) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return fmt.Errorf("failed to setup readyz check: %w", err) + } + + r := karpenter.Reconciler{ + Namespace: namespace, + ControlPlaneOperatorImage: controlPlaneOperatorImage, + } + if err := r.SetupWithManager(ctx, mgr, managementCluster); err != nil { + return fmt.Errorf("failed to setup controller with manager: %w", err) + } + + if err := mgr.Start(ctx); err != nil { + return fmt.Errorf("failed to start manager: %w", err) + } + + return nil +} + +func kubeconfigFromFile(path string) (*rest.Config, error) { + cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: path}, + &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + return nil, fmt.Errorf("failed to construct kubeconfig from path %s: %w", path, err) + } + return cfg, nil +} diff --git a/karpenter-operator/manifests/operator.go b/karpenter-operator/manifests/operator.go new file mode 100644 index 0000000000..e25031aa4e --- /dev/null +++ b/karpenter-operator/manifests/operator.go @@ -0,0 +1,367 @@ +package manifests + +import ( + "context" + "fmt" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" + "github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator" + "github.com/openshift/hypershift/support/config" + "github.com/openshift/hypershift/support/upsert" + "github.com/openshift/hypershift/support/util" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + intstr "k8s.io/apimachinery/pkg/util/intstr" + k8sutilspointer "k8s.io/utils/ptr" + client "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + name = "karpenter-operator" +) + +func KarpenterOperatorDeployment(namespace string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } +} + +func KarpenterOperatorServiceAccount(controlPlaneNamespace string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: controlPlaneNamespace, + Name: name, + }, + } +} + +func KarpenterOperatorRole(controlPlaneNamespace string) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: controlPlaneNamespace, + Name: name, + }, + } +} + +func KarpenterOperatorRoleBinding(controlPlaneNamespace string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: controlPlaneNamespace, + Name: name, + }, + } +} + +func karpenterOperatorSelector() map[string]string { + return map[string]string{ + "karpenter": "karpenter", + } +} + +func ReconcileKarpenterOperatorDeployment(deployment *appsv1.Deployment, + hcp *hyperv1.HostedControlPlane, + sa *corev1.ServiceAccount, + kubeConfigSecret *corev1.Secret, + hypershiftOperatorImage string, + controlPlaneOperatorImage string, + setDefaultSecurityContext bool, + ownerRef config.OwnerRef) error { + + deployment.Spec = appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: karpenterOperatorSelector(), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: karpenterOperatorSelector(), + }, + Spec: corev1.PodSpec{ + ServiceAccountName: sa.Name, + TerminationGracePeriodSeconds: k8sutilspointer.To(int64(10)), + Tolerations: []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "target-kubeconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: kubeConfigSecret.Name, + DefaultMode: k8sutilspointer.To(int32(0640)), + Items: []corev1.KeyToPath{ + { + Key: "value", + Path: "target-kubeconfig", + }, + }, + }, + }, + }, + { + Name: "serviceaccount-token", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: name, + Image: hypershiftOperatorImage, + ImagePullPolicy: corev1.PullAlways, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "target-kubeconfig", + MountPath: "/mnt/kubeconfig", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "MY_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + }, + Command: []string{ + "/usr/bin/karpenter-operator", + }, + Args: []string{ + "--target-kubeconfig=/mnt/kubeconfig/target-kubeconfig", + "--namespace=$(MY_NAMESPACE)", + "--control-plane-operator-image=" + controlPlaneOperatorImage, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 60, + PeriodSeconds: 60, + SuccessThreshold: 1, + FailureThreshold: 5, + TimeoutSeconds: 5, + }, + + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + TimeoutSeconds: 5, + }, + Ports: []corev1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 8000, + }, + { + Name: "http", + ContainerPort: 8081, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + }, + } + + util.AvailabilityProber(kas.InClusterKASReadyURL(hcp.Spec.Platform.Type), controlPlaneOperatorImage, &deployment.Spec.Template.Spec) + deploymentConfig := config.DeploymentConfig{ + AdditionalLabels: map[string]string{ + config.NeedManagementKASAccessLabel: "true", + }, + Scheduling: config.Scheduling{ + PriorityClass: config.DefaultPriorityClass, + }, + SetDefaultSecurityContext: setDefaultSecurityContext, + } + if hcp.Annotations[hyperv1.ControlPlanePriorityClass] != "" { + deploymentConfig.Scheduling.PriorityClass = hcp.Annotations[hyperv1.ControlPlanePriorityClass] + } + + replicas := k8sutilspointer.To(1) + deploymentConfig.SetDefaults(hcp, nil, replicas) + deploymentConfig.SetRestartAnnotation(hcp.ObjectMeta) + deploymentConfig.ApplyTo(deployment) + + return nil +} + +func ReconcileKarpenterOperatorRole(role *rbacv1.Role, owner config.OwnerRef) error { + role.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{ + "leases", + }, + Verbs: []string{ + "get", + "watch", + "create", + }, + }, + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{ + "leases", + }, + Verbs: []string{ + "patch", + "update", + }, + ResourceNames: []string{ + "karpenter-leader-election", + }, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{ + "deployments", + }, + Verbs: []string{ + "create", + "update", + "patch", + "delete", + "get", + "list", + "watch", + }, + }, + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + "serviceaccounts", + }, + Verbs: []string{ + "*", + }, + }, + { + APIGroups: []string{"hypershift.openshift.io"}, + Resources: []string{ + "hostedcontrolplanes", + "hostedcontrolplanes/finalizers", + }, + Verbs: []string{ + "*", + }, + }, + { + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{ + "roles", + "rolebindings", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + "deletecollection", + }, + }, + } + return nil +} + +func ReconcileKarpenterOperatorRoleBinding(binding *rbacv1.RoleBinding, role *rbacv1.Role, sa *corev1.ServiceAccount, owner config.OwnerRef) error { + binding.RoleRef = rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: role.Name, + } + + binding.Subjects = []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: sa.Name, + Namespace: sa.Namespace, + }, + } + + return nil +} + +// ReconcileKarpenter orchestrates reconciliation of karpenter components. +func ReconcileKarpenterOperator(ctx context.Context, createOrUpdate upsert.CreateOrUpdateFN, c client.Client, hypershiftOperatorImage, controlPlaneOperatorImage string, hcp *hyperv1.HostedControlPlane) error { + ownerRef := config.OwnerRefFrom(hcp) + setDefaultSecurityContext := false + + role := KarpenterOperatorRole(hcp.Namespace) + _, err := createOrUpdate(ctx, c, role, func() error { + return ReconcileKarpenterOperatorRole(role, ownerRef) + }) + if err != nil { + return fmt.Errorf("failed to reconcile karpenter role: %w", err) + } + + serviceAccount := KarpenterOperatorServiceAccount(hcp.Namespace) + _, err = createOrUpdate(ctx, c, serviceAccount, func() error { + util.EnsurePullSecret(serviceAccount, controlplaneoperator.PullSecret("").Name) + return nil + }) + if err != nil { + return fmt.Errorf("failed to reconcile karpenter service account: %w", err) + } + + roleBinding := KarpenterOperatorRoleBinding(hcp.Namespace) + _, err = createOrUpdate(ctx, c, roleBinding, func() error { + return ReconcileKarpenterOperatorRoleBinding(roleBinding, role, serviceAccount, ownerRef) + }) + if err != nil { + return fmt.Errorf("failed to reconcile karpenter role binding: %w", err) + } + + // The deployment depends on the kubeconfig being reported. + if hcp.Status.KubeConfig != nil { + // Resolve the kubeconfig secret for CAPI which is used for karpeneter for convience + capiKubeConfigSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: hcp.Namespace, + Name: fmt.Sprintf("%s-kubeconfig", hcp.Spec.InfraID), + }, + } + err = c.Get(ctx, client.ObjectKeyFromObject(capiKubeConfigSecret), capiKubeConfigSecret) + if err != nil { + return fmt.Errorf("failed to get hosted controlplane kubeconfig secret %q: %w", capiKubeConfigSecret.Name, err) + } + + deployment := KarpenterOperatorDeployment(hcp.Namespace) + _, err = createOrUpdate(ctx, c, deployment, func() error { + return ReconcileKarpenterOperatorDeployment(deployment, hcp, serviceAccount, capiKubeConfigSecret, hypershiftOperatorImage, controlPlaneOperatorImage, setDefaultSecurityContext, ownerRef) + }) + if err != nil { + return fmt.Errorf("failed to reconcile karpenter deployment: %w", err) + } + } + + return nil +} diff --git a/support/assets/readasset.go b/support/assets/readasset.go index ac206f064b..63ca5dac44 100644 --- a/support/assets/readasset.go +++ b/support/assets/readasset.go @@ -3,13 +3,13 @@ package assets import ( "fmt" - "github.com/openshift/hypershift/support/api" - hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/api" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -83,6 +83,12 @@ func ShouldNodePool(reader AssetReader, fileName string) *hyperv1.NodePool { return nodePool } +func MustCRD(reader AssetReader, fileName string) *apiextensionsv1.CustomResourceDefinition { + crd := &apiextensionsv1.CustomResourceDefinition{} + deserializeResource(reader, fileName, crd) + return crd +} + func deserializeResource(reader AssetReader, fileName string, obj runtime.Object) { data := MustAsset(reader, fileName) gvks, _, err := api.Scheme.ObjectKinds(obj) diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hosted_controlplane.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hosted_controlplane.go index 0c46641f8d..7dffc7e199 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hosted_controlplane.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hosted_controlplane.go @@ -175,6 +175,10 @@ type HostedControlPlaneSpec struct { // +optional Autoscaling ClusterAutoscaling `json:"autoscaling,omitempty"` + // autoNode specifies the configuration for the autoNode feature. + // +openshift:enable:FeatureGate=AutoNodeKarpenter + AutoNode *AutoNode `json:"autoNode,omitempty"` + // NodeSelector when specified, must be true for the pods managed by the HostedCluster to be scheduled. // // +optional diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go index da3e55f8b1..c541d9639b 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go @@ -460,6 +460,10 @@ type HostedClusterSpec struct { // +optional Autoscaling ClusterAutoscaling `json:"autoscaling,omitempty"` + // autoNode specifies the configuration for the autoNode feature. + // +openshift:enable:FeatureGate=AutoNodeKarpenter + AutoNode *AutoNode `json:"autoNode,omitempty"` + // etcd specifies configuration for the control plane etcd cluster. The // default managementType is Managed. Once set, the managementType cannot be // changed. @@ -1096,6 +1100,47 @@ type Release struct { Image string `json:"image"` } +// We expose here internal configuration knobs that won't be exposed to the service. +type AutoNode struct { + // provisioner is the implementation used for Node auto provisioning. + // +required + Provisioner *ProvisionerConfig `json:"provisionerConfig"` +} + +// ProvisionerConfig is a enum specifying the strategy for auto managing Nodes. +type ProvisionerConfig struct { + // name specifies the name of the provisioner to use. + // +required + // +kubebuilder:validation:Enum=Karpenter + Name Provisioner `json:"name"` + // karpenter specifies the configuration for the Karpenter provisioner. + // +optional + Karpenter *KarpenterConfig `json:"karpenter,omitempty"` +} + +type KarpenterConfig struct { + // platform specifies the platform-specific configuration for Karpenter. + // +required + Platform PlatformType `json:"platform"` + // aws specifies the AWS-specific configuration for Karpenter. + // +optional + AWS *KarpenterAWSConfig `json:"aws,omitempty"` +} + +type KarpenterAWSConfig struct { + //arn specifies the ARN of the Karpenter provisioner. + // +required + RoleARN string `json:"roleARN"` +} + +const ( + ProvisionerKarpeneter Provisioner = "Karpenter" +) + +// provisioner is a enum specifying the strategy for auto managing Nodes. +// +kubebuilder:validation:Enum=Karpenter +type Provisioner string + // ClusterAutoscaling specifies auto-scaling behavior that applies to all // NodePools associated with a control plane. type ClusterAutoscaling struct { diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go index f5c45b362d..850f73bfbc 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -554,6 +554,26 @@ func (in *AllocationPool) DeepCopy() *AllocationPool { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoNode) DeepCopyInto(out *AutoNode) { + *out = *in + if in.Provisioner != nil { + in, out := &in.Provisioner, &out.Provisioner + *out = new(ProvisionerConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoNode. +func (in *AutoNode) DeepCopy() *AutoNode { + if in == nil { + return nil + } + out := new(AutoNode) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureKMSKey) DeepCopyInto(out *AzureKMSKey) { *out = *in @@ -1327,6 +1347,11 @@ func (in *HostedClusterSpec) DeepCopyInto(out *HostedClusterSpec) { in.DNS.DeepCopyInto(&out.DNS) in.Networking.DeepCopyInto(&out.Networking) in.Autoscaling.DeepCopyInto(&out.Autoscaling) + if in.AutoNode != nil { + in, out := &in.AutoNode, &out.AutoNode + *out = new(AutoNode) + (*in).DeepCopyInto(*out) + } in.Etcd.DeepCopyInto(&out.Etcd) if in.Services != nil { in, out := &in.Services, &out.Services @@ -1573,6 +1598,11 @@ func (in *HostedControlPlaneSpec) DeepCopyInto(out *HostedControlPlaneSpec) { **out = **in } in.Autoscaling.DeepCopyInto(&out.Autoscaling) + if in.AutoNode != nil { + in, out := &in.AutoNode, &out.AutoNode + *out = new(AutoNode) + (*in).DeepCopyInto(*out) + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) @@ -1840,6 +1870,41 @@ func (in *KMSSpec) DeepCopy() *KMSSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KarpenterAWSConfig) DeepCopyInto(out *KarpenterAWSConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterAWSConfig. +func (in *KarpenterAWSConfig) DeepCopy() *KarpenterAWSConfig { + if in == nil { + return nil + } + out := new(KarpenterAWSConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KarpenterConfig) DeepCopyInto(out *KarpenterConfig) { + *out = *in + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = new(KarpenterAWSConfig) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterConfig. +func (in *KarpenterConfig) DeepCopy() *KarpenterConfig { + if in == nil { + return nil + } + out := new(KarpenterConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeVirtNodePoolStatus) DeepCopyInto(out *KubeVirtNodePoolStatus) { *out = *in @@ -2986,6 +3051,26 @@ func (in *PowerVSVPC) DeepCopy() *PowerVSVPC { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProvisionerConfig) DeepCopyInto(out *ProvisionerConfig) { + *out = *in + if in.Karpenter != nil { + in, out := &in.Karpenter, &out.Karpenter + *out = new(KarpenterConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerConfig. +func (in *ProvisionerConfig) DeepCopy() *ProvisionerConfig { + if in == nil { + return nil + } + out := new(ProvisionerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Release) DeepCopyInto(out *Release) { *out = *in diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests.yaml b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests.yaml index 608af7ce1f..2b6cc4f5d7 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests.yaml +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests.yaml @@ -96,6 +96,7 @@ hostedclusters.hypershift.openshift.io: Category: "" FeatureGates: - AROHCPManagedIdentities + - AutoNodeKarpenter - DynamicResourceAllocation - ExternalOIDC - HCPPodsLabels @@ -149,6 +150,7 @@ hostedcontrolplanes.hypershift.openshift.io: Category: cluster-api FeatureGates: - AROHCPManagedIdentities + - AutoNodeKarpenter - DynamicResourceAllocation - ExternalOIDC - NetworkDiagnosticsConfig