From 99d72d89d130a60fda2ff7bf3dfc116baf68c1c8 Mon Sep 17 00:00:00 2001 From: Dan Mace Date: Fri, 4 Dec 2020 15:14:33 -0500 Subject: [PATCH] Dynamically discover release image metadata - Dynamically discover release image metadata by extracting the serialized imagestream from the image - Use object references for secrets in the OpenShiftCluster API - Remove base domain from the API and instead compute it from the management cluster DNS config - Add an experimental default cluster generator using the Kustomize workflow - Clean up some of the build machinery --- .gitignore | 3 +- Makefile | 41 +++-- README.md | 40 +++-- api/v1alpha1/hosted_controlplane.go | 12 +- api/v1alpha1/openshiftcluster_types.go | 28 +++- api/v1alpha1/zz_generated.deepcopy.go | 20 +++ config/example-cluster/cluster.yaml | 14 ++ .../imagedefaulter-plugin.yaml | 4 + config/example-cluster/kustomization.yaml | 22 +++ .../imagedefaulter/ImageDefaulter | 6 + ...hift.openshift.io_hostedcontrolplanes.yaml | 17 +- ...rshift.openshift.io_openshiftclusters.yaml | 31 +++- config/hypershift-operator/kustomization.yaml | 6 - .../operator-clusterrole.yaml | 1 + .../operator-deployment.yaml | 8 - hack/update-generated-bindata.sh | 23 --- .../controllers/controlplane.go | 60 ++++--- hypershift-operator/controllers/helpers.go | 19 +++ .../hosted_controlplane_controller.go | 19 ++- hypershift-operator/controllers/infra.go | 20 ++- .../openshiftcluster_controller.go | 3 +- hypershift-operator/main.go | 30 +--- .../releaseinfo/releaseinfo.go | 149 +++++++++++++----- .../controlplane/hypershift/manifests.go | 6 +- .../render/controlplane/roks/manifests.go | 6 +- 25 files changed, 391 insertions(+), 197 deletions(-) create mode 100644 config/example-cluster/cluster.yaml create mode 100644 config/example-cluster/imagedefaulter-plugin.yaml create mode 100644 config/example-cluster/kustomization.yaml create mode 100755 config/example-cluster/plugin/hypershiftplugin/imagedefaulter/ImageDefaulter delete mode 100755 hack/update-generated-bindata.sh create mode 100644 hypershift-operator/controllers/helpers.go diff --git a/.gitignore b/.gitignore index 7a672c03c3..ac2948ff8f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,5 @@ bin .kube -config/hypershift-operator/release-info.json +config/example-cluster/ssh-key +config/example-cluster/pull-secret diff --git a/Makefile b/Makefile index 73a1316ecc..2331c6bc41 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,12 @@ +DIR := ${CURDIR} + # Image URL to use all building/pushing image targets IMG ?= hypershift:latest # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true" -CONTROLLER_GEN=GO111MODULE=on GOFLAGS=-mod=vendor go run sigs.k8s.io/controller-tools/cmd/controller-gen +CONTROLLER_GEN=GO111MODULE=on GOFLAGS=-mod=vendor go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen +BINDATA=GO111MODULE=on GOFLAGS=-mod=vendor go run ./vendor/github.com/kevinburke/go-bindata/go-bindata GO_GCFLAGS ?= -gcflags=all='-N -l' GO=GO111MODULE=on GOFLAGS=-mod=vendor go @@ -27,7 +30,22 @@ verify: build fmt vet # Generate code generate: - hack/update-generated-bindata.sh + $(BINDATA) -mode 420 -modtime 1 -pkg hypershift \ + -o ./hypershift-operator/assets/controlplane/hypershift/bindata.go \ + --prefix hypershift-operator/assets/controlplane/hypershift \ + --ignore bindata.go \ + ./hypershift-operator/assets/controlplane/hypershift/... + + gofmt -s -w ./hypershift-operator/assets/controlplane/hypershift/bindata.go + + $(BINDATA) -mode 420 -modtime 1 -pkg roks \ + -o ./hypershift-operator/assets/controlplane/roks/bindata.go \ + --prefix hypershift-operator/assets/controlplane/roks \ + --ignore bindata.go \ + ./hypershift-operator/assets/controlplane/roks/... + + gofmt -s -w ./hypershift-operator/assets/controlplane/roks/bindata.go + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." # Build hypershift-operator binary @@ -44,10 +62,10 @@ test: build # Generate Kube manifests (e.g. CRDs) manifests: - $(CONTROLLER_GEN) $(CRD_OPTIONS) webhook paths="./..." output:crd:artifacts:config=config/hypershift-operator + $(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./..." output:crd:artifacts:config=config/hypershift-operator # Installs hypershift into a cluster -install: manifests release-info-data +install: manifests kustomize build config/install/$(PROFILE) | oc apply -f - # Uninstalls hypershit from a cluster @@ -56,7 +74,7 @@ uninstall: manifests # Builds the config with Kustomize for manual usage .PHONY: config -config: release-info-data +config: kustomize build config/install/$(PROFILE) # Run go fmt against code @@ -75,8 +93,13 @@ docker-build: docker-push: docker push ${IMG} -release-info-data: - oc adm release info --output json > config/hypershift-operator/release-info.json - run-local: - bin/hypershift-operator run --release-info config/hypershift-operator/release-info.json + bin/hypershift-operator run + +BUILD_EXAMPLE_CLUSTER=KUSTOMIZE_PLUGIN_HOME=$(DIR)/config/example-cluster/plugin kustomize build --enable_alpha_plugins ./config/example-cluster + +example-cluster: + $(BUILD_EXAMPLE_CLUSTER) + +install-example-cluster: + $(BUILD_EXAMPLE_CLUSTER) | oc apply --namespace hypershift -f - diff --git a/README.md b/README.md index 5b4a6ae53a..bd398f7d3c 100644 --- a/README.md +++ b/README.md @@ -24,41 +24,39 @@ $ make uninstall ### Create a cluster -Create a new guest cluster by creating an `OpenShiftCluster` resource. For now, -the cluster will be based on the version of the management cluster itself. +First, create the following files containing secrets used by the example cluster: -Here's an example: +- `config/example-cluster/pull-secret` a valid pull secret for image pulls. +- `config/example-cluster/ssh-key` an SSH public key for guest node access -```yaml -apiVersion: hypershift.openshift.io/v1alpha1 -kind: OpenShiftCluster -metadata: - name: guest-hello -spec: - baseDomain: guest-hello.devcluster.openshift.com - pullSecret: '{"auths": { ... }}' - serviceCIDR: 172.31.0.0/16 - podCIDR: 10.132.0.0/14 - sshKey: 'ssh-rsa ...' - initialComputeReplicas: 1 +Install the example cluster: + +```bash +$ make install-example-cluster +``` + +If you want to see but not apply the example cluster resource (i.e. dry run), try: + +```bash +$ make example-cluster ``` -Get the guest cluster's kubeconfig using: +When the cluster is available, get the guest kubeconfig using: ```bash -$ oc get secret --namespace guest-hello admin-kubeconfig --template={{.data.kubeconfig}} | base64 -D +$ oc get secret --namespace example admin-kubeconfig --template={{.data.kubeconfig}} | base64 -D ``` -You can create additional nodePools: +To create additional node pools, create a resource like: ```yaml apiVersion: hypershift.openshift.io/v1alpha1 kind: NodePool metadata: - name: guest-hello-custom-nodepool namespace: hypershift + name: example-extended spec: - clusterName: guest-hello + clusterName: example autoScaling: max: 0 min: 0 @@ -71,5 +69,5 @@ spec: And delete the cluster using: ```bash -$ oc delete --namespace hypershift openshiftclusters/guest-hello +$ oc delete --namespace hypershift openshiftclusters/example ``` diff --git a/api/v1alpha1/hosted_controlplane.go b/api/v1alpha1/hosted_controlplane.go index d8050f0242..caa965ff71 100644 --- a/api/v1alpha1/hosted_controlplane.go +++ b/api/v1alpha1/hosted_controlplane.go @@ -1,6 +1,7 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -24,12 +25,11 @@ type HostedControlPlane struct { // HostedControlPlaneSpec defines the desired state of HostedControlPlane type HostedControlPlaneSpec struct { - BaseDomain string `json:"baseDomain"` - PullSecret string `json:"pullSecret"` - ServiceCIDR string `json:"serviceCIDR"` - PodCIDR string `json:"podCIDR"` - SSHKey string `json:"sshKey"` - ReleaseImage string `json:"releaseImage"` + ReleaseImage string `json:"releaseImage"` + PullSecret corev1.LocalObjectReference `json:"pullSecret"` + ServiceCIDR string `json:"serviceCIDR"` + PodCIDR string `json:"podCIDR"` + SSHKey corev1.LocalObjectReference `json:"sshKey"` } // HostedControlPlaneStatus defines the observed state of HostedControlPlane diff --git a/api/v1alpha1/openshiftcluster_types.go b/api/v1alpha1/openshiftcluster_types.go index ed7811be40..b19949c399 100644 --- a/api/v1alpha1/openshiftcluster_types.go +++ b/api/v1alpha1/openshiftcluster_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -28,13 +29,26 @@ type OpenShiftClusterSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - BaseDomain string `json:"baseDomain"` - PullSecret string `json:"pullSecret"` - ServiceCIDR string `json:"serviceCIDR"` - PodCIDR string `json:"podCIDR"` - SSHKey string `json:"sshKey"` - InitialComputeReplicas int `json:"initialComputeReplicas"` - ReleaseImage string `json:"releaseImage"` + Release ReleaseSpec `json:"release"` + + InitialComputeReplicas int `json:"initialComputeReplicas"` + + // PullSecret is a pull secret injected into the container runtime of guest + // workers. It should have an ".dockerconfigjson" key containing the pull secret JSON. + PullSecret corev1.LocalObjectReference `json:"pullSecret"` + + SSHKey corev1.LocalObjectReference `json:"sshKey"` + + ServiceCIDR string `json:"serviceCIDR"` + PodCIDR string `json:"podCIDR"` +} + +type ReleaseSpec struct { + // +kubebuilder:validation:Optional + Channel string `json:"channel"` + // Image is the release image pullspec for the control plane + // +kubebuilder:validation:Required + Image string `json:"image"` } // OpenShiftClusterStatus defines the observed state of OpenShiftCluster diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6a330dbaaa..a33189207e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -206,6 +206,8 @@ func (in *HostedControlPlaneList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HostedControlPlaneSpec) DeepCopyInto(out *HostedControlPlaneSpec) { *out = *in + out.PullSecret = in.PullSecret + out.SSHKey = in.SSHKey } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostedControlPlaneSpec. @@ -422,6 +424,9 @@ func (in *OpenShiftClusterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenShiftClusterSpec) DeepCopyInto(out *OpenShiftClusterSpec) { *out = *in + out.Release = in.Release + out.PullSecret = in.PullSecret + out.SSHKey = in.SSHKey } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenShiftClusterSpec. @@ -448,3 +453,18 @@ func (in *OpenShiftClusterStatus) DeepCopy() *OpenShiftClusterStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReleaseSpec) DeepCopyInto(out *ReleaseSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReleaseSpec. +func (in *ReleaseSpec) DeepCopy() *ReleaseSpec { + if in == nil { + return nil + } + out := new(ReleaseSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/example-cluster/cluster.yaml b/config/example-cluster/cluster.yaml new file mode 100644 index 0000000000..d482451251 --- /dev/null +++ b/config/example-cluster/cluster.yaml @@ -0,0 +1,14 @@ +apiVersion: hypershift.openshift.io/v1alpha1 +kind: OpenShiftCluster +metadata: + name: example +spec: + release: + image: quay.io/openshift-release-dev/ocp-release:4.6.7-x86_64 + initialComputeReplicas: 1 + serviceCIDR: 172.30.0.0/16 + podCIDR: 10.128.0.0/14 + pullSecret: + name: pull-secret + sshKey: + name: ssh-key diff --git a/config/example-cluster/imagedefaulter-plugin.yaml b/config/example-cluster/imagedefaulter-plugin.yaml new file mode 100644 index 0000000000..592561049e --- /dev/null +++ b/config/example-cluster/imagedefaulter-plugin.yaml @@ -0,0 +1,4 @@ +apiVersion: hypershiftplugin +kind: ImageDefaulter +metadata: + name: imagedefaulter diff --git a/config/example-cluster/kustomization.yaml b/config/example-cluster/kustomization.yaml new file mode 100644 index 0000000000..9108e5f14d --- /dev/null +++ b/config/example-cluster/kustomization.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- cluster.yaml + +transformers: +- imagedefaulter-plugin.yaml + +secretGenerator: +- name: pull-secret + options: + disableNameSuffixHash: true + files: + - .dockerconfigjson=pull-secret + type: Opaque +- name: ssh-key + options: + disableNameSuffixHash: true + files: + - id_rsa.pub=ssh-key + type: Opaque diff --git a/config/example-cluster/plugin/hypershiftplugin/imagedefaulter/ImageDefaulter b/config/example-cluster/plugin/hypershiftplugin/imagedefaulter/ImageDefaulter new file mode 100755 index 0000000000..79e4430d9b --- /dev/null +++ b/config/example-cluster/plugin/hypershiftplugin/imagedefaulter/ImageDefaulter @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +FROM="quay.io/openshift-release-dev/ocp-release:4.6.7-x86_64" +TO="$(oc get clusterversion/version -o jsonpath='{.status.desired.image}')" +cat | sed s#$FROM#$TO#g diff --git a/config/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml b/config/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml index 55028a8a61..796a73ce4d 100644 --- a/config/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml +++ b/config/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml @@ -37,20 +37,27 @@ spec: spec: description: HostedControlPlaneSpec defines the desired state of HostedControlPlane properties: - baseDomain: - type: string podCIDR: type: string pullSecret: - type: string + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object releaseImage: type: string serviceCIDR: type: string sshKey: - type: string + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object required: - - baseDomain - podCIDR - pullSecret - releaseImage diff --git a/config/hypershift-operator/hypershift.openshift.io_openshiftclusters.yaml b/config/hypershift-operator/hypershift.openshift.io_openshiftclusters.yaml index e77223c93b..e77e663b4a 100644 --- a/config/hypershift-operator/hypershift.openshift.io_openshiftclusters.yaml +++ b/config/hypershift-operator/hypershift.openshift.io_openshiftclusters.yaml @@ -35,26 +35,41 @@ spec: spec: description: OpenShiftClusterSpec defines the desired state of OpenShiftCluster properties: - baseDomain: - type: string initialComputeReplicas: type: integer podCIDR: type: string pullSecret: - type: string - releaseImage: - type: string + description: PullSecret is a pull secret injected into the container runtime of guest workers. It should have an ".dockerconfigjson" key containing the pull secret JSON. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + release: + properties: + channel: + type: string + image: + description: Image is the release image pullspec for the control plane + type: string + required: + - image + type: object serviceCIDR: type: string sshKey: - type: string + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object required: - - baseDomain - initialComputeReplicas - podCIDR - pullSecret - - releaseImage + - release - serviceCIDR - sshKey type: object diff --git a/config/hypershift-operator/kustomization.yaml b/config/hypershift-operator/kustomization.yaml index 96e779159f..ed18109e43 100644 --- a/config/hypershift-operator/kustomization.yaml +++ b/config/hypershift-operator/kustomization.yaml @@ -12,12 +12,6 @@ resources: - operator-clusterrolebinding.yaml - operator-deployment.yaml -configMapGenerator: -- name: release-info - namespace: hypershift - files: - - release-info.json - patchesStrategicMerge: - |- apiVersion: apiextensions.k8s.io/v1 diff --git a/config/hypershift-operator/operator-clusterrole.yaml b/config/hypershift-operator/operator-clusterrole.yaml index 2c5632fd31..1986a17ad0 100644 --- a/config/hypershift-operator/operator-clusterrole.yaml +++ b/config/hypershift-operator/operator-clusterrole.yaml @@ -67,6 +67,7 @@ rules: - events - configmaps - pods + - pods/log - secrets - nodes - namespaces diff --git a/config/hypershift-operator/operator-deployment.yaml b/config/hypershift-operator/operator-deployment.yaml index 58107d3617..ad74136fb7 100644 --- a/config/hypershift-operator/operator-deployment.yaml +++ b/config/hypershift-operator/operator-deployment.yaml @@ -13,20 +13,12 @@ spec: labels: name: operator spec: - volumes: - - name: release-info - configMap: - name: release-info serviceAccountName: operator containers: - name: operator image: hypershift:latest imagePullPolicy: Always - volumeMounts: - - name: release-info - mountPath: /etc/release-info command: - /usr/bin/hypershift-operator args: - run - - --release-info=/etc/release-info/release-info.json diff --git a/hack/update-generated-bindata.sh b/hack/update-generated-bindata.sh deleted file mode 100755 index 268b329193..0000000000 --- a/hack/update-generated-bindata.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -euo pipefail - -OUTDIR="${OUTDIR:-$PWD}" - -# Using "-modtime 1" to make generate target deterministic. It sets all file -# time stamps to unix timestamp 1 -GO111MODULE=on GOFLAGS=-mod=vendor go run github.com/kevinburke/go-bindata/go-bindata -mode 420 -modtime 1 \ - -pkg hypershift \ - -o ${OUTDIR}/hypershift-operator/assets/controlplane/hypershift/bindata.go \ - --prefix hypershift-operator/assets/controlplane/hypershift \ - --ignore bindata.go \ - hypershift-operator/assets/controlplane/hypershift/... - -GO111MODULE=on GOFLAGS=-mod=vendor go run github.com/kevinburke/go-bindata/go-bindata -mode 420 -modtime 1 \ - -pkg roks \ - -o ${OUTDIR}/hypershift-operator/assets/controlplane/roks/bindata.go \ - --prefix hypershift-operator/assets/controlplane/roks \ - --ignore bindata.go \ - hypershift-operator/assets/controlplane/roks/... - -gofmt -s -w ${OUTDIR}/hypershift-operator/assets/controlplane/hypershift/bindata.go -gofmt -s -w ${OUTDIR}/hypershift-operator/assets/controlplane/roks/bindata.go diff --git a/hypershift-operator/controllers/controlplane.go b/hypershift-operator/controllers/controlplane.go index 3b1f8577e4..8e3cd0df87 100644 --- a/hypershift-operator/controllers/controlplane.go +++ b/hypershift-operator/controllers/controlplane.go @@ -27,12 +27,13 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/yaml" + client "sigs.k8s.io/controller-runtime/pkg/client" + hyperv1 "openshift.io/hypershift/api/v1alpha1" "openshift.io/hypershift/hypershift-operator/releaseinfo" hypershiftcp "openshift.io/hypershift/hypershift-operator/render/controlplane/hypershift" "openshift.io/hypershift/hypershift-operator/render/controlplane/hypershift/pki" rokscp "openshift.io/hypershift/hypershift-operator/render/controlplane/roks" - client "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -53,16 +54,7 @@ var ( version46 = semver.MustParse("4.6.0") ) -type CreateClusterOpts struct { - Directory string - Config hyperv1.OpenShiftCluster - ReleaseImageInfo *releaseinfo.ReleaseImageInfo - ControlPlaneOperatorImage string - - Client client.Client -} - -func (r *HostedControlPlaneReconciler) ensureControlPlane(ctx context.Context, hcp *hyperv1.HostedControlPlane, infraStatus InfrastructureStatus, releaseInfo *releaseinfo.ReleaseImageInfo) error { +func (r *HostedControlPlaneReconciler) ensureControlPlane(ctx context.Context, hcp *hyperv1.HostedControlPlane, infraStatus InfrastructureStatus, releaseImage *releaseinfo.ReleaseImage) error { workingDir, err := ioutil.TempDir("", "hypershift") if err != nil { return fmt.Errorf("failed to create temporary directory: %w", err) @@ -83,22 +75,40 @@ func (r *HostedControlPlaneReconciler) ensureControlPlane(ctx context.Context, h name := hcp.Name + var pullSecret corev1.Secret + err = r.Client.Get(ctx, client.ObjectKey{Namespace: hcp.Namespace, Name: hcp.Spec.PullSecret.Name}, &pullSecret) + if err != nil { + return fmt.Errorf("failed to get pull secret %s: %w", hcp.Spec.PullSecret.Name, err) + } + pullSecretData, hasPullSecretData := pullSecret.Data[".dockerconfigjson"] + if !hasPullSecretData { + return fmt.Errorf("pull secret %s is missing the .dockerconfigjson key", hcp.Spec.PullSecret.Name) + } pullSecretFile := filepath.Join(workingDir, "pull-secret") - if err := ioutil.WriteFile(pullSecretFile, []byte(hcp.Spec.PullSecret), 0644); err != nil { + if err := ioutil.WriteFile(pullSecretFile, pullSecretData, 0644); err != nil { return fmt.Errorf("failed to create temporary pull secret file: %v", err) } - releaseVersion, err := releaseInfo.ReleaseVersion() - if err != nil { - return fmt.Errorf("cannot obtain release version: %v", err) - } - version, err := semver.Parse(releaseVersion) + version, err := semver.Parse(releaseImage.Version()) if err != nil { - return fmt.Errorf("cannot parse release version (%s): %v", releaseVersion, err) + return fmt.Errorf("cannot parse release version (%s): %v", releaseImage.Version(), err) } controlPlaneOperatorImage, err := r.LookupControlPlaneOperatorImage(r.Client) if err != nil { return fmt.Errorf("failed to lookup control plane operator image: %w", err) } + var sshKeySecret corev1.Secret + err = r.Client.Get(ctx, client.ObjectKey{Namespace: hcp.Namespace, Name: hcp.Spec.SSHKey.Name}, &sshKeySecret) + if err != nil { + return fmt.Errorf("failed to get SSH key secret %s: %w", hcp.Spec.SSHKey.Name, err) + } + sshKeyData, hasSSHKeyData := sshKeySecret.Data["id_rsa.pub"] + if !hasSSHKeyData { + return fmt.Errorf("SSH key secret secret %s is missing the id_rsa.pub key", hcp.Spec.SSHKey.Name) + } + baseDomain, err := ClusterBaseDomain(r.Client, ctx, hcp.Name) + if err != nil { + return fmt.Errorf("couldn't determine cluster base domain name: %w", err) + } params := hypershiftcp.NewClusterParams() params.Namespace = name @@ -111,10 +121,10 @@ func (r *HostedControlPlaneReconciler) ensureControlPlane(ctx context.Context, h params.ExternalOauthPort = externalOauthPort params.ServiceCIDR = hcp.Spec.ServiceCIDR params.PodCIDR = hcp.Spec.PodCIDR - params.ReleaseImage = releaseInfo.Image - params.IngressSubdomain = fmt.Sprintf("apps.%s", hcp.Spec.BaseDomain) + params.ReleaseImage = hcp.Spec.ReleaseImage + params.IngressSubdomain = fmt.Sprintf("apps.%s", baseDomain) params.OpenShiftAPIClusterIP = infraStatus.OpenShiftAPIAddress - params.BaseDomain = hcp.Spec.BaseDomain + params.BaseDomain = baseDomain params.MachineConfigServerAddress = infraStatus.IgnitionProviderAddress params.CloudProvider = string(r.Infra.Status.PlatformStatus.Type) params.PlatformType = string(r.Infra.Status.PlatformStatus.Type) @@ -123,7 +133,7 @@ func (r *HostedControlPlaneReconciler) ensureControlPlane(ctx context.Context, h params.NetworkType = "OpenShiftSDN" params.ImageRegistryHTTPSecret = generateImageRegistrySecret() params.Replicas = "1" - params.SSHKey = hcp.Spec.SSHKey + params.SSHKey = string(sshKeyData) params.ControlPlaneOperatorImage = controlPlaneOperatorImage params.HypershiftOperatorControllers = []string{"route-sync", "auto-approver", "kubeadmin-password", "node"} @@ -184,11 +194,11 @@ func (r *HostedControlPlaneReconciler) ensureControlPlane(ctx context.Context, h } params.OpenshiftAPIServerCABundle = base64.StdEncoding.EncodeToString(caBytes) - if err = rokscp.RenderClusterManifests(&rokscp.ClusterParams{ClusterParams: *params}, releaseInfo, pullSecretFile, manifestsDir, true, false); err != nil { + if err = rokscp.RenderClusterManifests(&rokscp.ClusterParams{ClusterParams: *params}, releaseImage, pullSecretFile, manifestsDir, true, false); err != nil { return fmt.Errorf("failed to render roks manifests for cluster: %w", err) } - if err = hypershiftcp.RenderClusterManifests(params, releaseInfo, pullSecretFile, pkiDir, manifestsDir, true, true, true, true); err != nil { + if err = hypershiftcp.RenderClusterManifests(params, releaseImage, pullSecretFile, pkiDir, manifestsDir, true, true, true, true); err != nil { return fmt.Errorf("failed to render hypershift manifests for cluster: %w", err) } @@ -278,7 +288,7 @@ func (r *HostedControlPlaneReconciler) ensureControlPlane(ctx context.Context, h return fmt.Errorf("failed to generate kubeconfigSecret: %w", err) } - targetPullSecret, err := generateTargetPullSecret(r.Scheme(), []byte(hcp.Spec.PullSecret), name) + targetPullSecret, err := generateTargetPullSecret(r.Scheme(), pullSecretData, name) if err != nil { return fmt.Errorf("failed to create pull secret manifest for target cluster: %w", err) } diff --git a/hypershift-operator/controllers/helpers.go b/hypershift-operator/controllers/helpers.go new file mode 100644 index 0000000000..9232bb5c5c --- /dev/null +++ b/hypershift-operator/controllers/helpers.go @@ -0,0 +1,19 @@ +package controllers + +import ( + "context" + "fmt" + + configv1 "github.com/openshift/api/config/v1" + ctrl "sigs.k8s.io/controller-runtime/pkg/client" +) + +// TODO: base domain isn't actually part of our API +func ClusterBaseDomain(client ctrl.Client, ctx context.Context, clusterName string) (string, error) { + var dnsConfig configv1.DNS + err := client.Get(ctx, ctrl.ObjectKey{Name: "cluster"}, &dnsConfig) + if err != nil { + return "", fmt.Errorf("failed to get cluster dns config: %w", err) + } + return fmt.Sprintf("%s.%s", clusterName, dnsConfig.Spec.BaseDomain), nil +} diff --git a/hypershift-operator/controllers/hosted_controlplane_controller.go b/hypershift-operator/controllers/hosted_controlplane_controller.go index 48dab79560..89061d8630 100644 --- a/hypershift-operator/controllers/hosted_controlplane_controller.go +++ b/hypershift-operator/controllers/hosted_controlplane_controller.go @@ -30,7 +30,7 @@ type HostedControlPlaneReconciler struct { Infra *configv1.Infrastructure recorder record.EventRecorder LookupControlPlaneOperatorImage func(kubeClient client.Client) (string, error) - LookupReleaseInfo func(image string) (*releaseinfo.ReleaseImageInfo, error) + ReleaseProvider releaseinfo.Provider } func (r *HostedControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -125,12 +125,6 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R return result, nil } - releaseInfo, err := r.LookupReleaseInfo(hostedControlPlane.Spec.ReleaseImage) - if err != nil { - result.RequeueAfter = 5 * time.Second - return result, fmt.Errorf("no release info found for image %q: %w", hostedControlPlane.Spec.ReleaseImage, err) - } - // First, set up infrastructure infraStatus, err := r.ensureInfrastructure(ctx, hostedControlPlane) if err != nil { @@ -149,8 +143,17 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R Port: APIServerPort, } + releaseImage, err := r.ReleaseProvider.Lookup(ctx, hostedControlPlane.Spec.ReleaseImage) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to look up release info: %w", err) + } + componentVersions, err := releaseImage.ComponentVersions() + if err != nil { + return ctrl.Result{}, fmt.Errorf("invalid component versions found in release info: %w", err) + } + r.Log.Info("found release info for image", "releaseImage", hostedControlPlane.Spec.ReleaseImage, "info", releaseImage, "componentImages", releaseImage.ComponentImages(), "componentVersions", componentVersions) // Install the control plane into the infrastructure - err = r.ensureControlPlane(ctx, hostedControlPlane, infraStatus, releaseInfo) + err = r.ensureControlPlane(ctx, hostedControlPlane, infraStatus, releaseImage) if err != nil { r.Log.Error(err, "failed to ensure control plane") return result, fmt.Errorf("failed to ensure control plane: %w", err) diff --git a/hypershift-operator/controllers/infra.go b/hypershift-operator/controllers/infra.go index 720e43ca6e..9387b4c07a 100644 --- a/hypershift-operator/controllers/infra.go +++ b/hypershift-operator/controllers/infra.go @@ -61,10 +61,24 @@ func (r *HostedControlPlaneReconciler) ensureInfrastructure(ctx context.Context, // Create pull secret r.Log.Info("Creating pull secret") - if _, err := createPullSecret(r, name, hcp.Spec.PullSecret); err != nil { + var pullSecret corev1.Secret + err = r.Client.Get(ctx, ctrl.ObjectKey{Namespace: hcp.Namespace, Name: hcp.Spec.PullSecret.Name}, &pullSecret) + if err != nil { + return status, fmt.Errorf("failed to get pull secret %s: %w", hcp.Spec.PullSecret.Name, err) + } + pullSecretData, hasPullSecretData := pullSecret.Data[".dockerconfigjson"] + if !hasPullSecretData { + return status, fmt.Errorf("pull secret %s is missing the .dockerconfigjson key", hcp.Spec.PullSecret.Name) + } + if _, err := createPullSecret(r, name, pullSecretData); err != nil { return status, fmt.Errorf("failed to create pull secret: %w", err) } + baseDomain, err := ClusterBaseDomain(r.Client, ctx, hcp.Name) + if err != nil { + return status, fmt.Errorf("couldn't determine cluster base domain name: %w", err) + } + // Create Kube APIServer service r.Log.Info("Creating Kube API service") apiService, err := createKubeAPIServerService(r, name) @@ -94,7 +108,7 @@ func (r *HostedControlPlaneReconciler) ensureInfrastructure(ctx context.Context, } r.Log.Info("Creating router shard") - if err := createIngressController(r, name, hcp.Spec.BaseDomain); err != nil { + if err := createIngressController(r, name, baseDomain); err != nil { return status, fmt.Errorf("cannot create router shard: %w", err) } @@ -220,7 +234,7 @@ func createOauthService(client ctrl.Client, namespace string) (*corev1.Service, return svc, nil } -func createPullSecret(client ctrl.Client, namespace, data string) (*corev1.Secret, error) { +func createPullSecret(client ctrl.Client, namespace string, data []byte) (*corev1.Secret, error) { secret := &corev1.Secret{} secret.Namespace = namespace secret.Name = "pull-secret" diff --git a/hypershift-operator/controllers/openshiftcluster_controller.go b/hypershift-operator/controllers/openshiftcluster_controller.go index b24f23d432..87ff1cf7d6 100644 --- a/hypershift-operator/controllers/openshiftcluster_controller.go +++ b/hypershift-operator/controllers/openshiftcluster_controller.go @@ -127,12 +127,11 @@ func (r *OpenShiftClusterReconciler) Reconcile(ctx context.Context, req ctrl.Req Name: ocluster.GetName(), }, Spec: hyperv1.HostedControlPlaneSpec{ - BaseDomain: ocluster.Spec.BaseDomain, PullSecret: ocluster.Spec.PullSecret, ServiceCIDR: ocluster.Spec.ServiceCIDR, PodCIDR: ocluster.Spec.PodCIDR, SSHKey: ocluster.Spec.SSHKey, - ReleaseImage: ocluster.Spec.ReleaseImage, + ReleaseImage: ocluster.Spec.Release.Image, }, } guestCluster := &hyperv1.GuestCluster{ diff --git a/hypershift-operator/main.go b/hypershift-operator/main.go index cbda8784c2..fb9ee4cd90 100644 --- a/hypershift-operator/main.go +++ b/hypershift-operator/main.go @@ -18,9 +18,7 @@ package main import ( "context" - "encoding/json" "fmt" - "io/ioutil" "os" configv1 "github.com/openshift/api/config/v1" @@ -31,6 +29,7 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -86,14 +85,12 @@ func NewStartCommand() *cobra.Command { var metricsAddr string var enableLeaderElection bool var controlPlaneOperatorImage string - var releaseInfoFile string cmd.Flags().StringVar(&metricsAddr, "metrics-addr", "0", "The address the metric endpoint binds to.") cmd.Flags().BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") cmd.Flags().StringVar(&controlPlaneOperatorImage, "control-plane-operator-image", "", "A control plane operator image.") - cmd.Flags().StringVar(&releaseInfoFile, "release-info", "", "A static release info JSON file to use for all guest clusters") cmd.Run = func(cmd *cobra.Command, args []string) { ctrl.SetLogger(zap.New(zap.UseDevMode(true))) @@ -134,23 +131,10 @@ func NewStartCommand() *cobra.Command { return image, nil } - // For now read release info from a file to keep it simple and externalize - // the complexity of the ways we currently know to get the data (which involves - // authenticated registry interactions interactions via `oc adm release info`, etc.) - lookupReleaseInfo := func(image string) (*releaseinfo.ReleaseImageInfo, error) { - if len(releaseInfoFile) == 0 { - return nil, fmt.Errorf("release-info is currently required") - } - contents, err := ioutil.ReadFile(releaseInfoFile) - if err != nil { - return nil, fmt.Errorf("failed to read release info file %s: %w", releaseInfoFile, err) - } - var info releaseinfo.ReleaseImageInfo - err = json.Unmarshal(contents, &info) - if err != nil { - return nil, fmt.Errorf("invalid release info file %s: %w", releaseInfoFile, err) - } - return &info, nil + kubeClient, err := kubernetes.NewForConfig(mgr.GetConfig()) + if err != nil { + setupLog.Error(err, "unable to create kube client") + os.Exit(1) } if err = (&controllers.OpenShiftClusterReconciler{ @@ -163,7 +147,9 @@ func NewStartCommand() *cobra.Command { if err := (&controllers.HostedControlPlaneReconciler{ Client: mgr.GetClient(), LookupControlPlaneOperatorImage: lookupControlPlaneOperatorImage, - LookupReleaseInfo: lookupReleaseInfo, + ReleaseProvider: &releaseinfo.PodProvider{ + Pods: kubeClient.CoreV1().Pods("hypershift"), + }, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "hostedControlPlane") os.Exit(1) diff --git a/hypershift-operator/releaseinfo/releaseinfo.go b/hypershift-operator/releaseinfo/releaseinfo.go index ae994d6bbd..96c327e329 100644 --- a/hypershift-operator/releaseinfo/releaseinfo.go +++ b/hypershift-operator/releaseinfo/releaseinfo.go @@ -2,72 +2,147 @@ package releaseinfo import ( "bytes" + "context" + "encoding/json" "fmt" + "io/ioutil" "regexp" "sort" "strings" + "time" "github.com/blang/semver" imageapi "github.com/openshift/api/image/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + v1 "k8s.io/client-go/kubernetes/typed/core/v1" ) -type ReleaseImageInfo struct { - Image string `json:"image"` - Metadata ReleaseImageInfoMetadata `json:"metadata"` - References imageapi.ImageStream `json:"references"` +// Provider knows how to find the release image metadata for an image referred +// to by its pullspec. +type Provider interface { + Lookup(ctx context.Context, image string) (*ReleaseImage, error) } -type ReleaseImageInfoMetadata struct { - Kind string `json:"kind"` - Version string `json:"version"` -} +var _ Provider = (*PodProvider)(nil) + +// PodProvider finds the release image metadata for an image by launching a pod +// using the image and extracting the serialized ImageStream from the image +// filesystem assumed to be present at /release-manifests/image-references. +type PodProvider struct { + Pods v1.PodInterface -type ReleaseInfo struct { - Images map[string]string - Versions map[string]string + // TODO: consider something like ExpirationCache if performance becomes an issue } -func (info *ReleaseImageInfo) ReleaseInfo(originReleasePrefix string) (*ReleaseInfo, error) { - var newImagePrefix string - if !strings.Contains(info.Image, originReleasePrefix) { - newImagePrefix = strings.Replace(info.Image, ":", "-", -1) +func (p *PodProvider) Lookup(ctx context.Context, image string) (releaseImage *ReleaseImage, err error) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "hypershift", + GenerateName: "image-lookup", + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "lookup", + Image: image, + Command: []string{"/usr/bin/cat", "/release-manifests/image-references"}, + }, + }, + }, } - images := make(map[string]string) - for _, tag := range info.References.Spec.Tags { - name := tag.From.Name - if len(newImagePrefix) > 0 { - name = fmt.Sprintf("%s@%s", newImagePrefix, strings.Split(tag.From.Name, "@")[1]) + + // Launch the pod and ensure we clean up regardless of outcome + pod, err = p.Pods.Create(context.TODO(), pod, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to create image lookup pod: %w", err) + } + defer func() { + err := p.Pods.Delete(ctx, pod.Name, metav1.DeleteOptions{}) + if err != nil { + err = fmt.Errorf("failed to delete image lookup pod %q: %w", pod.Name, err) + } + }() + + // Wait for the pod to reach a terminate state + err = wait.PollImmediateUntil(1*time.Second, func() (bool, error) { + pod, err := p.Pods.Get(ctx, pod.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + switch pod.Status.Phase { + case corev1.PodSucceeded: + return true, nil + case corev1.PodFailed: + return true, fmt.Errorf("image lookup pod failed") + default: + return false, nil + } + }, ctx.Done()) + if err != nil { + return nil, fmt.Errorf("failed waiting for image lookup pod %q: %w", pod.Name, err) + } + + // Try and extract the pod's logs + req := p.Pods.GetLogs(pod.Name, &corev1.PodLogOptions{}) + logs, err := req.Stream(ctx) + if err != nil { + return nil, fmt.Errorf("failed to read image lookup pod %q logs: %w", pod.Name, err) + } + defer func() { + err := logs.Close() + if err != nil { + err = fmt.Errorf("failed to close pod %q log stream: %w", pod.Name, err) } - images[tag.Name] = name + }() + data, err := ioutil.ReadAll(logs) + + // The logs should be a serialized ImageStream resource + var imageStream imageapi.ImageStream + err = json.Unmarshal(data, &imageStream) + if err != nil { + return nil, fmt.Errorf("couldn't read image lookup pod %q logs as a serialized ImageStream: %w\nraw logs:\n%s", pod.Name, err, string(data)) } + releaseImage = &ReleaseImage{ImageStream: &imageStream} + return +} - componentVersions, err := readComponentVersions(&info.References) +// ReleaseImage wraps an ImageStream with some utilities that help the user +// discover constituent component image information. +type ReleaseImage struct { + *imageapi.ImageStream +} + +func (i *ReleaseImage) Version() string { + return i.ImageStream.Name +} + +func (i *ReleaseImage) ComponentImages() map[string]string { + images := make(map[string]string) + for _, tag := range i.ImageStream.Spec.Tags { + images[tag.Name] = tag.From.Name + } + return images +} + +func (i *ReleaseImage) ComponentVersions() (map[string]string, error) { + componentVersions, err := readComponentVersions(i.ImageStream) if err := errors.NewAggregate(err); err != nil { return nil, err } versions := make(map[string]string) - if len(info.Metadata.Version) > 0 { - versions["release"] = info.Metadata.Version + if len(i.ImageStream.Name) > 0 { + versions["release"] = i.ImageStream.Name } for component, version := range componentVersions { versions[component] = version.String() } - - return &ReleaseInfo{ - Images: images, - Versions: versions, - }, nil -} - -func (info *ReleaseImageInfo) ReleaseVersion() (string, error) { - releaseInfo, err := info.ReleaseInfo("") - if err != nil { - return "", err - } - return releaseInfo.Versions["release"], nil + return versions, nil } const ( diff --git a/hypershift-operator/render/controlplane/hypershift/manifests.go b/hypershift-operator/render/controlplane/hypershift/manifests.go index eb760eabe1..2ec7f55009 100644 --- a/hypershift-operator/render/controlplane/hypershift/manifests.go +++ b/hypershift-operator/render/controlplane/hypershift/manifests.go @@ -11,12 +11,12 @@ import ( ) // RenderClusterManifests renders manifests for a hosted control plane cluster -func RenderClusterManifests(params *ClusterParams, releaseImageInfo *releaseinfo.ReleaseImageInfo, pullSecretFile, pkiDir, outputDir string, etcd bool, vpn bool, externalOauth bool, includeRegistry bool) error { - releaseInfo, err := releaseImageInfo.ReleaseInfo(params.OriginReleasePrefix) +func RenderClusterManifests(params *ClusterParams, image *releaseinfo.ReleaseImage, pullSecretFile, pkiDir, outputDir string, etcd bool, vpn bool, externalOauth bool, includeRegistry bool) error { + componentVersions, err := image.ComponentVersions() if err != nil { return err } - ctx := newClusterManifestContext(releaseInfo.Images, releaseInfo.Versions, params, pkiDir, outputDir, vpn, pullSecretFile) + ctx := newClusterManifestContext(image.ComponentImages(), componentVersions, params, pkiDir, outputDir, vpn, pullSecretFile) ctx.setupManifests(etcd, vpn, externalOauth, includeRegistry) return ctx.renderManifests() } diff --git a/hypershift-operator/render/controlplane/roks/manifests.go b/hypershift-operator/render/controlplane/roks/manifests.go index 8205f2322c..4f197db1e5 100644 --- a/hypershift-operator/render/controlplane/roks/manifests.go +++ b/hypershift-operator/render/controlplane/roks/manifests.go @@ -11,13 +11,13 @@ import ( ) // RenderClusterManifests renders manifests for a hosted control plane cluster -func RenderClusterManifests(params *ClusterParams, releaseImageInfo *releaseinfo.ReleaseImageInfo, pullSecretFile, outputDir string, externalOauth, includeRegistry bool) error { - releaseInfo, err := releaseImageInfo.ReleaseInfo(params.OriginReleasePrefix) +func RenderClusterManifests(params *ClusterParams, image *releaseinfo.ReleaseImage, pullSecretFile, outputDir string, externalOauth, includeRegistry bool) error { + componentVersions, err := image.ComponentVersions() if err != nil { return err } includeMetrics := len(params.ROKSMetricsImage) > 0 - ctx := newClusterManifestContext(releaseInfo.Images, releaseInfo.Versions, params, outputDir) + ctx := newClusterManifestContext(image.ComponentImages(), componentVersions, params, outputDir) ctx.setupManifests(externalOauth, includeRegistry, includeMetrics) return ctx.renderManifests() }