From d91566c888a008c823384446646176fab80c21e7 Mon Sep 17 00:00:00 2001 From: Dan Mace Date: Tue, 1 Dec 2020 19:35:36 -0500 Subject: [PATCH] Improve installation This commit introduces some major refactoring to make installation easier. * Get rid of the `manifests` directory and handle all configuration through Kustomize. * Add a deployment (and all supporting resources) for the operator itself. * Introduce reasonable defaults for discovering the control plane image to use based on the operator pod (with the ability to override with a flag). * Abstract away the release image metadata handling and integrate it with Kustomize. For now all guest clusters will use same release version as the management cluster, but the commit also also cleans up some of the plugin points to help us improve in a followup. * Clean up and improve the Makefile to be the entrypoint for all of these tasks. * Simplify the installation procedures in the README. --- .gitignore | 2 + HACKING.md | 24 ++ Makefile | 84 +++-- README.md | 49 +-- .../cluster.x-k8s.io_clusters.yaml | 0 .../cluster.x-k8s.io_machinedeployments.yaml | 0 .../cluster.x-k8s.io_machinehealthchecks.yaml | 0 .../cluster.x-k8s.io_machines.yaml | 0 .../cluster.x-k8s.io_machinesets.yaml | 0 config/cluster-api/kustomization.yaml | 13 + .../cluster-api/manager-clusterrole.yaml | 0 .../manager-clusterrolebinding.yaml | 0 .../cluster-api/manager-deployment.yaml | 3 +- .../cluster-api/manager-serviceaccount.yaml | 0 ...hypershift.openshift.io_guestclusters.yaml | 0 ...hift.openshift.io_hostedcontrolplanes.yaml | 0 .../hypershift.openshift.io_nodepools.yaml | 0 ...rshift.openshift.io_openshiftclusters.yaml | 0 config/hypershift-operator/kustomization.yaml | 35 ++ .../operator-clusterrole.yaml | 94 +++++ .../operator-clusterrolebinding.yaml | 12 + .../operator-deployment.yaml | 41 +++ .../operator-namespace.yaml | 0 .../operator-serviceaccount.yaml | 5 + config/install/development/kustomization.yaml | 23 ++ config/install/production/kustomization.yaml | 14 + config/kustomization.yaml | 9 - config/rbac/role.yaml | 28 -- hack/generate-release-images.rb | 28 -- .../controllers/controlplane.go | 6 +- .../hosted_controlplane_controller.go | 38 +- .../openshiftcluster_controller.go | 3 +- hypershift-operator/main.go | 66 +++- manifests/hypershift.openshift.io_crds.yaml | 347 ------------------ 34 files changed, 412 insertions(+), 512 deletions(-) rename {manifests => config/cluster-api}/cluster.x-k8s.io_clusters.yaml (100%) rename {manifests => config/cluster-api}/cluster.x-k8s.io_machinedeployments.yaml (100%) rename {manifests => config/cluster-api}/cluster.x-k8s.io_machinehealthchecks.yaml (100%) rename {manifests => config/cluster-api}/cluster.x-k8s.io_machines.yaml (100%) rename {manifests => config/cluster-api}/cluster.x-k8s.io_machinesets.yaml (100%) create mode 100644 config/cluster-api/kustomization.yaml rename manifests/cluster-api-clusterrole.yaml => config/cluster-api/manager-clusterrole.yaml (100%) rename manifests/cluster-api-clusterrolebinding.yaml => config/cluster-api/manager-clusterrolebinding.yaml (100%) rename manifests/cluster-api-deployment.yaml => config/cluster-api/manager-deployment.yaml (86%) rename manifests/cluster-api-serviceaccount.yaml => config/cluster-api/manager-serviceaccount.yaml (100%) rename config/{bases => hypershift-operator}/hypershift.openshift.io_guestclusters.yaml (100%) rename config/{bases => hypershift-operator}/hypershift.openshift.io_hostedcontrolplanes.yaml (100%) rename config/{bases => hypershift-operator}/hypershift.openshift.io_nodepools.yaml (100%) rename config/{bases => hypershift-operator}/hypershift.openshift.io_openshiftclusters.yaml (100%) create mode 100644 config/hypershift-operator/kustomization.yaml create mode 100644 config/hypershift-operator/operator-clusterrole.yaml create mode 100644 config/hypershift-operator/operator-clusterrolebinding.yaml create mode 100644 config/hypershift-operator/operator-deployment.yaml rename manifests/00_namespace.yaml => config/hypershift-operator/operator-namespace.yaml (100%) create mode 100644 config/hypershift-operator/operator-serviceaccount.yaml create mode 100644 config/install/development/kustomization.yaml create mode 100644 config/install/production/kustomization.yaml delete mode 100644 config/kustomization.yaml delete mode 100644 config/rbac/role.yaml delete mode 100755 hack/generate-release-images.rb delete mode 100644 manifests/hypershift.openshift.io_crds.yaml diff --git a/.gitignore b/.gitignore index 83366ad86e..7a672c03c3 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ bin .envrc .kube + +config/hypershift-operator/release-info.json diff --git a/HACKING.md b/HACKING.md index f9ade6198b..f7a05783d1 100644 --- a/HACKING.md +++ b/HACKING.md @@ -1,5 +1,29 @@ # Hacking +### Development workflow + +Often it's easiest to develop the operator locally connected to a remote +cluster. + +In this case, you might want to install with the development Kustomize +profile which uses 0 replicas for the operator deployment by default. This +makes it easy to iterate on the non-deployment manifests in conjunction +with the operator binary itself. + +Starting from clean management cluster, run the following to get started: + +```bash +$ make build + +$ make install PROFILE=development + +$ bin/hypershift-operator run --release-info config/bases/release-info.json +``` + +Or you might want to run your own image in the cluster to do integration +testing, in which case you may want to use the default (production) profile +and use `kubectl set image` (for example) to update the deployment. + ### Visualizing dependencies MacOS diff --git a/Makefile b/Makefile index 14ad30e43b..79e57b28f9 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,16 @@ -.PHONY: bin/hypershift-operator - # Image URL to use all building/pushing image targets -IMG ?= controller:latest +IMG ?= hypershift:latest # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true" CONTROLLER_GEN=GO111MODULE=on GOFLAGS=-mod=vendor go run sigs.k8s.io/controller-tools/cmd/controller-gen +GO_GCFLAGS ?= -gcflags=all='-N -l' GO=GO111MODULE=on GOFLAGS=-mod=vendor go -GO_BUILD_RECIPE=CGO_ENABLED=0 $(GO) build -o $(BIN) $(GO_GCFLAGS) $(MAIN_PACKAGE) +GO_BUILD_RECIPE=CGO_ENABLED=0 $(GO) build $(GO_GCFLAGS) + +# Kustomize overlay to use +PROFILE ?= production # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -17,65 +19,61 @@ else GOBIN=$(shell go env GOBIN) endif -all: hypershift-operator control-plane-operator +all: build manifests -# Run tests -test: generate fmt vet manifests - go test ./... -coverprofile cover.out +build: hypershift-operator control-plane-operator -# Build hypershift-operator binary -hypershift-operator: generate bindata manifests fmt vet bin/hypershift-operator +verify: build fmt vet -bin/hypershift-operator: - go build -o bin/hypershift-operator ./hypershift-operator +# Generate code +generate: + hack/update-generated-bindata.sh + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +# Build hypershift-operator binary +hypershift-operator: generate + $(GO_BUILD_RECIPE) -o bin/hypershift-operator ./hypershift-operator # Build control-plane-operator binary -control-plane-operator: generate fmt vet - go build -o bin/control-plane-operator ./control-plane-operator +control-plane-operator: generate + $(GO_BUILD_RECIPE) -o bin/control-plane-operator ./control-plane-operator + +# Run tests +test: build + $(GO) test ./... -coverprofile cover.out -# Run against the configured Kubernetes cluster in ~/.kube/config -run: generate fmt vet manifests - go run ./hypershift-operator/main.go +# Generate Kube manifests (e.g. CRDs) +manifests: + $(CONTROLLER_GEN) $(CRD_OPTIONS) webhook paths="./..." output:crd:artifacts:config=config/hypershift-operator -# Install CRDs into a cluster -install: manifests - kustomize build config/crd | kubectl apply -f - +# Installs hypershift into a cluster +install: manifests release-info-data + kustomize build config/install/$(PROFILE) | oc apply -f - -# Uninstall CRDs from a cluster +# Uninstalls hypershit from a cluster uninstall: manifests - kustomize build config/crd | kubectl delete -f - - -# Deploy controller in the configured Kubernetes cluster in ~/.kube/config -deploy: manifests - cd config/manager && kustomize edit set image controller=${IMG} - kustomize build config/default | kubectl apply -f - + kustomize build config/install/$(PROFILE) | oc delete -f - -# Generate manifests e.g. CRD, RBAC etc. -.PHONY: manifests -manifests: - $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=hypershift-operator-role webhook paths="./..." output:crd:artifacts:config=config/bases - cd config && kustomize build > ../manifests/hypershift.openshift.io_crds.yaml +# Builds the config with Kustomize for manual usage +.PHONY: config +config: release-info-data + kustomize build config/install/$(PROFILE) # Run go fmt against code fmt: - go fmt ./... + $(GO) fmt ./... # Run go vet against code vet: - go vet ./... - -# Generate code -generate: - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: bindata -bindata: - hack/update-generated-bindata.sh + $(GO) vet ./... # Build the docker image -docker-build: test +docker-build: docker build . -t ${IMG} # Push the docker image docker-push: docker push ${IMG} + +release-info-data: + oc adm release info --output json > config/hypershift-operator/release-info.json diff --git a/README.md b/README.md index 7962ebef20..5b4a6ae53a 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,40 @@ -# Hypershift POC +## HyperShift -All the following assumes `KUBECONFIG` points to the management cluster. +Guest clustering for [OpenShift](https://openshift.io). -Build binaries: +### Prerequisites -``` -$ make -``` +* Admin access to an OpenShift cluster. +* The OpenShift `oc` CLI tool. +* [Kustomize](https://kustomize.io) -Install the operator's supporting resources into the management cluster: -``` -$ oc apply --filename manifests/ -``` +### Installation -Define release image info to be referenced by clusters: +Install HyperShift into the management cluster: -``` -hack/generate-release-images.rb | oc apply --filename - +```bash +$ make install ``` -Run the operator: -``` -$ bin/hypershift-operator run --control-plane-operator-image quay.io/hypershift/hypershift:latest +Remove HyperShift from the management cluster: + +```bash +$ make uninstall ``` -Create a cluster, referencing a release image present in the `release-images` configmap -previously created: +### Create a cluster + +Create a new guest cluster by creating an `OpenShiftCluster` resource. For now, +the cluster will be based on the version of the management cluster itself. + +Here's an example: ```yaml apiVersion: hypershift.openshift.io/v1alpha1 kind: OpenShiftCluster metadata: - namespace: hypershift name: guest-hello spec: - releaseImage: quay.io/openshift-release-dev/ocp-release@sha256:d78292e9730dd387ff6198197c8b0598da340be7678e8e1e4810b557a926c2b9 baseDomain: guest-hello.devcluster.openshift.com pullSecret: '{"auths": { ... }}' serviceCIDR: 172.31.0.0/16 @@ -43,12 +43,14 @@ spec: initialComputeReplicas: 1 ``` -Get the cluster kubeconfig using: -``` +Get the guest cluster's kubeconfig using: + +```bash $ oc get secret --namespace guest-hello admin-kubeconfig --template={{.data.kubeconfig}} | base64 -D ``` You can create additional nodePools: + ```yaml apiVersion: hypershift.openshift.io/v1alpha1 kind: NodePool @@ -65,8 +67,9 @@ spec: aws: instanceType: m5.large ``` + And delete the cluster using: -``` +```bash $ oc delete --namespace hypershift openshiftclusters/guest-hello ``` diff --git a/manifests/cluster.x-k8s.io_clusters.yaml b/config/cluster-api/cluster.x-k8s.io_clusters.yaml similarity index 100% rename from manifests/cluster.x-k8s.io_clusters.yaml rename to config/cluster-api/cluster.x-k8s.io_clusters.yaml diff --git a/manifests/cluster.x-k8s.io_machinedeployments.yaml b/config/cluster-api/cluster.x-k8s.io_machinedeployments.yaml similarity index 100% rename from manifests/cluster.x-k8s.io_machinedeployments.yaml rename to config/cluster-api/cluster.x-k8s.io_machinedeployments.yaml diff --git a/manifests/cluster.x-k8s.io_machinehealthchecks.yaml b/config/cluster-api/cluster.x-k8s.io_machinehealthchecks.yaml similarity index 100% rename from manifests/cluster.x-k8s.io_machinehealthchecks.yaml rename to config/cluster-api/cluster.x-k8s.io_machinehealthchecks.yaml diff --git a/manifests/cluster.x-k8s.io_machines.yaml b/config/cluster-api/cluster.x-k8s.io_machines.yaml similarity index 100% rename from manifests/cluster.x-k8s.io_machines.yaml rename to config/cluster-api/cluster.x-k8s.io_machines.yaml diff --git a/manifests/cluster.x-k8s.io_machinesets.yaml b/config/cluster-api/cluster.x-k8s.io_machinesets.yaml similarity index 100% rename from manifests/cluster.x-k8s.io_machinesets.yaml rename to config/cluster-api/cluster.x-k8s.io_machinesets.yaml diff --git a/config/cluster-api/kustomization.yaml b/config/cluster-api/kustomization.yaml new file mode 100644 index 0000000000..dc09be8389 --- /dev/null +++ b/config/cluster-api/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- cluster.x-k8s.io_clusters.yaml +- cluster.x-k8s.io_machinesets.yaml +- cluster.x-k8s.io_machines.yaml +- cluster.x-k8s.io_machinedeployments.yaml +- cluster.x-k8s.io_machinehealthchecks.yaml +- manager-serviceaccount.yaml +- manager-clusterrole.yaml +- manager-clusterrolebinding.yaml +- manager-deployment.yaml diff --git a/manifests/cluster-api-clusterrole.yaml b/config/cluster-api/manager-clusterrole.yaml similarity index 100% rename from manifests/cluster-api-clusterrole.yaml rename to config/cluster-api/manager-clusterrole.yaml diff --git a/manifests/cluster-api-clusterrolebinding.yaml b/config/cluster-api/manager-clusterrolebinding.yaml similarity index 100% rename from manifests/cluster-api-clusterrolebinding.yaml rename to config/cluster-api/manager-clusterrolebinding.yaml diff --git a/manifests/cluster-api-deployment.yaml b/config/cluster-api/manager-deployment.yaml similarity index 86% rename from manifests/cluster-api-deployment.yaml rename to config/cluster-api/manager-deployment.yaml index e3fd4e4fd9..266dc390a9 100644 --- a/manifests/cluster-api-deployment.yaml +++ b/config/cluster-api/manager-deployment.yaml @@ -16,7 +16,8 @@ spec: serviceAccountName: cluster-api containers: - name: manager - image: quay.io/hypershift/cluster-api:hypershift + image: cluster-api:latest + imagePullPolicy: Always command: - /manager args: diff --git a/manifests/cluster-api-serviceaccount.yaml b/config/cluster-api/manager-serviceaccount.yaml similarity index 100% rename from manifests/cluster-api-serviceaccount.yaml rename to config/cluster-api/manager-serviceaccount.yaml diff --git a/config/bases/hypershift.openshift.io_guestclusters.yaml b/config/hypershift-operator/hypershift.openshift.io_guestclusters.yaml similarity index 100% rename from config/bases/hypershift.openshift.io_guestclusters.yaml rename to config/hypershift-operator/hypershift.openshift.io_guestclusters.yaml diff --git a/config/bases/hypershift.openshift.io_hostedcontrolplanes.yaml b/config/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml similarity index 100% rename from config/bases/hypershift.openshift.io_hostedcontrolplanes.yaml rename to config/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml diff --git a/config/bases/hypershift.openshift.io_nodepools.yaml b/config/hypershift-operator/hypershift.openshift.io_nodepools.yaml similarity index 100% rename from config/bases/hypershift.openshift.io_nodepools.yaml rename to config/hypershift-operator/hypershift.openshift.io_nodepools.yaml diff --git a/config/bases/hypershift.openshift.io_openshiftclusters.yaml b/config/hypershift-operator/hypershift.openshift.io_openshiftclusters.yaml similarity index 100% rename from config/bases/hypershift.openshift.io_openshiftclusters.yaml rename to config/hypershift-operator/hypershift.openshift.io_openshiftclusters.yaml diff --git a/config/hypershift-operator/kustomization.yaml b/config/hypershift-operator/kustomization.yaml new file mode 100644 index 0000000000..96e779159f --- /dev/null +++ b/config/hypershift-operator/kustomization.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - hypershift.openshift.io_openshiftclusters.yaml + - hypershift.openshift.io_hostedcontrolplanes.yaml + - hypershift.openshift.io_guestclusters.yaml + - hypershift.openshift.io_nodepools.yaml + - operator-namespace.yaml + - operator-serviceaccount.yaml + - operator-clusterrole.yaml + - operator-clusterrolebinding.yaml + - operator-deployment.yaml + +configMapGenerator: +- name: release-info + namespace: hypershift + files: + - release-info.json + +patchesStrategicMerge: +- |- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: guestclusters.hypershift.openshift.io + labels: + cluster.x-k8s.io/v1alpha4: v1alpha1 +- |- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: hostedcontrolplanes.hypershift.openshift.io + labels: + cluster.x-k8s.io/v1alpha4: v1alpha1 diff --git a/config/hypershift-operator/operator-clusterrole.yaml b/config/hypershift-operator/operator-clusterrole.yaml new file mode 100644 index 0000000000..2c5632fd31 --- /dev/null +++ b/config/hypershift-operator/operator-clusterrole.yaml @@ -0,0 +1,94 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: hypershift-operator +rules: +- apiGroups: + - hypershift.openshift.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - config.openshift.io + resources: + - '*' + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - bootstrap.cluster.x-k8s.io + - controlplane.cluster.x-k8s.io + - infrastructure.cluster.x-k8s.io + - machines.cluster.x-k8s.io + - exp.infrastructure.cluster.x-k8s.io + - addons.cluster.x-k8s.io + - exp.cluster.x-k8s.io + - cluster.x-k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - operator.openshift.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - route.openshift.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - '*' +- apiGroups: + - rbac.authorization.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - "" + resources: + - events + - configmaps + - pods + - secrets + - nodes + - namespaces + - serviceaccounts + - services + verbs: + - '*' +- apiGroups: + - apps + resources: + - deployments + verbs: + - '*' +- apiGroups: + - etcd.database.coreos.com + resources: + - '*' + verbs: + - '*' +- apiGroups: + - machine.openshift.io + resources: + - '*' + verbs: + - '*' diff --git a/config/hypershift-operator/operator-clusterrolebinding.yaml b/config/hypershift-operator/operator-clusterrolebinding.yaml new file mode 100644 index 0000000000..d18196572d --- /dev/null +++ b/config/hypershift-operator/operator-clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: hypershift-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hypershift-operator +subjects: +- kind: ServiceAccount + name: operator + namespace: hypershift diff --git a/config/hypershift-operator/operator-deployment.yaml b/config/hypershift-operator/operator-deployment.yaml new file mode 100644 index 0000000000..9a06962f08 --- /dev/null +++ b/config/hypershift-operator/operator-deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: hypershift + name: operator +spec: + replicas: 1 + selector: + matchLabels: + name: operator + template: + metadata: + labels: + name: operator + spec: + volumes: + - name: release-info + configMap: + name: release-info + serviceAccountName: operator + containers: + - name: operator + image: hypershift:latest + imagePullPolicy: Always + volumeMounts: + - name: release-info + mountPath: /etc/release-info + command: + - /usr/bin/hypershift-operator + args: + - run + - --release-info=/etc/release-info/release-info.json + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/manifests/00_namespace.yaml b/config/hypershift-operator/operator-namespace.yaml similarity index 100% rename from manifests/00_namespace.yaml rename to config/hypershift-operator/operator-namespace.yaml diff --git a/config/hypershift-operator/operator-serviceaccount.yaml b/config/hypershift-operator/operator-serviceaccount.yaml new file mode 100644 index 0000000000..6527f95d49 --- /dev/null +++ b/config/hypershift-operator/operator-serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: hypershift + name: operator diff --git a/config/install/development/kustomization.yaml b/config/install/development/kustomization.yaml new file mode 100644 index 0000000000..46b09b0bc9 --- /dev/null +++ b/config/install/development/kustomization.yaml @@ -0,0 +1,23 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../cluster-api +- ../../hypershift-operator + +images: +- name: cluster-api + newName: quay.io/hypershift/cluster-api + newTag: hypershift +- name: hypershift + newName: quay.io/hypershift/hypershift + newTag: latest + +patches: +- patch: |- + - op: replace + path: /spec/replicas + value: 0 + target: + kind: Deployment + name: operator diff --git a/config/install/production/kustomization.yaml b/config/install/production/kustomization.yaml new file mode 100644 index 0000000000..9bc5105349 --- /dev/null +++ b/config/install/production/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../cluster-api +- ../../hypershift-operator + +images: +- name: cluster-api + newName: quay.io/hypershift/cluster-api + newTag: hypershift +- name: hypershift + newName: quay.io/hypershift/hypershift + newTag: latest diff --git a/config/kustomization.yaml b/config/kustomization.yaml deleted file mode 100644 index 97e1628473..0000000000 --- a/config/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - bases/hypershift.openshift.io_guestclusters.yaml - - bases/hypershift.openshift.io_hostedcontrolplanes.yaml - - bases/hypershift.openshift.io_openshiftclusters.yaml - - bases/hypershift.openshift.io_nodepools.yaml -commonLabels: - cluster.x-k8s.io/v1alpha4: v1alpha1 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml deleted file mode 100644 index 9076540a23..0000000000 --- a/config/rbac/role.yaml +++ /dev/null @@ -1,28 +0,0 @@ - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: hypershift-operator-role -rules: -- apiGroups: - - hypershift.openshift.io - resources: - - openshiftclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - hypershift.openshift.io - resources: - - openshiftclusters/status - verbs: - - get - - patch - - update diff --git a/hack/generate-release-images.rb b/hack/generate-release-images.rb deleted file mode 100755 index cc5b43c743..0000000000 --- a/hack/generate-release-images.rb +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env ruby -require 'yaml' -require 'json' -require 'net/http' -require 'base64' - -major_versions = ["4.6"] - -images = {} -major_versions.each do |major| - data = Net::HTTP.get("mirror.openshift.com", "/pub/openshift-v4/clients/ocp/candidate-#{major}/release.txt") - image = data.match(/Pull From.*$/)[0].split[2] - images[image] = JSON.load(`oc adm release info --output json #{image}`) -end - -configmap = { - "apiVersion" => "v1", - "kind" => "ConfigMap", - "metadata" => { - "namespace" => "hypershift", - "name" => "release-images", - }, - "data" => { - "images.json" => JSON.dump(images) - } -} - -puts YAML.dump(configmap) diff --git a/hypershift-operator/controllers/controlplane.go b/hypershift-operator/controllers/controlplane.go index efb29130c4..3b1f8577e4 100644 --- a/hypershift-operator/controllers/controlplane.go +++ b/hypershift-operator/controllers/controlplane.go @@ -95,6 +95,10 @@ func (r *HostedControlPlaneReconciler) ensureControlPlane(ctx context.Context, h if err != nil { return fmt.Errorf("cannot parse release version (%s): %v", releaseVersion, err) } + controlPlaneOperatorImage, err := r.LookupControlPlaneOperatorImage(r.Client) + if err != nil { + return fmt.Errorf("failed to lookup control plane operator image: %w", err) + } params := hypershiftcp.NewClusterParams() params.Namespace = name @@ -120,7 +124,7 @@ func (r *HostedControlPlaneReconciler) ensureControlPlane(ctx context.Context, h params.ImageRegistryHTTPSecret = generateImageRegistrySecret() params.Replicas = "1" params.SSHKey = hcp.Spec.SSHKey - params.ControlPlaneOperatorImage = r.ControlPlaneOperatorImage + params.ControlPlaneOperatorImage = controlPlaneOperatorImage params.HypershiftOperatorControllers = []string{"route-sync", "auto-approver", "kubeadmin-password", "node"} // Generate PKI data just once and store it in a secret. PKI generation isn't diff --git a/hypershift-operator/controllers/hosted_controlplane_controller.go b/hypershift-operator/controllers/hosted_controlplane_controller.go index af0722f11a..48dab79560 100644 --- a/hypershift-operator/controllers/hosted_controlplane_controller.go +++ b/hypershift-operator/controllers/hosted_controlplane_controller.go @@ -2,7 +2,6 @@ package controllers import ( "context" - "encoding/json" "fmt" "time" @@ -14,22 +13,24 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" - hyperv1 "openshift.io/hypershift/api/v1alpha1" - "openshift.io/hypershift/hypershift-operator/releaseinfo" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/patch" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + hyperv1 "openshift.io/hypershift/api/v1alpha1" + "openshift.io/hypershift/hypershift-operator/releaseinfo" ) type HostedControlPlaneReconciler struct { client.Client - Log logr.Logger - Infra *configv1.Infrastructure - recorder record.EventRecorder - ControlPlaneOperatorImage string + Log logr.Logger + Infra *configv1.Infrastructure + recorder record.EventRecorder + LookupControlPlaneOperatorImage func(kubeClient client.Client) (string, error) + LookupReleaseInfo func(image string) (*releaseinfo.ReleaseImageInfo, error) } func (r *HostedControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -124,25 +125,10 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R return result, nil } - // The images.json key contains a hash of image pullspec to release info - // obtained from `oc adm release info $image -o json`. - releaseImages := &corev1.ConfigMap{} - if err := r.Client.Get(ctx, client.ObjectKey{Namespace: "hypershift", Name: "release-images"}, releaseImages); err != nil { - result.RequeueAfter = 5 * time.Second - return result, fmt.Errorf("no release images found") - } - imagesJSON, hasImagesJSON := releaseImages.Data["images.json"] - if !hasImagesJSON { - return result, fmt.Errorf("no images.json found in release images configmap") - } - images := map[string]releaseinfo.ReleaseImageInfo{} - if err := json.Unmarshal([]byte(imagesJSON), &images); err != nil { - return result, fmt.Errorf("failed to read images.json: %w", err) - } - releaseInfo, hasReleaseInfo := images[hostedControlPlane.Spec.ReleaseImage] - if !hasReleaseInfo { + releaseInfo, err := r.LookupReleaseInfo(hostedControlPlane.Spec.ReleaseImage) + if err != nil { result.RequeueAfter = 5 * time.Second - return result, fmt.Errorf("no release info found for image %q", hostedControlPlane.Spec.ReleaseImage) + return result, fmt.Errorf("no release info found for image %q: %w", hostedControlPlane.Spec.ReleaseImage, err) } // First, set up infrastructure @@ -164,7 +150,7 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R } // Install the control plane into the infrastructure - err = r.ensureControlPlane(ctx, hostedControlPlane, infraStatus, &releaseInfo) + err = r.ensureControlPlane(ctx, hostedControlPlane, infraStatus, releaseInfo) if err != nil { r.Log.Error(err, "failed to ensure control plane") return result, fmt.Errorf("failed to ensure control plane: %w", err) diff --git a/hypershift-operator/controllers/openshiftcluster_controller.go b/hypershift-operator/controllers/openshiftcluster_controller.go index 02bf75bf63..b24f23d432 100644 --- a/hypershift-operator/controllers/openshiftcluster_controller.go +++ b/hypershift-operator/controllers/openshiftcluster_controller.go @@ -44,8 +44,7 @@ const ( // OpenShiftClusterReconciler reconciles a OpenShiftCluster object type OpenShiftClusterReconciler struct { client.Client - Log logr.Logger - ControlPlaneOperatorImage string + Log logr.Logger } // +kubebuilder:rbac:groups=hypershift.openshift.io,resources=openshiftclusters,verbs=get;list;watch;create;update;patch;delete diff --git a/hypershift-operator/main.go b/hypershift-operator/main.go index df3a6a78af..85cf44465f 100644 --- a/hypershift-operator/main.go +++ b/hypershift-operator/main.go @@ -17,7 +17,10 @@ limitations under the License. package main import ( + "context" + "encoding/json" "fmt" + "io/ioutil" "os" configv1 "github.com/openshift/api/config/v1" @@ -25,13 +28,17 @@ import ( routev1 "github.com/openshift/api/route/v1" securityv1 "github.com/openshift/api/security/v1" "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" hyperv1 "openshift.io/hypershift/api/v1alpha1" "openshift.io/hypershift/hypershift-operator/controllers" + "openshift.io/hypershift/hypershift-operator/releaseinfo" + capiv1 "sigs.k8s.io/cluster-api/api/v1alpha4" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log/zap" // +kubebuilder:scaffold:imports ) @@ -77,11 +84,14 @@ func NewStartCommand() *cobra.Command { var metricsAddr string var enableLeaderElection bool var controlPlaneOperatorImage string + var releaseInfoFile string + cmd.Flags().StringVar(&metricsAddr, "metrics-addr", "0", "The address the metric endpoint binds to.") cmd.Flags().BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") cmd.Flags().StringVar(&controlPlaneOperatorImage, "control-plane-operator-image", "", "A control plane operator image.") + cmd.Flags().StringVar(&releaseInfoFile, "release-info", "", "A static release info JSON file to use for all guest clusters") cmd.Run = func(cmd *cobra.Command, args []string) { ctrl.SetLogger(zap.New(zap.UseDevMode(true))) @@ -97,17 +107,65 @@ func NewStartCommand() *cobra.Command { os.Exit(1) } + // Add some flexibility to getting the control plane operator image. Use the + // flag if given, but if that's empty and we're running in a pod, use the + // hypershift operator's image for the control plane by default. + lookupControlPlaneOperatorImage := func(kubeClient client.Client) (string, error) { + if len(controlPlaneOperatorImage) > 0 { + return controlPlaneOperatorImage, nil + } + + podName := os.Getenv("MY_POD_NAME") + podNamespace := os.Getenv("MY_POD_NAMESPACE") + runningInPod := len(podName) > 0 && len(podNamespace) > 0 + if !runningInPod { + return "", nil + } + + var image string + pod := corev1.Pod{} + if err := kubeClient.Get(context.TODO(), client.ObjectKey{Namespace: podNamespace, Name: podName}, &pod); err != nil { + return "", fmt.Errorf("failed to get operator's pod %s/%s: %w", podNamespace, podName, err) + } + for _, container := range pod.Spec.Containers { + if container.Name == "operator" { + image = container.Image + break + } + } + return image, nil + } + + // For now read release info from a file to keep it simple and externalize + // the complexity of the ways we currently know to get the data (which involves + // authenticated registry interactions interactions via `oc adm release info`, etc.) + lookupReleaseInfo := func(image string) (*releaseinfo.ReleaseImageInfo, error) { + if len(releaseInfoFile) == 0 { + return nil, fmt.Errorf("release-info is currently required") + } + contents, err := ioutil.ReadFile(releaseInfoFile) + if err != nil { + return nil, fmt.Errorf("failed to read release info file %s: %w", releaseInfoFile, err) + } + var info releaseinfo.ReleaseImageInfo + err = json.Unmarshal(contents, &info) + if err != nil { + return nil, fmt.Errorf("invalid release info file %s: %w", releaseInfoFile, err) + } + return &info, nil + } + if err = (&controllers.OpenShiftClusterReconciler{ - Client: mgr.GetClient(), - ControlPlaneOperatorImage: controlPlaneOperatorImage, + Client: mgr.GetClient(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "OpenShiftCluster") os.Exit(1) } if err := (&controllers.HostedControlPlaneReconciler{ - Client: mgr.GetClient(), - ControlPlaneOperatorImage: controlPlaneOperatorImage, + Client: mgr.GetClient(), + LookupControlPlaneOperatorImage: lookupControlPlaneOperatorImage, + LookupReleaseInfo: lookupReleaseInfo, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "hostedControlPlane") os.Exit(1) diff --git a/manifests/hypershift.openshift.io_crds.yaml b/manifests/hypershift.openshift.io_crds.yaml deleted file mode 100644 index c600d37653..0000000000 --- a/manifests/hypershift.openshift.io_crds.yaml +++ /dev/null @@ -1,347 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - labels: - cluster.x-k8s.io/v1alpha4: v1alpha1 - name: guestclusters.hypershift.openshift.io -spec: - group: hypershift.openshift.io - names: - categories: - - cluster-api - kind: GuestCluster - listKind: GuestClusterList - plural: guestclusters - shortNames: - - gc - - gcs - singular: guestcluster - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: GuestCluster is the Schema for the GuestCluster API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: GuestClusterSpec defines the desired state of GuestCluster - properties: - computeReplicas: - type: integer - controlPlaneEndpoint: - description: ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. - properties: - host: - description: Host is the hostname on which the API server is serving. - type: string - port: - description: Port is the port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port - type: object - type: object - status: - description: GuestClusterStatus defines the observed state of GuestCluster - properties: - ready: - type: boolean - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - labels: - cluster.x-k8s.io/v1alpha4: v1alpha1 - name: hostedcontrolplanes.hypershift.openshift.io -spec: - group: hypershift.openshift.io - names: - categories: - - cluster-api - kind: HostedControlPlane - listKind: HostedControlPlaneList - plural: hostedcontrolplanes - shortNames: - - hcp - - hcps - singular: hostedcontrolplane - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: HostedControlPlane defines the desired state of HostedControlPlane - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HostedControlPlaneSpec defines the desired state of HostedControlPlane - properties: - baseDomain: - type: string - podCIDR: - type: string - pullSecret: - type: string - releaseImage: - type: string - serviceCIDR: - type: string - sshKey: - type: string - required: - - baseDomain - - podCIDR - - pullSecret - - releaseImage - - serviceCIDR - - sshKey - type: object - status: - description: HostedControlPlaneStatus defines the observed state of HostedControlPlane - properties: - controlPlaneEndpoint: - properties: - host: - description: Host is the hostname on which the API server is serving. - type: string - port: - description: Port is the port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port - type: object - ready: - default: false - description: Ready denotes that the HostedControlPlane API Server is ready to receive requests - type: boolean - required: - - controlPlaneEndpoint - - ready - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - labels: - cluster.x-k8s.io/v1alpha4: v1alpha1 - name: nodepools.hypershift.openshift.io -spec: - group: hypershift.openshift.io - names: - kind: NodePool - listKind: NodePoolList - plural: nodepools - shortNames: - - np - - nps - singular: nodepool - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Available Nodes - jsonPath: .status.NodeCount - name: NodeCount - type: integer - name: v1alpha1 - schema: - openAPIV3Schema: - description: NodePool defines the desired state of NodePool - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodePoolSpec defines the desired state of NodePool - properties: - autoScaling: - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - clusterName: - description: ClusterName is the name of the Cluster this object belongs to. - type: string - nodeCount: - type: integer - platform: - description: NodePoolPlatform is the platform-specific configuration for a node pool. Only one of the platforms should be set. - properties: - aws: - description: AWS is the configuration used when installing on AWS. - properties: - instanceType: - description: InstanceType defines the ec2 instance type. eg. m4-large - type: string - required: - - instanceType - type: object - type: object - required: - - clusterName - - nodeCount - - platform - type: object - status: - description: NodePoolStatus defines the observed state of NodePool - properties: - nodeCount: - description: NodeCount is the most recently observed number of replicas. - type: integer - required: - - nodeCount - type: object - type: object - served: true - storage: true - subresources: - scale: - specReplicasPath: .spec.nodeCount - statusReplicasPath: .status.nodeCount - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - labels: - cluster.x-k8s.io/v1alpha4: v1alpha1 - name: openshiftclusters.hypershift.openshift.io -spec: - group: hypershift.openshift.io - names: - kind: OpenShiftCluster - listKind: OpenShiftClusterList - plural: openshiftclusters - shortNames: - - oc - - ocs - singular: openshiftcluster - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: OpenShiftCluster is the Schema for the openshiftclusters API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: OpenShiftClusterSpec defines the desired state of OpenShiftCluster - properties: - baseDomain: - type: string - initialComputeReplicas: - type: integer - podCIDR: - type: string - pullSecret: - type: string - releaseImage: - type: string - serviceCIDR: - type: string - sshKey: - type: string - required: - - baseDomain - - initialComputeReplicas - - podCIDR - - pullSecret - - releaseImage - - serviceCIDR - - sshKey - type: object - status: - description: OpenShiftClusterStatus defines the observed state of OpenShiftCluster - properties: - ready: - type: boolean - required: - - ready - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: []