Improve installation

This commit introduces some major refactoring to make installation easier. * Get rid of the `manifests` directory and handle all configuration through Kustomize. * Add a deployment (and all supporting resources) for the operator itself. * Introduce reasonable defaults for discovering the control plane image to use based on the operator pod (with the ability to override with a flag). * Abstract away the release image metadata handling and integrate it with Kustomize. For now all guest clusters will use same release version as the management cluster, but the commit also also cleans up some of the plugin points to help us improve in a followup. * Clean up and improve the Makefile to be the entrypoint for all of these tasks. * Simplify the installation procedures in the README.
openshift · Dec 2, 2020 · d91566c · d91566c
1 parent a6c1914
commit d91566c
Show file tree

Hide file tree

Showing 34 changed files with 412 additions and 512 deletions.
diff --git a/.gitignore b/.gitignore
@@ -26,3 +26,5 @@ bin
 .envrc
 
 .kube
+
+config/hypershift-operator/release-info.json
diff --git a/HACKING.md b/HACKING.md
@@ -1,5 +1,29 @@
 # Hacking
 
+### Development workflow
+
+Often it's easiest to develop the operator locally connected to a remote
+cluster.
+
+In this case, you might want to install with the development Kustomize
+profile which uses 0 replicas for the operator deployment by default. This
+makes it easy to iterate on the non-deployment manifests in conjunction
+with the operator binary itself.
+
+Starting from clean management cluster, run the following to get started: 
+
+```bash
+$ make build
+
+$ make install PROFILE=development
+
+$ bin/hypershift-operator run --release-info config/bases/release-info.json
+```
+
+Or you might want to run your own image in the cluster to do integration
+testing, in which case you may want to use the default (production) profile
+and use `kubectl set image` (for example) to update the deployment.
+
 ### Visualizing dependencies
 
 MacOS

diff --git a/Makefile b/Makefile
@@ -1,14 +1,16 @@
-.PHONY: bin/hypershift-operator
-
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
+IMG ?= hypershift:latest
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
 
 CONTROLLER_GEN=GO111MODULE=on GOFLAGS=-mod=vendor go run sigs.k8s.io/controller-tools/cmd/controller-gen
 
+GO_GCFLAGS ?= -gcflags=all='-N -l'
 GO=GO111MODULE=on GOFLAGS=-mod=vendor go
-GO_BUILD_RECIPE=CGO_ENABLED=0 $(GO) build -o $(BIN) $(GO_GCFLAGS) $(MAIN_PACKAGE)
+GO_BUILD_RECIPE=CGO_ENABLED=0 $(GO) build $(GO_GCFLAGS)
+
+# Kustomize overlay to use
+PROFILE ?= production
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -17,65 +19,61 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
-all: hypershift-operator control-plane-operator
+all: build manifests
 
-# Run tests
-test: generate fmt vet manifests
-	go test ./... -coverprofile cover.out
+build: hypershift-operator control-plane-operator
 
-# Build hypershift-operator binary
-hypershift-operator: generate bindata manifests fmt vet bin/hypershift-operator
+verify: build fmt vet
 
-bin/hypershift-operator:
-	go build -o bin/hypershift-operator ./hypershift-operator
+# Generate code
+generate:
+	hack/update-generated-bindata.sh
+	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+
+# Build hypershift-operator binary
+hypershift-operator: generate
+	$(GO_BUILD_RECIPE) -o bin/hypershift-operator ./hypershift-operator
 
 # Build control-plane-operator binary
-control-plane-operator: generate fmt vet
-	go build -o bin/control-plane-operator ./control-plane-operator
+control-plane-operator: generate
+	$(GO_BUILD_RECIPE) -o bin/control-plane-operator ./control-plane-operator
+
+# Run tests
+test: build
+	$(GO) test ./... -coverprofile cover.out
 
-# Run against the configured Kubernetes cluster in ~/.kube/config
-run: generate fmt vet manifests
-	go run ./hypershift-operator/main.go
+# Generate Kube manifests (e.g. CRDs)
+manifests:
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) webhook paths="./..." output:crd:artifacts:config=config/hypershift-operator
 
-# Install CRDs into a cluster
-install: manifests
-	kustomize build config/crd | kubectl apply -f -
+# Installs hypershift into a cluster
+install: manifests release-info-data
+	kustomize build config/install/$(PROFILE) | oc apply -f -
 
-# Uninstall CRDs from a cluster
+# Uninstalls hypershit from a cluster
 uninstall: manifests
-	kustomize build config/crd | kubectl delete -f -
-
-# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
-deploy: manifests
-	cd config/manager && kustomize edit set image controller=${IMG}
-	kustomize build config/default | kubectl apply -f -
+	kustomize build config/install/$(PROFILE) | oc delete -f -
 
-# Generate manifests e.g. CRD, RBAC etc.
-.PHONY: manifests
-manifests:
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=hypershift-operator-role webhook paths="./..." output:crd:artifacts:config=config/bases
-	cd config && kustomize build > ../manifests/hypershift.openshift.io_crds.yaml
+# Builds the config with Kustomize for manual usage
+.PHONY: config
+config: release-info-data
+	kustomize build config/install/$(PROFILE)
 
 # Run go fmt against code
 fmt:
-	go fmt ./...
+	$(GO) fmt ./...
 
 # Run go vet against code
 vet:
-	go vet ./...
-
-# Generate code
-generate:
-	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
-
-.PHONY: bindata
-bindata:
-	hack/update-generated-bindata.sh
+	$(GO) vet ./...
 
 # Build the docker image
-docker-build: test
+docker-build:
 	docker build . -t ${IMG}
 
 # Push the docker image
 docker-push:
 	docker push ${IMG}
+
+release-info-data:
+	oc adm release info --output json > config/hypershift-operator/release-info.json
diff --git a/README.md b/README.md
@@ -1,40 +1,40 @@
-# Hypershift POC
+## HyperShift
 
-All the following assumes `KUBECONFIG` points to the management cluster.
+Guest clustering for [OpenShift](https://openshift.io).
 
-Build binaries: 
+### Prerequisites
 
-```
-$ make
-```
+* Admin access to an OpenShift cluster.
+* The OpenShift `oc` CLI tool.
+* [Kustomize](https://kustomize.io)
 
-Install the operator's supporting resources into the management cluster:
-```
-$ oc apply --filename manifests/
-```
+### Installation
 
-Define release image info to be referenced by clusters:
+Install HyperShift into the management cluster:
 
-```
-hack/generate-release-images.rb | oc apply --filename -
+```bash
+$ make install
 ```
 
-Run the operator:
-```
-$ bin/hypershift-operator run --control-plane-operator-image quay.io/hypershift/hypershift:latest
+Remove HyperShift from the management cluster:
+
+```bash
+$ make uninstall
 ```
 
-Create a cluster, referencing a release image present in the `release-images` configmap
-previously created:
+### Create a cluster
+
+Create a new guest cluster by creating an `OpenShiftCluster` resource. For now,
+the cluster will be based on the version of the management cluster itself.
+
+Here's an example:
 
 ```yaml
 apiVersion: hypershift.openshift.io/v1alpha1
 kind: OpenShiftCluster
 metadata:
-  namespace: hypershift
   name: guest-hello
 spec:
-  releaseImage: quay.io/openshift-release-dev/ocp-release@sha256:d78292e9730dd387ff6198197c8b0598da340be7678e8e1e4810b557a926c2b9
   baseDomain: guest-hello.devcluster.openshift.com
   pullSecret: '{"auths": { ... }}'
   serviceCIDR: 172.31.0.0/16
@@ -43,12 +43,14 @@ spec:
   initialComputeReplicas: 1
 ```
 
-Get the cluster kubeconfig using:
-```
+Get the guest cluster's kubeconfig using:
+
+```bash
 $ oc get secret --namespace guest-hello admin-kubeconfig --template={{.data.kubeconfig}} | base64 -D
 ```
 
 You can create additional nodePools:
+
 ```yaml
 apiVersion: hypershift.openshift.io/v1alpha1
 kind: NodePool
@@ -65,8 +67,9 @@ spec:
     aws:
       instanceType: m5.large
 ```
+
 And delete the cluster using:
 
-```
+```bash
 $ oc delete --namespace hypershift openshiftclusters/guest-hello
 ```
diff --git a/manifests/cluster.x-k8s.io_clusters.yaml → ...luster-api/cluster.x-k8s.io_clusters.yaml b/manifests/cluster.x-k8s.io_clusters.yaml → ...luster-api/cluster.x-k8s.io_clusters.yaml
diff --git a/.../cluster.x-k8s.io_machinedeployments.yaml → .../cluster.x-k8s.io_machinedeployments.yaml b/.../cluster.x-k8s.io_machinedeployments.yaml → .../cluster.x-k8s.io_machinedeployments.yaml
diff --git a/...cluster.x-k8s.io_machinehealthchecks.yaml → ...cluster.x-k8s.io_machinehealthchecks.yaml b/...cluster.x-k8s.io_machinehealthchecks.yaml → ...cluster.x-k8s.io_machinehealthchecks.yaml
diff --git a/manifests/cluster.x-k8s.io_machines.yaml → ...luster-api/cluster.x-k8s.io_machines.yaml b/manifests/cluster.x-k8s.io_machines.yaml → ...luster-api/cluster.x-k8s.io_machines.yaml
diff --git a/manifests/cluster.x-k8s.io_machinesets.yaml → ...ter-api/cluster.x-k8s.io_machinesets.yaml b/manifests/cluster.x-k8s.io_machinesets.yaml → ...ter-api/cluster.x-k8s.io_machinesets.yaml
diff --git a/config/cluster-api/kustomization.yaml b/config/cluster-api/kustomization.yaml
@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- cluster.x-k8s.io_clusters.yaml
+- cluster.x-k8s.io_machinesets.yaml
+- cluster.x-k8s.io_machines.yaml
+- cluster.x-k8s.io_machinedeployments.yaml
+- cluster.x-k8s.io_machinehealthchecks.yaml
+- manager-serviceaccount.yaml
+- manager-clusterrole.yaml
+- manager-clusterrolebinding.yaml
+- manager-deployment.yaml
diff --git a/manifests/cluster-api-clusterrole.yaml → config/cluster-api/manager-clusterrole.yaml b/manifests/cluster-api-clusterrole.yaml → config/cluster-api/manager-clusterrole.yaml
diff --git a/...fests/cluster-api-clusterrolebinding.yaml → ...uster-api/manager-clusterrolebinding.yaml b/...fests/cluster-api-clusterrolebinding.yaml → ...uster-api/manager-clusterrolebinding.yaml
diff --git a/manifests/cluster-api-deployment.yaml → config/cluster-api/manager-deployment.yaml b/manifests/cluster-api-deployment.yaml → config/cluster-api/manager-deployment.yaml
@@ -16,7 +16,8 @@ spec:
       serviceAccountName: cluster-api
       containers:
       - name: manager
-        image: quay.io/hypershift/cluster-api:hypershift
+        image: cluster-api:latest
+        imagePullPolicy: Always
         command:
         - /manager
         args:

diff --git a/manifests/cluster-api-serviceaccount.yaml → ...g/cluster-api/manager-serviceaccount.yaml b/manifests/cluster-api-serviceaccount.yaml → ...g/cluster-api/manager-serviceaccount.yaml
diff --git a/...ypershift.openshift.io_guestclusters.yaml → ...ypershift.openshift.io_guestclusters.yaml b/...ypershift.openshift.io_guestclusters.yaml → ...ypershift.openshift.io_guestclusters.yaml
diff --git a/...ift.openshift.io_hostedcontrolplanes.yaml → ...ift.openshift.io_hostedcontrolplanes.yaml b/...ift.openshift.io_hostedcontrolplanes.yaml → ...ift.openshift.io_hostedcontrolplanes.yaml
diff --git a/...es/hypershift.openshift.io_nodepools.yaml → ...or/hypershift.openshift.io_nodepools.yaml b/...es/hypershift.openshift.io_nodepools.yaml → ...or/hypershift.openshift.io_nodepools.yaml
diff --git a/...shift.openshift.io_openshiftclusters.yaml → ...shift.openshift.io_openshiftclusters.yaml b/...shift.openshift.io_openshiftclusters.yaml → ...shift.openshift.io_openshiftclusters.yaml
diff --git a/config/hypershift-operator/kustomization.yaml b/config/hypershift-operator/kustomization.yaml
@@ -0,0 +1,35 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - hypershift.openshift.io_openshiftclusters.yaml
+  - hypershift.openshift.io_hostedcontrolplanes.yaml
+  - hypershift.openshift.io_guestclusters.yaml
+  - hypershift.openshift.io_nodepools.yaml
+  - operator-namespace.yaml
+  - operator-serviceaccount.yaml
+  - operator-clusterrole.yaml
+  - operator-clusterrolebinding.yaml
+  - operator-deployment.yaml
+
+configMapGenerator:
+- name: release-info
+  namespace: hypershift
+  files:
+  - release-info.json
+
+patchesStrategicMerge:
+- |-
+  apiVersion: apiextensions.k8s.io/v1
+  kind: CustomResourceDefinition
+  metadata:
+    name: guestclusters.hypershift.openshift.io
+    labels:
+      cluster.x-k8s.io/v1alpha4: v1alpha1
+- |-
+  apiVersion: apiextensions.k8s.io/v1
+  kind: CustomResourceDefinition
+  metadata:
+    name: hostedcontrolplanes.hypershift.openshift.io
+    labels:
+      cluster.x-k8s.io/v1alpha4: v1alpha1
Original file line number	Diff line number	Diff line change
Expand Up		@@ -26,3 +26,5 @@ bin
		.envrc

		.kube

		config/hypershift-operator/release-info.json