Upgrade Kubebuilder to 3.0.0 and optimize Github workflow

1. Deprecate genclient and deepcopy-gen markers 2. Add health/ready check and use nonroot user in deployment 3. Fix github workflow issue Signed-off-by: Jiaxin Shan <seedjeffwan@gmail.com>
ray-project · Jeffwan · Sep 16, 2021 · Sep 14, 2021 · Sep 14, 2021 · a4a97a24bdadc210a3e60f4db05e4e0f951ff179
commit a4a97a24bdadc210a3e60f4db05e4e0f951ff179
diff --git a/.github/workflows/test-job.yaml b/.github/workflows/test-job.yaml
@@ -16,38 +16,36 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.x
+    - name: Set up Go 1.15.x
       uses: actions/setup-go@v2
       with:
-        go-version: ^1.13
+        go-version: '~1.15.1'
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
-        ref: master
+        # When checking out the repository that
+        # triggered a workflow, this defaults to the reference or SHA for that event.
+        # Default value should work for both pull_request and merge(push) event.
+        ref: ${{github.event.pull_request.head.sha}}
 
     - name: list directories
       working-directory: ${{env.working-directory}}
       run: ls -R
 
     - name: install kubebuilder
       run: |
-        wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.0.0-alpha.1/kubebuilder_2.0.0-alpha.1_linux_amd64.tar.gz
-        tar -zxvf  kubebuilder_2.0.0-alpha.1_linux_amd64.tar.gz
-        sudo mv kubebuilder_2.0.0-alpha.1_linux_amd64 /usr/local/kubebuilder
-
+        wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.0.0/kubebuilder_$(go env GOOS)_$(go env GOARCH)
+        sudo mv kubebuilder_$(go env GOOS)_$(go env GOARCH) /usr/local/bin/kubebuilder
+
     - name: Get dependencies
       working-directory: ${{env.working-directory}}
-      run: |
-        go get -v -t -d ./...
-        if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-        fi
+      run: go mod download
+
     - name: Build
-      run: go build -v ./...
+      run: make build
       working-directory: ${{env.working-directory}}
 
     - name: Test
-      run: go test -v ./... -cover
+      run: make test
       working-directory: ${{env.working-directory}}
diff --git a/ray-operator/.dockerignore b/ray-operator/.dockerignore
@@ -0,0 +1,5 @@
+# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
+# Ignore all files which are not go type
+!**/*.go
+!**/*.mod
+!**/*.sum
diff --git a/ray-operator/.gitignore b/ray-operator/.gitignore
@@ -30,6 +30,7 @@ user.bazelrc
 *.dll
 *.so
 *.dylib
+testbin/*
 
 # Test binary, built with `go test -c`
 *.test

diff --git a/ray-operator/Dockerfile b/ray-operator/Dockerfile
@@ -22,6 +22,6 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
 COPY --from=builder /workspace/manager .
-USER nonroot:nonroot
+USER 65532:65532
 
 ENTRYPOINT ["/manager"]
diff --git a/ray-operator/Makefile b/ray-operator/Makefile
@@ -1,6 +1,6 @@
 
 # Image URL to use all building/pushing image targets
-IMG ?=  ray-controller:latest
+IMG ?= controller:latest
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:maxDescLen=100,trivialVersions=true,preserveUnknownFields=false"
 
@@ -11,66 +11,98 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
-all: manager
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# This is a requirement for 'setup-envtest.sh' in the test target.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
 
-# Run tests
-test: generate fmt vet manifests
-	go test ./... -coverprofile cover.out
+all: build
 
-# Build manager binary
-manager: generate fmt vet
-	go build -o bin/manager main.go
+##@ General
 
-# Run against the configured Kubernetes cluster in ~/.kube/config
-run: generate fmt vet manifests
-	go run ./main.go
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
 
-# Install CRDs into a cluster
-install: manifests
-	kustomize build config/crd | kubectl apply -f -
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
-deploy: manifests
-	cd config/manager && kustomize edit set image controller=${IMG}
-	kustomize build config/default | kubectl apply -f -
+##@ Development
 
-# Generate manifests e.g. CRD, RBAC etc.
-manifests: controller-gen
+manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
-# Run go fmt against code
-fmt:
+generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+
+fmt: ## Run go fmt against code.
 	go fmt ./...
 
-# Run go vet against code
-vet:
+vet: ## Run go vet against code.
 	go vet ./...
 
-# Generate code
-generate: controller-gen
-	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
+ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
+test: manifests generate fmt vet ## Run tests.
+	mkdir -p ${ENVTEST_ASSETS_DIR}
+	test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh
+	source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out
 
-# Build the docker image
-docker-build: test
-	docker build . -t ${IMG}
+##@ Build
+
+build: generate fmt vet ## Build manager binary.
+	go build -o bin/manager main.go
 
-# Push the docker image
-docker-push:
+run: manifests generate fmt vet ## Run a controller from your host.
+	go run ./main.go
+
+docker-build: test ## Build docker image with the manager.
+	docker build -t ${IMG} .
+
+docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
 
-# find or download controller-gen
-# download controller-gen if necessary
-controller-gen:
-ifeq (, $(shell which controller-gen))
-	@{ \
-	set -e ;\
-	CONTROLLER_GEN_TMP_DIR="$$(mktemp -d)" ;\
-	cd "$$CONTROLLER_GEN_TMP_DIR" ;\
-	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.5.0 ;\
-	rm -rf "$$CONTROLLER_GEN_TMP_DIR" ;\
-	}
-CONTROLLER_GEN=$(GOBIN)/controller-gen
-else
-CONTROLLER_GEN=$(shell which controller-gen)
-endif
+##@ Deployment
+
+install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+
+uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl delete -f -
+
+deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/default | kubectl apply -f -
+
+undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/default | kubectl delete -f -
+
+
+CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+controller-gen: ## Download controller-gen locally if necessary.
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.5.0)
+
+KUSTOMIZE = $(shell pwd)/bin/kustomize
+kustomize: ## Download kustomize locally if necessary.
+	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7)
+
+# go-get-tool will 'go get' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+define go-get-tool
+@[ -f $(1) ] || { \
+set -e ;\
+TMP_DIR=$$(mktemp -d) ;\
+cd $$TMP_DIR ;\
+go mod init tmp ;\
+echo "Downloading $(2)" ;\
+GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
+rm -rf $$TMP_DIR ;\
+}
+endef
diff --git a/ray-operator/api/v1alpha1/raycluster_types.go b/ray-operator/api/v1alpha1/raycluster_types.go
@@ -101,10 +101,8 @@ const (
 )
 
 // RayCluster is the Schema for the RayClusters API
-// +genclient
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-// +kubebuilder:object:root=true
-// +kubebuilder:subresource:status
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
 type RayCluster struct {
 	// Standard object metadata.
 	metav1.TypeMeta   `json:",inline"`
@@ -115,7 +113,7 @@ type RayCluster struct {
 	Status RayClusterStatus `json:"status,omitempty"`
 }
 
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+//+kubebuilder:object:root=true
 
 // RayClusterList contains a list of RayCluster
 type RayClusterList struct {

diff --git a/ray-operator/config/manager/manager.yaml b/ray-operator/config/manager/manager.yaml
@@ -22,13 +22,17 @@ spec:
       labels:
         control-plane: ray-operator
     spec:
+      securityContext:
+        runAsNonRoot: true
       containers:
       - command:
         - /manager
 #        args:
 #        - --enable-leader-election
         image: controller:latest
         name: ray-manager
+        securityContext:
+          allowPrivilegeEscalation: false
         resources:
           limits:
             cpu: 100m

diff --git a/ray-operator/controllers/raycluster_controller.go b/ray-operator/controllers/raycluster_controller.go
@@ -68,7 +68,7 @@ type RayClusterReconciler struct {
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch
 // Reconcile used to bridge the desired state with the current state
-func (r *RayClusterReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+func (r *RayClusterReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
 	_ = r.Log.WithValues("raycluster", request.NamespacedName)
 	log.Info("reconciling RayCluster", "cluster name", request.Name)
 
@@ -77,19 +77,19 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, request reconcile.
 	if err := r.Get(context.TODO(), request.NamespacedName, instance); err != nil {
 		log.Error(err, "Read request instance error!")
 		// Error reading the object - requeue the request.
-		return reconcile.Result{}, client.IgnoreNotFound(err)
+		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
 	if err := r.reconcileServices(instance); err != nil {
-		return reconcile.Result{RequeueAfter: DefaultRequeueDuration}, err
+		return ctrl.Result{RequeueAfter: DefaultRequeueDuration}, err
 	}
 	if err := r.reconcilePods(instance); err != nil {
-		return reconcile.Result{RequeueAfter: DefaultRequeueDuration}, err
+		return ctrl.Result{RequeueAfter: DefaultRequeueDuration}, err
 	}
 
 	// update the status if needed
 	r.updateStatus(instance)
-	return reconcile.Result{}, nil
+	return ctrl.Result{}, nil
 }
 
 func (r *RayClusterReconciler) reconcileServices(instance *rayiov1alpha1.RayCluster) error {

diff --git a/ray-operator/main.go b/ray-operator/main.go
@@ -3,15 +3,17 @@ package main
 import (
 	"flag"
 	"os"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 
-	rayv1 "github.com/ray-project/ray-contrib/ray-operator/api/v1alpha1"
 	"github.com/ray-project/ray-contrib/ray-operator/controllers"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	rayiov1alpha1 "github.com/ray-project/ray-contrib/ray-operator/api/v1alpha1"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -21,31 +23,36 @@ var (
 )
 
 func init() {
-	_ = clientgoscheme.AddToScheme(scheme)
-
-	_ = rayv1.AddToScheme(scheme)
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(rayiov1alpha1.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }
 
 func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
+	var probeAddr string
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
+	opts := zap.Options{
+		Development: true,
+	}
+	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
-	ctrl.SetLogger(zap.New(func(o *zap.Options) {
-		o.Development = true
-	}))
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
 	setupLog.Info("the operator", "version:", os.Getenv("OPERATOR_VERSION"))
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:             scheme,
-		MetricsBindAddress: metricsAddr,
-		LeaderElection:     enableLeaderElection,
-		Port:               9443,
+		Scheme:                 scheme,
+		MetricsBindAddress:     metricsAddr,
+		Port:                   9443,
+		HealthProbeBindAddress: probeAddr,
+		LeaderElection:         enableLeaderElection,
+		LeaderElectionID:       "ray-operator-leader",
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
@@ -58,6 +65,15 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
-Original file line number
+Diff line change
@@ @@ -30,6 +30,7 @@ user.bazelrc @@
     *.dll
     *.so
     *.dylib
+    testbin/*
     # Test binary, built with `go test -c`
     *.test