Scripts for replicating Docker images using GCB to support private GK…

…E and VPC service controls. (kubeflow#3080) * Scripts for replicating Docker images using GCB to support private GKE and VPC service controls. * With private GKE we can't pull docker images from non-GCR registries (e.g. quoay) * To support private GKE clusters we want to make it easy for users to mirror Kubeflow images to their own registry * We create a GCB workflow to retag Kubeflow images * Using GCB is advantageous because it avoids pulling the networks over the user's network. * Update the script to update the Kubeflow components to use the images in the user's registry. * The kubeflow cluster isn't fully accessible yet. looks like pipelines and some other components still have images that need to be ported over. Related to kubeflow#2086 * Update the images.
swiftdiaries · May 2, 2019 · 3f3783e · 3f3783e
1 parent 671c09f
commit 3f3783e
Show file tree

Hide file tree

Showing 4 changed files with 145 additions and 34 deletions.
diff --git a/.gitignore b/.gitignore
@@ -50,3 +50,5 @@ components/gcp-click-to-deploy/src/user_config/**
 
 # This is generated by bootstrap
 **/reg_tmp
+
+scripts/gke/build/**
diff --git a/scripts/gke/Makefile b/scripts/gke/Makefile
@@ -0,0 +1,16 @@
+#--ext-str imageBase=$(IMG) \
+#	  --ext-str gitVersion=$(GIT_VERSION) --ext-str tag=$(TAG) \
+#	  --ext-str useImageCache=$(USE_IMAGE_CACHE) \
+
+PROJECT ?= cloud-ml-dev
+NEW_REGISTRY ?= gcr.io/$(PROJECT)
+
+build-gcb-spec: gcb_copy_images.jsonnet
+	rm -rf ./build
+	mkdir  -p build
+	jsonnet ./gcb_copy_images.jsonnet  --ext-str newRegistry=$(NEW_REGISTRY) \
+		> ./build/gcb_copy_images.json
+
+copy-gcb: build-gcb-spec		
+	gcloud builds submit --machine-type=n1-highcpu-32 --project=$(PROJECT) --config=./build/gcb_copy_images.json \
+		--timeout=3600 --no-source
diff --git a/scripts/gke/gcb_copy_images.jsonnet b/scripts/gke/gcb_copy_images.jsonnet
@@ -0,0 +1,54 @@
+// This is a jsonnet file to generate a GCB workflow to copy Kubeflow docker images to a personal GCR registry.
+//
+// The primary purpose of this workflow is to copy Docker images hosted outside of GCR to a 
+// a GCR registry so they can be used with private GKE clusters.
+{
+
+  // The newRegistry for the image
+  local newRegistry = std.extVar("newRegistry"),
+
+  // A template for defining the steps  to retag each image.
+  local subGraphTemplate(image) = {
+    local imagePieces = std.split(image, "/"),
+    local nameAndTag = std.split(imagePieces[std.length(imagePieces) -1], ":"),
+
+    local name = nameAndTag[0],
+
+    local template = self,
+
+    local newImage = std.join("/", [newRegistry] +  imagePieces[1:]),
+
+    images+: [newImage],
+
+    local pullName = "pull-" + name,
+    steps+: [
+      {
+        id: pullName,
+        name: "gcr.io/cloud-builders/docker",
+        args: ["pull", image],
+        waitFor: ["-"],
+      },
+      {
+        id: "tag-" + name,
+        name: "gcr.io/cloud-builders/docker",
+        args: ["tag", image, newImage],
+        waitFor: ["pull-" + name],
+      },
+    ],
+  },
+
+  local images = [
+    "argoproj/argoui:v2.2.0",
+    "argoproj/argoexec:v2.2.0",
+    "argoproj/workflow-controller:v2.2.0",
+    "metacontroller/metacontroller:v0.3.0",
+    "minio/minio:RELEASE.2018-02-09T22-40-05Z",
+    "mysql:8.0.3",
+    "quay.io/datawire/ambassador:0.37.0",
+  ],
+
+  local steps = std.map(subGraphTemplate, images),
+
+  local combine(l, r) = l+r,
+  all: std.foldl(combine, steps, {}),
+}.all
diff --git a/scripts/gke/use_gcr_for_all_images.sh b/scripts/gke/use_gcr_for_all_images.sh
@@ -3,37 +3,76 @@
 # app directory. It sets the docker image params in all the components to use the images
 # from gcr.io registries instead of non-gcr.io registries. This is useful when deploying
 # private GKE clusters where one can only pull images from gcr.io
-# To push an image from DockerHub / Quay to gcr.io/kubeflow-images-public registry, use
-# the following bash function
-# sync_image() {
-#   local source="${1}"
-#   local target="gcr.io/kubeflow-images-public/${1}"
-#   docker pull "${source}"
-#   docker tag "${source}" "${target}"
-#   docker push "${target}"
-# }
-# Example invocations:
-# sync_image prom/statsd-exporter:v0.6.0
-# sync_image quay.io/datawire/ambassador:0.37.0
-
-set -x
-
-if ks component list | awk '{print $1}' | grep -q "^argo$"; then
-  ks param set argo workflowControllerImage gcr.io/kubeflow-images-public/argoproj/workflow-controller:v2.2.0
-  ks param set argo uiImage gcr.io/kubeflow-images-public/argoproj/argoui:v2.2.0
-  ks param set argo executorImage gcr.io/kubeflow-images-public/argoproj/argoexec:v2.2.0
-fi
-
-if ks component list | awk '{print $1}' | grep -q "^cert-manager$"; then
-  ks param set cert-manager certManagerImage gcr.io/kubeflow-images-public/quay.io/jetstack/cert-manager-controller:v0.2.4
-  ks param set cert-manager certManagerIngressShimImage gcr.io/kubeflow-images-public/quay.io/jetstack/cert-manager-ingress-shim:v0.2.4
-fi
-
-if ks component list | awk '{print $1}' | grep -q "^ambassador$"; then
-  ks param set ambassador ambassadorImage gcr.io/kubeflow-images-public/quay.io/datawire/ambassador:0.37.0
-fi
-
-if ks component list | awk '{print $1}' | grep -q "^katib$"; then
-  ks param set katib modeldbDatabaseImage gcr.io/kubeflow-images-public/mongo:3.4
-  ks param set katib vizierDbImage gcr.io/kubeflow-images-public/mysql:8.0.3
-fi
+#
+# To sync the images to your registry use
+# PROJECT=$(PROJET) make copy-gcb
+
+set -xe
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+parseArgs() {
+  # Parse all command line options
+  while [[ $# -gt 0 ]]; do
+	# Parameters should be of the form
+	# --{name}=${value}
+	echo parsing "$1"
+	if [[ $1 =~ ^--(.*)=(.*)$ ]]; then
+	  name=${BASH_REMATCH[1]}
+	  value=${BASH_REMATCH[2]}
+
+	  eval ${name}="${value}"
+	elif [[ $1 =~ ^--(.*)$ ]]; then
+	name=${BASH_REMATCH[1]}
+	value=true
+	eval ${name}="${value}"
+	else
+	  echo "Argument $1 did not match the pattern --{name}={value} or --{name}"
+	fi
+	shift
+  done
+}
+
+usage() {
+  echo "Usage: use_gcr_for_all_images --registry=<REGISTRY>"
+}
+
+main() {
+	# List of required parameters
+  names=(registry)
+
+  missingParam=false
+  for i in ${names[@]}; do
+	if [ -z ${!i} ]; then
+	  echo "--${i} not set"
+	  missingParam=true
+	fi
+  done
+
+  if ks component list | awk '{print $1}' | grep -q "^argo$"; then
+    ks param set argo workflowControllerImage ${registry}/workflow-controller:v2.2.0
+    ks param set argo uiImage ${registry}/argoui:v2.2.0
+    ks param set argo executorImage ${registry}/argoexec:v2.2.0
+  fi
+
+  if ks component list | awk '{print $1}' | grep -q "^ambassador$"; then
+    ks param set ambassador ambassadorImage ${registry}/datawire/ambassador:0.37.0
+  fi
+
+  if ks component list | awk '{print $1}' | grep -q "^katib$"; then
+    ks param set katib vizierDbImage ${registry}/mysql:8.0.3
+  fi
+
+  if ks component list | awk '{print $1}' | grep -q "^metacontroller$"; then
+    ks param set metacontroller image ${registry}/metacontroller:v0.3.0
+  fi
+
+    if ks component list | awk '{print $1}' | grep -q "^pipeline$"; then
+    ks param set pipeline mysqlImage ${registry}/minio:RELEASE.2018-02-09T22-40-05Z
+    ks param set minioImage mysqlImage ${registry}/mysql:8.0.3
+  fi
+
+}
+
+parseArgs $*
+main
Original file line number	Diff line number	Diff line change
Expand Up		@@ -50,3 +50,5 @@ components/gcp-click-to-deploy/src/user_config/**

		# This is generated by bootstrap
		**/reg_tmp

		scripts/gke/build/**