From 3f3783eba1380ad0574e5b409a57e9530abdb17f Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Wed, 24 Apr 2019 19:48:45 -0700 Subject: [PATCH] Scripts for replicating Docker images using GCB to support private GKE and VPC service controls. (#3080) * Scripts for replicating Docker images using GCB to support private GKE and VPC service controls. * With private GKE we can't pull docker images from non-GCR registries (e.g. quoay) * To support private GKE clusters we want to make it easy for users to mirror Kubeflow images to their own registry * We create a GCB workflow to retag Kubeflow images * Using GCB is advantageous because it avoids pulling the networks over the user's network. * Update the script to update the Kubeflow components to use the images in the user's registry. * The kubeflow cluster isn't fully accessible yet. looks like pipelines and some other components still have images that need to be ported over. Related to #2086 * Update the images. --- .gitignore | 2 + scripts/gke/Makefile | 16 ++++ scripts/gke/gcb_copy_images.jsonnet | 54 +++++++++++++ scripts/gke/use_gcr_for_all_images.sh | 107 ++++++++++++++++++-------- 4 files changed, 145 insertions(+), 34 deletions(-) create mode 100644 scripts/gke/Makefile create mode 100644 scripts/gke/gcb_copy_images.jsonnet diff --git a/.gitignore b/.gitignore index d3588e49f20..b7bb1d05ddf 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,5 @@ components/gcp-click-to-deploy/src/user_config/** # This is generated by bootstrap **/reg_tmp + +scripts/gke/build/** \ No newline at end of file diff --git a/scripts/gke/Makefile b/scripts/gke/Makefile new file mode 100644 index 00000000000..fd5076dbe46 --- /dev/null +++ b/scripts/gke/Makefile @@ -0,0 +1,16 @@ +#--ext-str imageBase=$(IMG) \ +# --ext-str gitVersion=$(GIT_VERSION) --ext-str tag=$(TAG) \ +# --ext-str useImageCache=$(USE_IMAGE_CACHE) \ + +PROJECT ?= cloud-ml-dev +NEW_REGISTRY ?= gcr.io/$(PROJECT) + +build-gcb-spec: gcb_copy_images.jsonnet + rm -rf ./build + mkdir -p build + jsonnet ./gcb_copy_images.jsonnet --ext-str newRegistry=$(NEW_REGISTRY) \ + > ./build/gcb_copy_images.json + +copy-gcb: build-gcb-spec + gcloud builds submit --machine-type=n1-highcpu-32 --project=$(PROJECT) --config=./build/gcb_copy_images.json \ + --timeout=3600 --no-source \ No newline at end of file diff --git a/scripts/gke/gcb_copy_images.jsonnet b/scripts/gke/gcb_copy_images.jsonnet new file mode 100644 index 00000000000..13596ee5ff9 --- /dev/null +++ b/scripts/gke/gcb_copy_images.jsonnet @@ -0,0 +1,54 @@ +// This is a jsonnet file to generate a GCB workflow to copy Kubeflow docker images to a personal GCR registry. +// +// The primary purpose of this workflow is to copy Docker images hosted outside of GCR to a +// a GCR registry so they can be used with private GKE clusters. +{ + + // The newRegistry for the image + local newRegistry = std.extVar("newRegistry"), + + // A template for defining the steps to retag each image. + local subGraphTemplate(image) = { + local imagePieces = std.split(image, "/"), + local nameAndTag = std.split(imagePieces[std.length(imagePieces) -1], ":"), + + local name = nameAndTag[0], + + local template = self, + + local newImage = std.join("/", [newRegistry] + imagePieces[1:]), + + images+: [newImage], + + local pullName = "pull-" + name, + steps+: [ + { + id: pullName, + name: "gcr.io/cloud-builders/docker", + args: ["pull", image], + waitFor: ["-"], + }, + { + id: "tag-" + name, + name: "gcr.io/cloud-builders/docker", + args: ["tag", image, newImage], + waitFor: ["pull-" + name], + }, + ], + }, + + local images = [ + "argoproj/argoui:v2.2.0", + "argoproj/argoexec:v2.2.0", + "argoproj/workflow-controller:v2.2.0", + "metacontroller/metacontroller:v0.3.0", + "minio/minio:RELEASE.2018-02-09T22-40-05Z", + "mysql:8.0.3", + "quay.io/datawire/ambassador:0.37.0", + ], + + local steps = std.map(subGraphTemplate, images), + + local combine(l, r) = l+r, + all: std.foldl(combine, steps, {}), +}.all \ No newline at end of file diff --git a/scripts/gke/use_gcr_for_all_images.sh b/scripts/gke/use_gcr_for_all_images.sh index a73b49d06a2..3e7f799db5c 100755 --- a/scripts/gke/use_gcr_for_all_images.sh +++ b/scripts/gke/use_gcr_for_all_images.sh @@ -3,37 +3,76 @@ # app directory. It sets the docker image params in all the components to use the images # from gcr.io registries instead of non-gcr.io registries. This is useful when deploying # private GKE clusters where one can only pull images from gcr.io -# To push an image from DockerHub / Quay to gcr.io/kubeflow-images-public registry, use -# the following bash function -# sync_image() { -# local source="${1}" -# local target="gcr.io/kubeflow-images-public/${1}" -# docker pull "${source}" -# docker tag "${source}" "${target}" -# docker push "${target}" -# } -# Example invocations: -# sync_image prom/statsd-exporter:v0.6.0 -# sync_image quay.io/datawire/ambassador:0.37.0 - -set -x - -if ks component list | awk '{print $1}' | grep -q "^argo$"; then - ks param set argo workflowControllerImage gcr.io/kubeflow-images-public/argoproj/workflow-controller:v2.2.0 - ks param set argo uiImage gcr.io/kubeflow-images-public/argoproj/argoui:v2.2.0 - ks param set argo executorImage gcr.io/kubeflow-images-public/argoproj/argoexec:v2.2.0 -fi - -if ks component list | awk '{print $1}' | grep -q "^cert-manager$"; then - ks param set cert-manager certManagerImage gcr.io/kubeflow-images-public/quay.io/jetstack/cert-manager-controller:v0.2.4 - ks param set cert-manager certManagerIngressShimImage gcr.io/kubeflow-images-public/quay.io/jetstack/cert-manager-ingress-shim:v0.2.4 -fi - -if ks component list | awk '{print $1}' | grep -q "^ambassador$"; then - ks param set ambassador ambassadorImage gcr.io/kubeflow-images-public/quay.io/datawire/ambassador:0.37.0 -fi - -if ks component list | awk '{print $1}' | grep -q "^katib$"; then - ks param set katib modeldbDatabaseImage gcr.io/kubeflow-images-public/mongo:3.4 - ks param set katib vizierDbImage gcr.io/kubeflow-images-public/mysql:8.0.3 -fi +# +# To sync the images to your registry use +# PROJECT=$(PROJET) make copy-gcb + +set -xe + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +parseArgs() { + # Parse all command line options + while [[ $# -gt 0 ]]; do + # Parameters should be of the form + # --{name}=${value} + echo parsing "$1" + if [[ $1 =~ ^--(.*)=(.*)$ ]]; then + name=${BASH_REMATCH[1]} + value=${BASH_REMATCH[2]} + + eval ${name}="${value}" + elif [[ $1 =~ ^--(.*)$ ]]; then + name=${BASH_REMATCH[1]} + value=true + eval ${name}="${value}" + else + echo "Argument $1 did not match the pattern --{name}={value} or --{name}" + fi + shift + done +} + +usage() { + echo "Usage: use_gcr_for_all_images --registry=" +} + +main() { + # List of required parameters + names=(registry) + + missingParam=false + for i in ${names[@]}; do + if [ -z ${!i} ]; then + echo "--${i} not set" + missingParam=true + fi + done + + if ks component list | awk '{print $1}' | grep -q "^argo$"; then + ks param set argo workflowControllerImage ${registry}/workflow-controller:v2.2.0 + ks param set argo uiImage ${registry}/argoui:v2.2.0 + ks param set argo executorImage ${registry}/argoexec:v2.2.0 + fi + + if ks component list | awk '{print $1}' | grep -q "^ambassador$"; then + ks param set ambassador ambassadorImage ${registry}/datawire/ambassador:0.37.0 + fi + + if ks component list | awk '{print $1}' | grep -q "^katib$"; then + ks param set katib vizierDbImage ${registry}/mysql:8.0.3 + fi + + if ks component list | awk '{print $1}' | grep -q "^metacontroller$"; then + ks param set metacontroller image ${registry}/metacontroller:v0.3.0 + fi + + if ks component list | awk '{print $1}' | grep -q "^pipeline$"; then + ks param set pipeline mysqlImage ${registry}/minio:RELEASE.2018-02-09T22-40-05Z + ks param set minioImage mysqlImage ${registry}/mysql:8.0.3 + fi + +} + +parseArgs $* +main