First iteration and common utils for Jupyter web app CD (#5663)

* ci(utils): Add helpsrs for creating a kaniko task Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com> * ci(utils): Don't always add an exit dag Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com> * cd(jwa): Add CD workflow for building jwa Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com> * ci(utils): Add docker-config Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com> * cicd: Use base-commit[0:8] for the image tag Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com> * cd(config): Add a config file with list of images Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com> * Make CD workflow to be a postsubmit job Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>
kubeflow · Mar 5, 2021 · 764ce33 · 764ce33
1 parent a2c8b07
commit 764ce33
Show file tree

Hide file tree

Showing 8 changed files with 148 additions and 30 deletions.
diff --git a/prow_config.yaml b/prow_config.yaml
@@ -14,17 +14,12 @@ workflows:
     include_dirs:
       - components/crud-web-apps/common/frontend/kubeflow-common-lib/*
     kwargs: {}
-  # Image Auto Release workflows.
-  # The workflows below are related to auto building our Docker images.
-  # We have separate pre/postsubmit jobs because we want to use different
-  # registries
-  #- app_dir: kubeflow/kubeflow/components/tensorflow-notebook-image/releaser
-  #  component: workflows
-  #  name: tf-notebook-release
-  #  job_types:
-  #    - presubmit
-  #  params:
-  #    registry: "gcr.io/kubeflow-ci"
-  #    step_image: "gcr.io/kubeflow-ci/test-worker/test-worker:v20190116-b7abb8d-e3b0c4"
-  #  include_dirs:
-  #    - components/tensorflow-notebook-image/*
+  # post submit jobs
+  - py_func: kubeflow.kubeflow.cd.jwa.create_workflow
+    name: jwa-build
+    job_types:
+      - postsubmit
+    include_dirs:
+      - components/crud-web-apps/common/frontend/kubeflow-common-lib/*
+      - components/crud-web-apps/jupyter/*
+    kwargs: {}
diff --git a/py/kubeflow/kubeflow/cd/Makefile b/py/kubeflow/kubeflow/cd/Makefile
@@ -0,0 +1,16 @@
+# This file is only intended for development purposes
+
+# change this env var based on where kubeflow/testing repo lives
+# in your local machine
+KUBEFLOW_TESTING_REPO ?= /tmp/kubeflow/testing
+KUBEFLOW_KUBEFLOW_REPO ?= /tmp/kubeflow/kubeflow
+PYTHONPATH ?= "${KUBEFLOW_KUBEFLOW_REPO}/py:${KUBEFLOW_TESTING_REPO}/py"
+
+check-local-jwa-kaniko-build:
+	@PYTHONPATH=${PYTHONPATH} \
+	LOCAL_TESTING=True \
+	python jwa_runner.py
+
+check-prod-jwa-kaniko-build:
+	@PYTHONPATH=${PYTHONPATH} \
+	python jwa_runner.py
diff --git a/py/kubeflow/kubeflow/cd/__init__.py b/py/kubeflow/kubeflow/cd/__init__.py
diff --git a/py/kubeflow/kubeflow/cd/config.py b/py/kubeflow/kubeflow/cd/config.py
@@ -0,0 +1,12 @@
+AWS_REGISTRY = "public.ecr.aws/j1r0q0g6"
+
+# list of public images
+JUPYTER_WEB_APP_IMAGE = "%s/jupyter-web-app" % AWS_REGISTRY
+TENSORBOARDS_WEB_APP_IMAGE = "%s/tensorboards-web-app" % AWS_REGISTRY
+CENTRAL_DASHBOARD_IMAGE = "%s/central-dashboard" % AWS_REGISTRY
+
+ADMISSION_WEBHOOK_IMAGE = "%s/admission-webhook" % AWS_REGISTRY
+
+PROFILE_CONTROLLER_IMAGE = "%s/profile_controller" % AWS_REGISTRY
+NOTEBOOK_CONTROLLER_IMAGE = "%s/notebook-controller" % AWS_REGISTRY
+TENSORBOARD_CONTROLLER_IMAGE = "%s/tensorboard-controller" % AWS_REGISTRY
diff --git a/py/kubeflow/kubeflow/cd/jwa.py b/py/kubeflow/kubeflow/cd/jwa.py
@@ -0,0 +1,44 @@
+""""Argo Workflow for building Jupyter web app's OCI image using Kaniko"""
+from kubeflow.kubeflow import ci
+from kubeflow.kubeflow.cd import config
+from kubeflow.testing import argo_build_util
+
+
+class Builder(ci.workflow_utils.ArgoTestBuilder):
+    def __init__(self, name=None, namespace=None, bucket=None,
+                 test_target_name=None, **kwargs):
+        super().__init__(name=name, namespace=namespace, bucket=bucket,
+                         test_target_name=test_target_name, **kwargs)
+
+    def build(self):
+        """Build the Argo workflow graph"""
+        workflow = self.build_init_workflow(exit_dag=False)
+        task_template = self.build_task_template()
+
+        # Build JWA using Kaniko
+        dockerfile = ("%s/components/crud-web-apps"
+                      "/jupyter/Dockerfile") % self.src_dir
+        context = "dir://%s/components/crud-web-apps" % self.src_dir
+        destination = config.JUPYTER_WEB_APP_IMAGE
+
+        kaniko_task = self.create_kaniko_task(task_template, dockerfile,
+                                              context, destination)
+        argo_build_util.add_task_to_dag(workflow,
+                                        ci.workflow_utils.E2E_DAG_NAME,
+                                        kaniko_task, [self.mkdir_task_name])
+
+        # Set the labels on all templates
+        workflow = argo_build_util.set_task_template_labels(workflow)
+
+        return workflow
+
+
+def create_workflow(name=None, namespace=None, bucket=None, **kwargs):
+    """
+    Args:
+        name: Name to give to the workflow. This can also be used to name
+              things associated with the workflow.
+    """
+    builder = Builder(name=name, namespace=namespace, bucket=bucket, **kwargs)
+
+    return builder.build()
diff --git a/py/kubeflow/kubeflow/cd/jwa_runner.py b/py/kubeflow/kubeflow/cd/jwa_runner.py
@@ -0,0 +1,12 @@
+# This file is only intended for development purposes
+import os
+
+import yaml
+
+from kubeflow.kubeflow.cd import jwa
+
+WORKFLOW_NAME = os.getenv("WORKFLOW_NAME", "jwa-build")
+WORKFLOW_NAMESPACE = os.getenv("WORKFLOW_NAMESPACE", "kubeflow-user")
+
+if __name__ == "__main__":
+    print(yaml.dump(jwa.create_workflow(WORKFLOW_NAME, WORKFLOW_NAMESPACE)))
diff --git a/py/kubeflow/kubeflow/ci/__init__.py b/py/kubeflow/kubeflow/ci/__init__.py
@@ -0,0 +1 @@
+from . import workflow_utils  # noqa F401, F403
diff --git a/py/kubeflow/kubeflow/ci/workflow_utils.py b/py/kubeflow/kubeflow/ci/workflow_utils.py
@@ -13,10 +13,14 @@
 
 
 LOCAL_TESTING = os.getenv("LOCAL_TESTING", "False")
-CREDENTIALS_VOLUME = {"name": "aws-secret",
-                      "secret": {"secretName": "aws-secret"}}
-CREDENTIALS_MOUNT = {"mountPath": "/root/.aws/",
-                     "name": "aws-secret"}
+DOCKER_CONFIG_VOLUME = {"name": "docker-config",
+                        "configMap": {"name": "docker-config"}}
+DOCKER_CONFIG_MOUNT = {"name": "docker-config",
+                       "mountPath": "/kaniko/.docker/"}
+AWS_CREDENTIALS_VOLUME = {"name": "aws-secret",
+                          "secret": {"secretName": "aws-secret"}}
+AWS_CREDENTIALS_MOUNT = {"mountPath": "/root/.aws/",
+                         "name": "aws-secret"}
 
 AWS_WORKER_IMAGE = "public.ecr.aws/j1r0q0g6/kubeflow-testing:latest"
 
@@ -68,7 +72,7 @@ def __init__(self, name=None, namespace=None, bucket=None,
 
         self.go_path = self.test_dir
 
-    def _build_workflow(self):
+    def _build_workflow(self, exit_dag=True):
         """Create a scaffolding CR for the Argo workflow"""
         volumes = [{
             "name": DATA_VOLUME,
@@ -77,7 +81,8 @@ def _build_workflow(self):
             },
         }]
         if LOCAL_TESTING == "False":
-            volumes.append(CREDENTIALS_VOLUME)
+            volumes.append(AWS_CREDENTIALS_VOLUME)
+            volumes.append(DOCKER_CONFIG_VOLUME)
 
         workflow = {
             "apiVersion": "argoproj.io/v1alpha1",
@@ -98,24 +103,26 @@ def _build_workflow(self):
                 # Have argo garbage collect old workflows otherwise we overload
                 # the API server.
                 "volumes": volumes,
-                "onExit": EXIT_DAG_NAME,
                 "templates": [
                     {
                         "dag": {
                             "tasks": []
                         },
                         "name": E2E_DAG_NAME
                     },
-                    {
-                        "dag": {
-                            "tasks": []
-                        },
-                        "name":EXIT_DAG_NAME
-                    },
                 ],
             },  # spec
         }  # workflow
 
+        if exit_dag:
+            workflow["spec"]["onExit"] = EXIT_DAG_NAME
+            workflow["spec"]["templates"].append({
+                "dag": {
+                    "tasks": []
+                },
+                "name": EXIT_DAG_NAME
+            })
+
         return workflow
 
     def build_task_template(self):
@@ -125,7 +132,8 @@ def build_task_template(self):
             "name": DATA_VOLUME
         }]
         if LOCAL_TESTING == "False":
-            volume_mounts.append(CREDENTIALS_MOUNT)
+            volume_mounts.append(AWS_CREDENTIALS_MOUNT)
+            volume_mounts.append(DOCKER_CONFIG_MOUNT)
 
         image = AWS_WORKER_IMAGE
 
@@ -183,6 +191,36 @@ def build_task_template(self):
 
         return task_template
 
+    def create_kaniko_task(self, task_template, dockerfile, context,
+                           destination, no_push=False):
+        """
+        A task for building images inside a cluster container using Kaniko.
+        If we are testing the workflow locally then we won't be pushing images
+        to any registries. This will make it easier for people to try out and
+        extend the code.
+        """
+        kaniko = argo_build_util.deep_copy(task_template)
+
+        # append the tag base-commit[0:7]
+        if ":" not in destination:
+            sha = os.getenv("PULL_BASE_SHA", "12341234kanikotest")
+            base = os.getenv("PULL_BASE_REF", "master")
+            destination += ":%s-%s" % (base, sha[0:8])
+
+        kaniko["name"] = "kaniko-build-push"
+        kaniko["container"]["image"] = "gcr.io/kaniko-project/executor:v1.5.0"
+        kaniko["container"]["command"] = ["/kaniko/executor"]
+        kaniko["container"]["args"] = ["--dockerfile=%s" % dockerfile,
+                                       "--context=%s" % context,
+                                       "--destination=%s" % destination]
+
+        # don't push the image to a registry if trying out the produced
+        # Argo Workflow yaml locally
+        if LOCAL_TESTING == "True" or no_push:
+            kaniko["container"]["args"].append("--no-push")
+
+        return kaniko
+
     def _create_checkout_task(self, task_template):
         """Checkout the kubeflow/testing and kubeflow/kubeflow code"""
         main_repo = argo_build_util.get_repo_from_prow_env()
@@ -217,9 +255,9 @@ def _create_make_dir_task(self, task_template):
 
         return mkdir_step
 
-    def build_init_workflow(self):
+    def build_init_workflow(self, exit_dag=True):
         """Build the Argo workflow graph"""
-        workflow = self._build_workflow()
+        workflow = self._build_workflow(exit_dag)
         task_template = self.build_task_template()
 
         # checkout the code