Add notebook-server-jupyter-pytorch dockerfiles and CI/CD (kubeflow#5626

) * Add pytorch base images * cleanup jupyter pytorch notebook image * rename jupyter pytorch dockerfiles * fix typo Co-authored-by: Mathew Wicks <thesuperzapper@users.noreply.github.com>
hiramf · Mar 21, 2021 · 8dea170 · 8dea170
1 parent b3fe73b
commit 8dea170
Show file tree

Hide file tree

Showing 10 changed files with 121 additions and 1 deletion.
diff --git a/components/example-notebook-servers/jupyter-pytorch/cpu.Dockerfile b/components/example-notebook-servers/jupyter-pytorch/cpu.Dockerfile
@@ -0,0 +1,6 @@
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-b3fe73b2
+
+# install - requirements.txt
+COPY --chown=jovyan:users requirements-cpu.txt /tmp/requirements.txt
+RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html \
+ && rm -f /tmp/requirements.txt
diff --git a/components/example-notebook-servers/jupyter-pytorch/cuda.Dockerfile b/components/example-notebook-servers/jupyter-pytorch/cuda.Dockerfile
@@ -0,0 +1,11 @@
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-b3fe73b2
+
+# nvidia configs
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
+
+# install - requirements.txt
+COPY --chown=jovyan:users requirements-cuda.txt /tmp/requirements.txt
+RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html \
+ && rm -f /tmp/requirements.txt
diff --git a/components/example-notebook-servers/jupyter-pytorch/requirements-cpu.txt b/components/example-notebook-servers/jupyter-pytorch/requirements-cpu.txt
@@ -0,0 +1,3 @@
+torch==1.7.1+cpu
+torchvision==0.8.2+cpu
+torchaudio==0.7.2
diff --git a/components/example-notebook-servers/jupyter-pytorch/requirements-cuda.txt b/components/example-notebook-servers/jupyter-pytorch/requirements-cuda.txt
@@ -0,0 +1,3 @@
+torch==1.7.1+cu110
+torchvision==0.8.2+cu110
+torchaudio==0.7.2
diff --git a/py/kubeflow/kubeflow/cd/kaniko_builder.py b/py/kubeflow/kubeflow/cd/kaniko_builder.py
@@ -10,6 +10,7 @@ def __init__(self, name=None, namespace=None, bucket=None,
                          test_target_name=test_target_name, **kwargs)
 
     def build(self, dockerfile, context, destination,
+              second_dockerfile=None, second_destination=None,
               mem_override=None, deadline_override=None):
         """Build the Argo workflow graph"""
         workflow = self.build_init_workflow(exit_dag=False)
@@ -26,6 +27,17 @@ def build(self, dockerfile, context, destination,
                                         ci.workflow_utils.E2E_DAG_NAME,
                                         kaniko_task, [self.mkdir_task_name])
 
+        if second_dockerfile and second_destination:
+            dockerfile = ("%s/%s") % (self.src_dir, second_dockerfile)
+            destination = second_destination
+
+            second_kaniko_task = self.create_kaniko_task(task_template, dockerfile,
+                                              context, destination)
+
+            argo_build_util.add_task_to_dag(workflow,
+                                        ci.workflow_utils.E2E_DAG_NAME,
+                                        second_kaniko_task, [self.mkdir_task_name])
+
         # Set the labels on all templates
         workflow = argo_build_util.set_task_template_labels(workflow)
 

diff --git a/py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch.py b/py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch.py
@@ -0,0 +1,17 @@
+""""Argo Workflow for building the notebook-server-jupyter-pytorch OCI images using Kaniko"""
+from kubeflow.kubeflow.cd import config, kaniko_builder
+
+
+def create_workflow(name=None, namespace=None, bucket=None, **kwargs):
+    """
+    Args:
+        name: Name to give to the workflow. This can also be used to name
+              things associated with the workflow.
+    """
+    builder = kaniko_builder.Builder(name=name, namespace=namespace, bucket=bucket, **kwargs)
+
+    return builder.build(dockerfile="components/example-notebook-servers/jupyter-pytorch/cpu.Dockerfile",
+                         context="components/example-notebook-servers/jupyter-pytorch/",
+                         destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH,
+                         second_dockerfile="components/example-notebook-servers/jupyter-pytorch/cuda.Dockerfile",
+                         second_destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA)
diff --git a/py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch_runner.py b/py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch_runner.py
@@ -0,0 +1,5 @@
+# This file is only intended for development purposes
+from kubeflow.kubeflow.cd import base_runner
+
+base_runner.main(component_name="notebook_servers.notebook_server_jupyter_pytorch",
+                 workflow_name="nb-j-pt-build")
diff --git a/py/kubeflow/kubeflow/ci/notebook_servers/notebook_server_jupyter_pytorch_tests.py b/py/kubeflow/kubeflow/ci/notebook_servers/notebook_server_jupyter_pytorch_tests.py
@@ -0,0 +1,53 @@
+""""Argo Workflow for testing the notebook-server-jupyter-pytorch OCI images"""
+from kubeflow.kubeflow.ci import workflow_utils
+from kubeflow.testing import argo_build_util
+
+
+class Builder(workflow_utils.ArgoTestBuilder):
+    def __init__(self, name=None, namespace=None, bucket=None,
+                 test_target_name=None, **kwargs):
+        super().__init__(name=name, namespace=namespace, bucket=bucket,
+                         test_target_name=test_target_name, **kwargs)
+
+    def build(self):
+        """Build the Argo workflow graph"""
+        workflow = self.build_init_workflow(exit_dag=False)
+        task_template = self.build_task_template()
+
+        # Test building notebook-server-jupyter-pytorch images using Kaniko
+        dockerfile = ("%s/components/example-notebook-servers"
+                      "/jupyter-pytorch/cpu.Dockerfile") % self.src_dir
+        context = "dir://%s/components/example-notebook-servers/jupyter-pytorch/" % self.src_dir
+        destination = "notebook-server-jupyter-pytorch-cpu-test"
+
+        kaniko_task = self.create_kaniko_task(task_template, dockerfile,
+                                              context, destination, no_push=True)
+        argo_build_util.add_task_to_dag(workflow,
+                                        workflow_utils.E2E_DAG_NAME,
+                                        kaniko_task, [self.mkdir_task_name])
+
+        dockerfile_cuda = ("%s/components/example-notebook-servers"
+                      "/jupyter-pytorch/cuda.Dockerfile") % self.src_dir
+        destination_cuda = "notebook-server-jupyter-pytorch-cuda-test"
+
+        kaniko_task_cuda = self.create_kaniko_task(task_template, dockerfile_cuda,
+                                              context, destination_cuda, no_push=True)
+        argo_build_util.add_task_to_dag(workflow,
+                                        workflow_utils.E2E_DAG_NAME,
+                                        kaniko_task_cuda, [self.mkdir_task_name])
+
+        # Set the labels on all templates
+        workflow = argo_build_util.set_task_template_labels(workflow)
+
+        return workflow
+
+
+def create_workflow(name=None, namespace=None, bucket=None, **kwargs):
+    """
+    Args:
+        name: Name to give to the workflow. This can also be used to name
+              things associated with the workflow.
+    """
+    builder = Builder(name=name, namespace=namespace, bucket=bucket, **kwargs)
+
+    return builder.build()
diff --git a/py/kubeflow/kubeflow/ci/notebook_servers/notebook_server_jupyter_pytorch_tests_runner.py b/py/kubeflow/kubeflow/ci/notebook_servers/notebook_server_jupyter_pytorch_tests_runner.py
@@ -0,0 +1,5 @@
+# This file is only intended for development purposes
+from kubeflow.kubeflow.ci import base_runner
+
+base_runner.main(component_name="notebook_servers.notebook_server_jupyter_pytorch_tests",
+                 workflow_name="nb-j-pt-tests")
diff --git a/py/kubeflow/kubeflow/ci/workflow_utils.py b/py/kubeflow/kubeflow/ci/workflow_utils.py
@@ -1,5 +1,7 @@
 import logging
 import os
+import string
+import random
 
 from kubeflow.testing import argo_build_util
 
@@ -206,14 +208,17 @@ def create_kaniko_task(self, task_template, dockerfile, context,
         extend the code.
         """
         kaniko = argo_build_util.deep_copy(task_template)
+        # for short UUID generation
+        alphabet = string.ascii_lowercase + string.digits
 
         # append the tag base-commit[0:7]
         if ":" not in destination:
             sha = os.getenv("PULL_BASE_SHA", "12341234kanikotest")
             base = os.getenv("PULL_BASE_REF", "master")
             destination += ":%s-%s" % (base, sha[0:8])
 
-        kaniko["name"] = "kaniko-build-push"
+        # add short UUID to step name to ensure it is unique
+        kaniko["name"] = "kaniko-build-push-" + ''.join(random.choices(alphabet, k=8))
         kaniko["container"]["image"] = "gcr.io/kaniko-project/executor:v1.5.0"
         kaniko["container"]["command"] = ["/kaniko/executor"]
         kaniko["container"]["args"] = ["--dockerfile=%s" % dockerfile,