Add opinionated PyTorch images with common tools

kubeflow · Mar 21, 2021 · c398c0c · c398c0c
1 parent c542fd8
commit c398c0c
Show file tree

Hide file tree

Showing 9 changed files with 131 additions and 1 deletion.
diff --git a/components/example-notebook-servers/jupyter-pytorch-full/Dockerfile.CPU b/components/example-notebook-servers/jupyter-pytorch-full/Dockerfile.CPU
@@ -0,0 +1,6 @@
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-pytorch:master-8dea1707
+
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
diff --git a/components/example-notebook-servers/jupyter-pytorch-full/Dockerfile.CUDA b/components/example-notebook-servers/jupyter-pytorch-full/Dockerfile.CUDA
@@ -0,0 +1,10 @@
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-pytorch-cuda:master-8dea1707
+
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
+
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir\
+    && rm -f requirements.txt
diff --git a/components/example-notebook-servers/jupyter-pytorch-full/requirements.txt b/components/example-notebook-servers/jupyter-pytorch-full/requirements.txt
@@ -0,0 +1,17 @@
+kfp==1.0.4
+kfp-server-api==1.0.4
+kfserving==0.4.1
+ipywidgets==7.6.3
+ipympl==0.6.3
+cloudpickle==1.6.0
+dill==0.3.3
+bokeh==2.3.0
+seaborn==0.11.1
+scipy==1.6.1
+scikit-learn==0.24.1
+scikit-image==0.18.1
+pandas==1.2.3
+matplotlib==3.3.4
+xgboost==1.3.3
+#torchelastic==0.2.2 this currently causes a dependency conflict, should be fixed very soon
+fastai==2.2.7
diff --git a/py/kubeflow/kubeflow/cd/kaniko_builder.py b/py/kubeflow/kubeflow/cd/kaniko_builder.py
@@ -10,6 +10,7 @@ def __init__(self, name=None, namespace=None, bucket=None,
                          test_target_name=test_target_name, **kwargs)
 
     def build(self, dockerfile, context, destination,
+              second_dockerfile=None, second_destination=None,
               mem_override=None, deadline_override=None):
         """Build the Argo workflow graph"""
         workflow = self.build_init_workflow(exit_dag=False)
@@ -26,6 +27,17 @@ def build(self, dockerfile, context, destination,
                                         ci.workflow_utils.E2E_DAG_NAME,
                                         kaniko_task, [self.mkdir_task_name])
 
+        if second_dockerfile and second_destination:
+            dockerfile = ("%s/%s") % (self.src_dir, second_dockerfile)
+            destination = second_destination
+
+            second_kaniko_task = self.create_kaniko_task(task_template, dockerfile,
+                                              context, destination)
+
+            argo_build_util.add_task_to_dag(workflow,
+                                        ci.workflow_utils.E2E_DAG_NAME,
+                                        second_kaniko_task, [self.mkdir_task_name])
+
         # Set the labels on all templates
         workflow = argo_build_util.set_task_template_labels(workflow)
 

diff --git a/py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch_full.py b/py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch_full.py
@@ -0,0 +1,17 @@
+""""Argo Workflow for building the notebook-server-jupyter-pytorch-full OCI images using Kaniko"""
+from kubeflow.kubeflow.cd import config, kaniko_builder
+
+
+def create_workflow(name=None, namespace=None, bucket=None, **kwargs):
+    """
+    Args:
+        name: Name to give to the workflow. This can also be used to name
+              things associated with the workflow.
+    """
+    builder = kaniko_builder.Builder(name=name, namespace=namespace, bucket=bucket, **kwargs)
+
+    return builder.build(dockerfile="components/example-notebook-servers/jupyter-pytorch-full/Dockerfile.CPU",
+                         context="components/example-notebook-servers/jupyter-pytorch-full/",
+                         destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH_FULL,
+                         second_dockerfile="components/example-notebook-servers/jupyter-pytorch-full/Dockerfile.CUDA",
+                         second_destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA_FULL)
diff --git a/py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch_full_runner.py b/py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch_full_runner.py
@@ -0,0 +1,5 @@
+# This file is only intended for development purposes
+from kubeflow.kubeflow.cd import base_runner
+
+base_runner.main(component_name="notebook_servers.notebook_server_jupyter_pytorch_full",
+                 workflow_name="nb-j-pt-f-build")
diff --git a/py/kubeflow/kubeflow/ci/notebook_servers/notebook_server_jupyter_pytorch_full_tests.py b/py/kubeflow/kubeflow/ci/notebook_servers/notebook_server_jupyter_pytorch_full_tests.py
@@ -0,0 +1,53 @@
+""""Argo Workflow for testing notebook-server-jupyter-pytorch-full OCI images"""
+from kubeflow.kubeflow.ci import workflow_utils
+from kubeflow.testing import argo_build_util
+
+
+class Builder(workflow_utils.ArgoTestBuilder):
+    def __init__(self, name=None, namespace=None, bucket=None,
+                 test_target_name=None, **kwargs):
+        super().__init__(name=name, namespace=namespace, bucket=bucket,
+                         test_target_name=test_target_name, **kwargs)
+
+    def build(self):
+        """Build the Argo workflow graph"""
+        workflow = self.build_init_workflow(exit_dag=False)
+        task_template = self.build_task_template()
+
+        # Test building notebook-server-jupyter-pytorch-full images using Kaniko
+        dockerfile = ("%s/components/example-notebook-servers"
+                      "/jupyter-pytorch-full/Dockerfile.CPU") % self.src_dir
+        context = "dir://%s/components/example-notebook-servers/jupyter-pytorch-full/" % self.src_dir
+        destination = "notebook-server-jupyter-pytorch-full-cpu-test"
+
+        kaniko_task = self.create_kaniko_task(task_template, dockerfile,
+                                              context, destination, no_push=True)
+        argo_build_util.add_task_to_dag(workflow,
+                                        workflow_utils.E2E_DAG_NAME,
+                                        kaniko_task, [self.mkdir_task_name])
+
+        dockerfile_cuda = ("%s/components/example-notebook-servers"
+                      "/jupyter-pytorch-full/Dockerfile.CUDA") % self.src_dir
+        destination_cuda = "notebook-server-jupyter-pytorch-full-cuda-test"
+
+        kaniko_task_cuda = self.create_kaniko_task(task_template, dockerfile_cuda,
+                                              context, destination_cuda, no_push=True)
+        argo_build_util.add_task_to_dag(workflow,
+                                        workflow_utils.E2E_DAG_NAME,
+                                        kaniko_task_cuda, [self.mkdir_task_name])
+
+        # Set the labels on all templates
+        workflow = argo_build_util.set_task_template_labels(workflow)
+
+        return workflow
+
+
+def create_workflow(name=None, namespace=None, bucket=None, **kwargs):
+    """
+    Args:
+        name: Name to give to the workflow. This can also be used to name
+              things associated with the workflow.
+    """
+    builder = Builder(name=name, namespace=namespace, bucket=bucket, **kwargs)
+
+    return builder.build()
diff --git a/...ubeflow/kubeflow/ci/notebook_servers/notebook_server_jupyter_pytorch_full_tests_runner.py b/...ubeflow/kubeflow/ci/notebook_servers/notebook_server_jupyter_pytorch_full_tests_runner.py
@@ -0,0 +1,5 @@
+# This file is only intended for development purposes
+from kubeflow.kubeflow.ci import base_runner
+
+base_runner.main(component_name="notebook_servers.notebook_server_jupyter_pytorch_full_tests",
+                 workflow_name="nb-j-pt-f-tests")
diff --git a/py/kubeflow/kubeflow/ci/workflow_utils.py b/py/kubeflow/kubeflow/ci/workflow_utils.py
@@ -1,5 +1,7 @@
 import logging
 import os
+import string
+import random
 
 from kubeflow.testing import argo_build_util
 
@@ -206,14 +208,17 @@ def create_kaniko_task(self, task_template, dockerfile, context,
         extend the code.
         """
         kaniko = argo_build_util.deep_copy(task_template)
+        # for shurt UUID generation
+        alphabet = string.ascii_lowercase + string.digits
 
         # append the tag base-commit[0:7]
         if ":" not in destination:
             sha = os.getenv("PULL_BASE_SHA", "12341234kanikotest")
             base = os.getenv("PULL_BASE_REF", "master")
             destination += ":%s-%s" % (base, sha[0:8])
 
-        kaniko["name"] = "kaniko-build-push"
+        # add short UUID to step name to ensure it is unique
+        kaniko["name"] = "kaniko-build-push-" + ''.join(random.choices(alphabet, k=8))
         kaniko["container"]["image"] = "gcr.io/kaniko-project/executor:v1.5.0"
         kaniko["container"]["command"] = ["/kaniko/executor"]
         kaniko["container"]["args"] = ["--dockerfile=%s" % dockerfile,