Adjust ZenBytes to zenml version 0.8.0. (#9)

* Adjust ZenBytes to zenml version 0.8.0. * Further updates for 0.8.0.
zenml-io · May 24, 2022 · 1f72bf2 · 1f72bf2
1 parent f0c1709
commit 1f72bf2
Show file tree

Hide file tree

Showing 10 changed files with 60 additions and 55 deletions.
diff --git a/1-1_Pipelines.ipynb b/1-1_Pipelines.ipynb
@@ -27,7 +27,7 @@
    "outputs": [],
    "source": [
     "%pip install zenml\n",
-    "!zenml integration install sklearn -f\n",
+    "!zenml integration install sklearn -y\n",
     "%pip install pyparsing==2.4.2  # required for Colab\n",
     "\n",
     "import IPython\n",
@@ -221,10 +221,10 @@
  ],
  "metadata": {
   "interpreter": {
-   "hash": "9f70ec6e6bd16014ded89c8222361cbe53cd9507d51ebdcdf3ab6e494d45cf74"
+   "hash": "a35bb4b4bceaf970a493ff7351e9d97180ab3fe9951c21e9e29c55a687242182"
   },
   "kernelspec": {
-   "display_name": "Python 3.8.13 64-bit ('zenbytes')",
+   "display_name": "Python 3.8.13 64-bit ('zenbytes-latest')",
    "language": "python",
    "name": "python3"
   },

diff --git a/1-2_Artifact_Lineage.ipynb b/1-2_Artifact_Lineage.ipynb
@@ -27,7 +27,7 @@
    "outputs": [],
    "source": [
     "%pip install zenml\n",
-    "!zenml integration install sklearn dash -f\n",
+    "!zenml integration install sklearn dash -y\n",
     "%pip install pyparsing==2.4.2  # required for Colab\n",
     "\n",
     "import IPython\n",
@@ -45,7 +45,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -92,9 +92,9 @@
    "source": [
     "from zenml.pipelines import pipeline\n",
     "\n",
+    "from steps.evaluator import evaluator\n",
     "from steps.importer import importer\n",
     "from steps.sklearn_trainer import svc_trainer\n",
-    "from steps.evaluator import evaluator\n",
     "\n",
     "\n",
     "@pipeline\n",
@@ -348,10 +348,10 @@
  ],
  "metadata": {
   "interpreter": {
-   "hash": "9f70ec6e6bd16014ded89c8222361cbe53cd9507d51ebdcdf3ab6e494d45cf74"
+   "hash": "a35bb4b4bceaf970a493ff7351e9d97180ab3fe9951c21e9e29c55a687242182"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3.8.13 64-bit ('zenbytes-latest')",
    "language": "python",
    "name": "python3"
   },

diff --git a/2-1_Experiment_Tracking.ipynb b/2-1_Experiment_Tracking.ipynb
@@ -28,7 +28,7 @@
    "outputs": [],
    "source": [
     "%pip install zenml\n",
-    "!zenml integration install sklearn mlflow wandb -f\n",
+    "!zenml integration install sklearn mlflow wandb -y\n",
     "%pip install pyparsing==2.4.2  # required for Colab\n",
     "\n",
     "import IPython\n",
@@ -111,9 +111,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from pipelines.digits_pipeline import digits_pipeline\n",
     "from steps.evaluator import evaluator\n",
-    "from steps.importer import importer\n",
-    "from pipelines.digits_pipeline import digits_pipeline"
+    "from steps.importer import importer"
    ]
   },
   {
@@ -135,12 +135,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import mlflow\n",
     "import numpy as np\n",
     "from sklearn.base import ClassifierMixin\n",
     "from sklearn.svm import SVC\n",
-    "import mlflow\n",
     "from zenml.integrations.mlflow.mlflow_step_decorator import enable_mlflow\n",
-    "from zenml.steps import step, BaseStepConfig\n",
+    "from zenml.steps import step\n",
     "\n",
     "\n",
     "def build_svc_mlflow_pipeline(gamma=1e-3):\n",
@@ -213,7 +213,7 @@
    "outputs": [],
    "source": [
     "# Register the MLflow experiment tracker\n",
-    "!zenml experiment-tracker register mlflow_tracker --type=mlflow\n",
+    "!zenml experiment-tracker register mlflow_tracker --flavor=mlflow\n",
     "\n",
     "# Add the MLflow experiment tracker into our default stack\n",
     "!zenml stack update default -e mlflow_tracker"
@@ -315,7 +315,7 @@
    "outputs": [],
    "source": [
     "# Register the W&B experiment tracker\n",
-    "!zenml experiment-tracker register wandb_tracker --type=wandb --api_key={WANDB_API_KEY} --entity={WANDB_ENTITY} --project_name={WANDB_PROJECT}\n",
+    "!zenml experiment-tracker register wandb_tracker --flavor=wandb --api_key={WANDB_API_KEY} --entity={WANDB_ENTITY} --project_name={WANDB_PROJECT}\n",
     "\n",
     "# Create a new MLOps stack with W&B experiment tracker in it\n",
     "!zenml stack register wandb_stack -m default -a default -o default -e wandb_tracker\n",
@@ -348,9 +348,10 @@
     "from sklearn.svm import SVC\n",
     "import wandb\n",
     "from zenml.integrations.wandb.wandb_step_decorator import enable_wandb\n",
-    "from zenml.steps import step, BaseStepConfig\n",
-    "from steps.importer import importer\n",
+    "from zenml.steps import step\n",
+    "\n",
     "from pipelines.digits_pipeline import digits_pipeline\n",
+    "from steps.importer import importer\n",
     "\n",
     "\n",
     "def build_svc_wandb_pipeline(gamma=1e-3):\n",
@@ -422,12 +423,12 @@
  ],
  "metadata": {
   "interpreter": {
-   "hash": "9f70ec6e6bd16014ded89c8222361cbe53cd9507d51ebdcdf3ab6e494d45cf74"
+   "hash": "a35bb4b4bceaf970a493ff7351e9d97180ab3fe9951c21e9e29c55a687242182"
   },
   "kernelspec": {
-   "display_name": "zenbytes",
+   "display_name": "Python 3.8.13 64-bit ('zenbytes-latest')",
    "language": "python",
-   "name": "zenbytes"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {

diff --git a/2-2_Local_Deployment.ipynb b/2-2_Local_Deployment.ipynb
@@ -26,7 +26,7 @@
    "outputs": [],
    "source": [
     "%pip install zenml matplotlib\n",
-    "!zenml integration install sklearn mlflow -f\n",
+    "!zenml integration install sklearn mlflow -y\n",
     "%pip install pyparsing==2.4.2  # required for Colab\n",
     "\n",
     "import IPython\n",
@@ -80,9 +80,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zenml.steps import step\n",
-    "from zenml.pipelines import pipeline\n",
     "from zenml.integrations.mlflow.steps import mlflow_model_deployer_step\n",
+    "from zenml.pipelines import pipeline\n",
+    "from zenml.steps import step\n",
     "\n",
     "\n",
     "@step\n",
@@ -134,10 +134,10 @@
     "!zenml stack set default\n",
     "\n",
     "# Define wandb experiment tracker from last lesson\n",
-    "!zenml experiment-tracker register mlflow_tracker --type=mlflow\n",
+    "!zenml experiment-tracker register mlflow_tracker --flavor=mlflow\n",
     "\n",
     "# Register the MLflow model deployer\n",
-    "!zenml model-deployer register mlflow --type=mlflow\n",
+    "!zenml model-deployer register mlflow --flavor=mlflow\n",
     "\n",
     "# Add the MLflow components into our default stack\n",
     "!zenml stack update default -d mlflow -e mlflow_tracker"
@@ -255,10 +255,10 @@
  ],
  "metadata": {
   "interpreter": {
-   "hash": "9f70ec6e6bd16014ded89c8222361cbe53cd9507d51ebdcdf3ab6e494d45cf74"
+   "hash": "a35bb4b4bceaf970a493ff7351e9d97180ab3fe9951c21e9e29c55a687242182"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3.8.13 64-bit ('zenbytes-latest')",
    "language": "python",
    "name": "python3"
   },

diff --git a/2-3_Inference_Pipelines.ipynb b/2-3_Inference_Pipelines.ipynb
@@ -39,10 +39,10 @@
    "outputs": [],
    "source": [
     "%pip install zenml\n",
-    "!zenml integration install sklearn mlflow -f\n",
+    "!zenml integration install sklearn mlflow -y\n",
     "!zenml stack set default\n",
-    "!zenml experiment-tracker register mlflow_tracker --type=mlflow\n",
-    "!zenml model-deployer register mlflow --type=mlflow\n",
+    "!zenml experiment-tracker register mlflow_tracker --flavor=mlflow\n",
+    "!zenml model-deployer register mlflow --flavor=mlflow\n",
     "!zenml stack update default -d mlflow -e mlflow_tracker\n",
     "\n",
     "%pip install pyparsing==2.4.2  # required for Colab\n",
@@ -74,6 +74,13 @@
     "    IN_COLAB = False"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Note:** If you get an error `NoSuchProcess: process no longer exists (pid=...)` when running the following cell, simply run the cell again."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -82,11 +89,11 @@
    "source": [
     "from zenml.integrations.mlflow.steps import mlflow_model_deployer_step\n",
     "\n",
+    "from pipelines.training_pipeline import train_evaluate_deploy_pipeline\n",
+    "from steps.deployment_trigger import deployment_trigger\n",
     "from steps.evaluator import evaluator\n",
     "from steps.importer import importer\n",
     "from steps.mlflow_trainer import svc_trainer_mlflow\n",
-    "from steps.deployment_trigger import deployment_trigger\n",
-    "from pipelines.training_pipeline import train_evaluate_deploy_pipeline\n",
     "\n",
     "train_evaluate_deploy_pipeline(\n",
     "    importer=importer(),\n",
@@ -161,9 +168,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zenml.steps import step, Output\n",
     "from zenml.services import BaseService\n",
     "from zenml.repository import Repository\n",
+    "from zenml.steps import step, Output\n",
     "\n",
     "\n",
     "@step(enable_cache=False)\n",
@@ -241,10 +248,10 @@
  ],
  "metadata": {
   "interpreter": {
-   "hash": "9f70ec6e6bd16014ded89c8222361cbe53cd9507d51ebdcdf3ab6e494d45cf74"
+   "hash": "a35bb4b4bceaf970a493ff7351e9d97180ab3fe9951c21e9e29c55a687242182"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3.8.13 64-bit ('zenbytes-latest')",
    "language": "python",
    "name": "python3"
   },

diff --git a/3-1_Data_Skew.ipynb b/3-1_Data_Skew.ipynb
@@ -33,7 +33,7 @@
    "outputs": [],
    "source": [
     "%pip install zenml\n",
-    "!zenml integration install sklearn mlflow evidently -f\n",
+    "!zenml integration install sklearn mlflow evidently -y\n",
     "%pip install pyparsing==2.4.2  # required for Colab\n",
     "\n",
     "import IPython\n",
@@ -91,8 +91,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from steps.importer import importer\n",
     "from steps.evaluator import evaluator\n",
+    "from steps.importer import importer\n",
     "from steps.sklearn_trainer import svc_trainer"
    ]
   },
@@ -236,10 +236,10 @@
  ],
  "metadata": {
   "interpreter": {
-   "hash": "9f70ec6e6bd16014ded89c8222361cbe53cd9507d51ebdcdf3ab6e494d45cf74"
+   "hash": "a35bb4b4bceaf970a493ff7351e9d97180ab3fe9951c21e9e29c55a687242182"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3.8.13 64-bit ('zenbytes-latest')",
    "language": "python",
    "name": "python3"
   },

diff --git a/4-1_Cloud_Deployment.ipynb b/4-1_Cloud_Deployment.ipynb
@@ -32,7 +32,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!zenml integration install kubeflow seldon s3 aws -f\n",
+    "!zenml integration install sklearn mlflow evidently kubeflow seldon s3 aws -y\n",
     "\n",
     "import IPython\n",
     "\n",
@@ -206,20 +206,20 @@
    "outputs": [],
    "source": [
     "# Register container registry\n",
-    "!zenml container-registry register ecr_registry --type=default --uri={ECR_REGISTRY_NAME}\n",
+    "!zenml container-registry register ecr_registry --flavor=default --uri={ECR_REGISTRY_NAME}\n",
     "\n",
     "# Register orchestrator (Kubeflow on AWS)\n",
-    "!zenml orchestrator register eks_orchestrator --type=kubeflow --kubernetes_context={KUBE_CONTEXT} --synchronous=True\n",
+    "!zenml orchestrator register eks_orchestrator --flavor=kubeflow --kubernetes_context={KUBE_CONTEXT} --synchronous=True\n",
     "\n",
     "# Register metadata store and artifact store\n",
-    "!zenml metadata-store register kubeflow_metadata_store --type=kubeflow\n",
-    "!zenml artifact-store register s3_store --type=s3 --path={S3_BUCKET_NAME}\n",
+    "!zenml metadata-store register kubeflow_metadata_store --flavor=kubeflow\n",
+    "!zenml artifact-store register s3_store --flavor=s3 --path={S3_BUCKET_NAME}\n",
     "\n",
     "# Register the Seldon Core model deployer (Seldon on AWS)\n",
-    "!zenml model-deployer register eks_seldon --type=seldon --kubernetes_context={KUBE_CONTEXT} --kubernetes_namespace={KUBEFLOW_NAMESPACE} --base_url=http://{INGRESS_HOST[0]} --secret=s3_store\n",
+    "!zenml model-deployer register eks_seldon --flavor=seldon --kubernetes_context={KUBE_CONTEXT} --kubernetes_namespace={KUBEFLOW_NAMESPACE} --base_url=http://{INGRESS_HOST[0]} --secret=s3_store\n",
     "\n",
     "# Register a secret manager\n",
-    "!zenml secrets-manager register aws_secret_manager --type=aws\n",
+    "!zenml secrets-manager register aws_secret_manager --flavor=aws\n",
     "\n",
     "# Register the aws_kubeflow_stack\n",
     "!zenml stack register aws_kubeflow_stack -m kubeflow_metadata_store -a s3_store -o eks_orchestrator -c ecr_registry -d eks_seldon -x aws_secret_manager"
@@ -371,16 +371,16 @@
    "outputs": [],
    "source": [
     "!zenml served-models delete <UUID-of-model-server>  # see output of `zenml served-models list` above\n",
-    "!zenml stack down -f"
+    "!zenml stack down -y"
    ]
   }
  ],
  "metadata": {
   "interpreter": {
-   "hash": "9f70ec6e6bd16014ded89c8222361cbe53cd9507d51ebdcdf3ab6e494d45cf74"
+   "hash": "a35bb4b4bceaf970a493ff7351e9d97180ab3fe9951c21e9e29c55a687242182"
   },
   "kernelspec": {
-   "display_name": "Python 3.8.13 64-bit ('zenbytes')",
+   "display_name": "Python 3.8.13 64-bit ('zenbytes-latest')",
    "language": "python",
    "name": "python3"
   },

diff --git a/README.md b/README.md
@@ -86,7 +86,7 @@ but we recommend you install all integrations ahead of time with the
 following command:
 
 ```bash
-zenml integration install sklearn dash wandb evidently mlflow kubeflow seldon s3 aws -f
+zenml integration install sklearn dash wandb evidently mlflow kubeflow seldon s3 aws -y
 ```
 
 ### Additional Requirements

diff --git a/pipelines/inference_pipeline.py b/pipelines/inference_pipeline.py
@@ -1,12 +1,9 @@
 from zenml.pipelines import pipeline
 
-# Path to a pip requirements file that contains requirements necessary to run
-# the pipeline
-
 
 @pipeline(
     enable_cache=False,
-    requirements_file="../requirements.txt",
+    requirements=["zenml"],
     required_integrations=["seldon", "mlflow", "evidently"],
 )
 def inference_pipeline(

diff --git a/pipelines/training_pipeline.py b/pipelines/training_pipeline.py
@@ -19,7 +19,7 @@ def train_evaluate_deploy_pipeline(
 
 @pipeline(
     enable_cache=False,
-    requirements_file="../requirements.txt",
+    requirements=["zenml"],
     required_integrations=["seldon", "mlflow", "evidently"],
 )
 def continuous_deployment_pipeline(