technical_testbench.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Installation procedure"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### turn on autoreloading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%reload_ext autoreload "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### install ray library\n",
    "uncomment this to run it once on your machine, then recomment it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting ray[default]\n",
      "  Downloading ray-1.6.0-cp38-cp38-win_amd64.whl (15.6 MB)\n",
      "Collecting grpcio>=1.28.1\n",
      "  Using cached grpcio-1.39.0-cp38-cp38-win_amd64.whl (3.2 MB)\n",
      "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from ray[default]) (1.0.2)\n",
      "Requirement already satisfied: click>=7.0 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from ray[default]) (7.1.2)\n",
      "Requirement already satisfied: pyyaml in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from ray[default]) (5.4.1)\n",
      "Requirement already satisfied: attrs in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from ray[default]) (21.2.0)\n",
      "Requirement already satisfied: filelock in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from ray[default]) (3.0.12)\n",
      "Requirement already satisfied: protobuf>=3.15.3 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from ray[default]) (3.17.2)\n",
      "Collecting redis>=3.5.0\n",
      "  Using cached redis-3.5.3-py2.py3-none-any.whl (72 kB)\n",
      "Requirement already satisfied: numpy>=1.16 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from ray[default]) (1.20.3)\n",
      "Requirement already satisfied: prometheus-client>=0.7.1 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from ray[default]) (0.11.0)\n",
      "Collecting aiohttp-cors\n",
      "  Using cached aiohttp_cors-0.7.0-py3-none-any.whl (27 kB)\n",
      "Collecting aioredis<2\n",
      "  Using cached aioredis-1.3.1-py3-none-any.whl (65 kB)\n",
      "Requirement already satisfied: requests in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from ray[default]) (2.25.1)\n",
      "Collecting opencensus\n",
      "  Using cached opencensus-0.7.13-py2.py3-none-any.whl (127 kB)\n",
      "Collecting py-spy>=0.2.0\n",
      "  Using cached py_spy-0.3.8-py2.py3-none-win_amd64.whl (1.4 MB)\n",
      "Collecting aiohttp\n",
      "  Using cached aiohttp-3.7.4.post0-cp38-cp38-win_amd64.whl (635 kB)\n",
      "Requirement already satisfied: jsonschema in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from ray[default]) (3.2.0)\n",
      "Collecting gpustat\n",
      "  Using cached gpustat-0.6.0-py3-none-any.whl\n",
      "Collecting colorful\n",
      "  Using cached colorful-0.5.4-py2.py3-none-any.whl (201 kB)\n",
      "Collecting hiredis\n",
      "  Using cached hiredis-2.0.0-cp38-cp38-win_amd64.whl (18 kB)\n",
      "Collecting async-timeout\n",
      "  Using cached async_timeout-3.0.1-py3-none-any.whl (8.2 kB)\n",
      "Requirement already satisfied: six>=1.5.2 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from grpcio>=1.28.1->ray[default]) (1.16.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.5 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from aiohttp->ray[default]) (3.10.0.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from aiohttp->ray[default]) (1.5.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from aiohttp->ray[default]) (5.1.0)\n",
      "Requirement already satisfied: chardet<5.0,>=2.0 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from aiohttp->ray[default]) (4.0.0)\n",
      "Requirement already satisfied: idna>=2.0 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from yarl<2.0,>=1.0->aiohttp->ray[default]) (2.10)\n",
      "Requirement already satisfied: colorama in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from colorful->ray[default]) (0.4.4)\n",
      "Requirement already satisfied: psutil in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from gpustat->ray[default]) (5.8.0)\n",
      "Collecting blessings>=1.6\n",
      "  Using cached blessings-1.7-py3-none-any.whl (18 kB)\n",
      "Collecting nvidia-ml-py3>=7.352.0\n",
      "  Using cached nvidia_ml_py3-7.352.0-py3-none-any.whl\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from jsonschema->ray[default]) (0.17.3)\n",
      "Requirement already satisfied: setuptools in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from jsonschema->ray[default]) (52.0.0.post20210125)\n",
      "Requirement already satisfied: google-api-core<2.0.0,>=1.0.0 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from opencensus->ray[default]) (1.25.1)\n",
      "Collecting opencensus-context==0.1.2\n",
      "  Using cached opencensus_context-0.1.2-py2.py3-none-any.whl (4.4 kB)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (1.53.0)\n",
      "Requirement already satisfied: pytz in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (2021.1)\n",
      "Requirement already satisfied: google-auth<2.0dev,>=1.21.1 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (1.21.3)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (4.2.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (4.7.2)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (0.4.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from requests->ray[default]) (2021.5.30)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\mclapp\\anaconda3\\envs\\fastai\\lib\\site-packages (from requests->ray[default]) (1.26.6)\n",
      "Installing collected packages: async-timeout, redis, opencensus-context, nvidia-ml-py3, hiredis, grpcio, blessings, aiohttp, ray, py-spy, opencensus, gpustat, colorful, aioredis, aiohttp-cors\n",
      "Successfully installed aiohttp-3.7.4.post0 aiohttp-cors-0.7.0 aioredis-1.3.1 async-timeout-3.0.1 blessings-1.7 colorful-0.5.4 gpustat-0.6.0 grpcio-1.39.0 hiredis-2.0.0 nvidia-ml-py3-7.352.0 opencensus-0.7.13 opencensus-context-0.1.2 py-spy-0.3.8 ray-1.6.0 redis-3.5.3\n"
     ]
    }
   ],
   "source": [
    "#!pip install -U \"ray[default]\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### start ray server\n",
    "\n",
    "you only need to run this once per jupyter session."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#MAC USERS: \n",
    "\n",
    "#Before running ipynb it is recommended to run this cell:\n",
    "!ray stop #command line "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-09-28 03:21:09,466\tINFO scripts.py:588 -- Local node IP: 192.168.1.208\n",
      "2021-09-28 03:21:18,407\tSUCC scripts.py:627 -- --------------------\n",
      "2021-09-28 03:21:18,407\tSUCC scripts.py:628 -- Ray runtime started.\n",
      "2021-09-28 03:21:18,407\tSUCC scripts.py:629 -- --------------------\n",
      "2021-09-28 03:21:18,407\tINFO scripts.py:631 -- Next steps\n",
      "2021-09-28 03:21:18,407\tINFO scripts.py:632 -- To connect to this Ray runtime from another node, run\n",
      "2021-09-28 03:21:18,407\tINFO scripts.py:636 --   ray start --address='192.168.1.208:6379' --redis-password='cbgt2'\n",
      "2021-09-28 03:21:18,407\tINFO scripts.py:641 -- Alternatively, use the following Python code:\n",
      "2021-09-28 03:21:18,408\tINFO scripts.py:644 -- import ray\n",
      "2021-09-28 03:21:18,408\tINFO scripts.py:645 -- ray.init(address='auto', _redis_password='cbgt2')\n",
      "2021-09-28 03:21:18,408\tINFO scripts.py:653 -- If connection fails, check your firewall settings and network configuration.\n",
      "2021-09-28 03:21:18,408\tINFO scripts.py:658 -- To terminate the Ray runtime, run\n",
      "2021-09-28 03:21:18,408\tINFO scripts.py:659 --   ray stop\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-09-28 03:21:16,186\tINFO services.py:1245 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n"
     ]
    }
   ],
   "source": [
    "#ALL THE USERS:\n",
    "\n",
    "#to obtain the local ip node, run:\n",
    "!ray start --head --port=6379 --redis-password=\"cbgt2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#use the local ip node obtained and run:\n",
    "!ray start --address='local ip node:6379' --redis-password='cbgt2' #command line "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#if it doesn't work, shut down the ray server and start it again\n",
    "#if some ray processes are still opened, run:\n",
    "!ray stop --force #command line "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### install autopep8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install --upgrade autopep8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### how to run autopep8\n",
    "\n",
    "uncomment and run this on any code files you create before pushing to github"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Example on how to use it to prevent indentation problems:\n",
    "#!autopep8 --in-place --aggressive --aggressive <filename>.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. How to create pipelines "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### import the core cbgt code and any of your files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-25 09:09:04,045\tINFO worker.py:800 -- Connecting to existing Ray cluster at address: 192.168.1.201:6379\n"
     ]
    }
   ],
   "source": [
    "import cbgt as cbgt\n",
    "#add import statement here for any files you make\n",
    "#Example: \n",
    "#from tracetype import *\n",
    "#import init_params as par "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create an empty pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pl = cbgt.Pipeline()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### basic variable assignment\n",
    "\n",
    "Use `pipeline.variablename` to write to a specific variable.\n",
    "\n",
    "Current two kinds of basic assignment are permitted:\n",
    "\n",
    "- assigning a \"constant\" (which can be a whole python expression but it's constant in that the value is calculated immediately and not during pipeline execution)\n",
    "- copying a value from another pipeline variable\n",
    "\n",
    "More is probably on the way"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pl.hello = \"Hello World!\"\n",
    "pl.world = pl.hello"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<backend.VariablePlaceholder at 0x1e9593e3670>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# these are special objects, not ordinary object properties\n",
    "pl.hello"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<backend.BasicAssignmentModule at 0x1e9593e3700>,\n",
       " <backend.BasicCopyModule at 0x1e9593e39a0>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#to check which kind of assignements we have introduced into the pipeline:\n",
    "pl.modulelist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### \"code block\" modules\n",
    "\n",
    "A \"code block\" can be created in the form of a function that takes \"self\" as its only argument. (You can technically use any variable name.)\n",
    "\n",
    "Python doesn't support true code blocks unlike some other languages.\n",
    "\n",
    "Use `pipeline.add(yourfunction)` to add a code block."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def helper1(number):\n",
    "    # this function is defined in a \"normal way\" outside the context of the pipeline.\n",
    "    # this function be basically anything, it's just to demonstrate that outside functions\n",
    "    # can be used in code block modules\n",
    "    return (number+1, number+2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def codeblock(self):\n",
    "    # you can do pretty much anything you want in here\n",
    "    self.y, self.z = helper1(self.x)\n",
    "    self.x += 1234567\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<backend.Pipeline at 0x1e9593dd700>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#add codeblock to the pipeline:\n",
    "pl.add(codeblock)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<backend.BasicAssignmentModule at 0x1e9593e3700>,\n",
       " <backend.BasicCopyModule at 0x1e9593e39a0>,\n",
       " <backend.CodeTaskFunctionModule at 0x1ea9d97e8b0>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#to check we have introduced the codeblock into the pipeline:\n",
    "pl.modulelist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### function modules\n",
    "\n",
    "Functions can be transformed into modules by specifying their inputs and outputs.  There is a special syntax for doing this.\n",
    "\n",
    "`pipeline.output = pipeline[function](arguments=whatever)`\n",
    "\n",
    "If a function has multiple outputs (a tuple) then use the `.shape()` function to change the output into the needed shape. This function takes a length (or list of lengths for nested tuples) as input. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def helper2(first,second):\n",
    "    # function with 1 output and 2 inputs\n",
    "    return \"\".join([first,second])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Without using a pipeline, we use to call function in this way:\n",
    "#joined = helper2(hello, second=\"!!!!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using a pipeline:\n",
    "pl.joined = pl[helper2](pl.hello, second=\"!!!!\")\n",
    "#arguments can be passed either by position or name, just like when you call the function normally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<backend.BasicAssignmentModule at 0x1e9593e3700>,\n",
       " <backend.BasicCopyModule at 0x1e9593e39a0>,\n",
       " <backend.CodeTaskFunctionModule at 0x1ea9d97e8b0>,\n",
       " <backend.FunctionModule at 0x1ea9d97ed60>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#to check we have introduced the function into the pipeline:\n",
    "pl.modulelist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you can pass in external values:\n",
    "pl.a,pl.b = pl[helper1](10000).shape(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<backend.BasicAssignmentModule at 0x1e9593e3700>,\n",
       " <backend.BasicCopyModule at 0x1e9593e39a0>,\n",
       " <backend.CodeTaskFunctionModule at 0x1ea9d97e8b0>,\n",
       " <backend.FunctionModule at 0x1ea9d97ed60>,\n",
       " <backend.FunctionModule at 0x1ea9d97ec10>]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#to check we have introduced a second function into the pipeline:\n",
    "pl.modulelist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### pipeline composition\n",
    "\n",
    "You can build up a pipeline and then `.add(...)` it to another pipeline.\n",
    "\n",
    "You can use multiple `.add()` in one line."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Defining a new function:\n",
    "\n",
    "def countingblock(self):\n",
    "    try:\n",
    "        self.counter += 1\n",
    "    except:\n",
    "        self.counter = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating another pipeline, containing the new function: \n",
    "anotherpipeline = cbgt.Pipeline().add(countingblock)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#You can create another pipeline, introducing more than 1 function (or the same function) at the same time:\n",
    "anotherpipeline = cbgt.Pipeline().add(countingblock).add(countingblock).add(countingblock)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<backend.CodeTaskFunctionModule at 0x1ea9d97ebe0>,\n",
       " <backend.CodeTaskFunctionModule at 0x1ea9d97ea30>,\n",
       " <backend.CodeTaskFunctionModule at 0x1ea9d97ebb0>]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#to check we have well-created the new pipeline:\n",
    "anotherpipeline.modulelist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<backend.Pipeline at 0x1e9593dd700>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Adding the new pipeline ('anotherpipeline') to the previous one ('pl'):\n",
    "pl.add(anotherpipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<backend.BasicAssignmentModule at 0x1e9593e3700>,\n",
       " <backend.BasicCopyModule at 0x1e9593e39a0>,\n",
       " <backend.CodeTaskFunctionModule at 0x1ea9d97e8b0>,\n",
       " <backend.FunctionModule at 0x1ea9d97ed60>,\n",
       " <backend.FunctionModule at 0x1ea9d97ec10>,\n",
       " <backend.PipelineModule at 0x1ea9d97e7c0>]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#To check we have added correctly 'anotherpipeline' to 'pl':\n",
    "pl.modulelist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. How to run a pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### running a pipeline\n",
    "\n",
    "1. You can optionally create a dictionary of variable values as the initial state of the pipeline\n",
    "\n",
    "2. You then use `executionmanager.run(pipeline,...)` which returns a new dictionary of the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create a dictionary:\n",
    "environment = {\n",
    "    'x': 100,    #fill in with whatever you need as the starting variables\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Execute the pipeline: \n",
    "results = cbgt.ExecutionManager(cores=7).run(pl,environment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'x': 1234667,\n",
       " 'hello': 'Hello World!',\n",
       " 'world': 'Hello World!',\n",
       " 'y': 101,\n",
       " 'z': 102,\n",
       " 'joined': 'Hello World!!!!!',\n",
       " 'a': 10001,\n",
       " 'b': 10002,\n",
       " 'counter': 3}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Print the obtained results: \n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "locals().update(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-26 01:52:51,219\tERROR import_thread.py:88 -- ImportThread: Error while reading from socket: (10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None)\n",
      "2021-08-26 01:52:51,241\tERROR worker.py:468 -- print_logs: Error while reading from socket: (10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None)\n",
      "2021-08-26 01:52:51,242\tERROR worker.py:1191 -- listen_error_messages_raylet: Error while reading from socket: (10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None)\n"
     ]
    }
   ],
   "source": [
    "counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}