notebooks/evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "## Imports"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3rxTYRroO2Rt",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "f4f1c0e6-63e3-4458-c519-c971e915b736"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Collecting openai\n",
      "  Downloading openai-0.28.1-py3-none-any.whl (76 kB)\n",
      "\u001B[?25l     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m0.0/77.0 kB\u001B[0m \u001B[31m?\u001B[0m eta \u001B[36m-:--:--\u001B[0m\r\u001B[2K     \u001B[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[90m╺\u001B[0m\u001B[90m━━\u001B[0m \u001B[32m71.7/77.0 kB\u001B[0m \u001B[31m2.4 MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\r\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m77.0/77.0 kB\u001B[0m \u001B[31m2.0 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai) (2.31.0)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.1)\n",
      "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai) (3.8.6)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (2.0.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (2023.7.22)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (23.1.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (6.0.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (4.0.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (1.3.1)\n",
      "Installing collected packages: openai\n",
      "Successfully installed openai-0.28.1\n",
      "Collecting tiktoken\n",
      "  Downloading tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m2.0/2.0 MB\u001B[0m \u001B[31m23.8 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
      "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2.31.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2.0.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2023.7.22)\n",
      "Installing collected packages: tiktoken\n",
      "Successfully installed tiktoken-0.5.1\n",
      "Collecting langchain\n",
      "  Downloading langchain-0.0.314-py3-none-any.whl (1.9 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.9/1.9 MB\u001B[0m \u001B[31m22.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.21)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.8.6)\n",
      "Requirement already satisfied: anyio<4.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.7.1)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n",
      "Collecting dataclasses-json<0.7,>=0.5.7 (from langchain)\n",
      "  Downloading dataclasses_json-0.6.1-py3-none-any.whl (27 kB)\n",
      "Collecting jsonpatch<2.0,>=1.33 (from langchain)\n",
      "  Downloading jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n",
      "Collecting langsmith<0.1.0,>=0.0.43 (from langchain)\n",
      "  Downloading langsmith-0.0.43-py3-none-any.whl (40 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m40.0/40.0 kB\u001B[0m \u001B[31m5.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.23.5)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.10.13)\n",
      "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.3)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (3.3.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0->langchain) (3.4)\n",
      "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0->langchain) (1.3.0)\n",
      "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4.0->langchain) (1.1.3)\n",
      "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.7,>=0.5.7->langchain)\n",
      "  Downloading marshmallow-3.20.1-py3-none-any.whl (49 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m49.4/49.4 kB\u001B[0m \u001B[31m6.1 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.7,>=0.5.7->langchain)\n",
      "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
      "Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain)\n",
      "  Downloading jsonpointer-2.4-py2.py3-none-any.whl (7.8 kB)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (4.5.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2.0.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2023.7.22)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.0)\n",
      "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json<0.7,>=0.5.7->langchain) (23.2)\n",
      "Collecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain)\n",
      "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
      "Installing collected packages: mypy-extensions, marshmallow, jsonpointer, typing-inspect, langsmith, jsonpatch, dataclasses-json, langchain\n",
      "Successfully installed dataclasses-json-0.6.1 jsonpatch-1.33 jsonpointer-2.4 langchain-0.0.314 langsmith-0.0.43 marshmallow-3.20.1 mypy-extensions-1.0.0 typing-inspect-0.9.0\n",
      "Collecting sentence-transformers\n",
      "  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m86.0/86.0 kB\u001B[0m \u001B[31m2.2 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25h  Preparing metadata (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "Collecting transformers<5.0.0,>=4.6.0 (from sentence-transformers)\n",
      "  Downloading transformers-4.34.0-py3-none-any.whl (7.7 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m7.7/7.7 MB\u001B[0m \u001B[31m70.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.66.1)\n",
      "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (2.0.1+cu118)\n",
      "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (0.15.2+cu118)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.23.5)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.2.2)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.11.3)\n",
      "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (3.8.1)\n",
      "Collecting sentencepiece (from sentence-transformers)\n",
      "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.3/1.3 MB\u001B[0m \u001B[31m91.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting huggingface-hub>=0.4.0 (from sentence-transformers)\n",
      "  Downloading huggingface_hub-0.18.0-py3-none-any.whl (301 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m302.0/302.0 kB\u001B[0m \u001B[31m25.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers) (3.12.4)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers) (2023.6.0)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers) (2.31.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers) (6.0.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers) (4.5.0)\n",
      "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers) (23.2)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers) (3.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers) (3.1.2)\n",
      "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers) (2.0.0)\n",
      "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence-transformers) (3.27.6)\n",
      "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence-transformers) (17.0.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (2023.6.3)\n",
      "Collecting tokenizers<0.15,>=0.14 (from transformers<5.0.0,>=4.6.0->sentence-transformers)\n",
      "  Downloading tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m3.8/3.8 MB\u001B[0m \u001B[31m108.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting safetensors>=0.3.1 (from transformers<5.0.0,>=4.6.0->sentence-transformers)\n",
      "  Downloading safetensors-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.3/1.3 MB\u001B[0m \u001B[31m91.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->sentence-transformers) (8.1.7)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->sentence-transformers) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers) (3.2.0)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->sentence-transformers) (9.4.0)\n",
      "Collecting huggingface-hub>=0.4.0 (from sentence-transformers)\n",
      "  Downloading huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m295.0/295.0 kB\u001B[0m \u001B[31m33.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->sentence-transformers) (2.1.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (2.0.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (2023.7.22)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->sentence-transformers) (1.3.0)\n",
      "Building wheels for collected packages: sentence-transformers\n",
      "  Building wheel for sentence-transformers (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "  Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125923 sha256=5ae6a6fe38ac9d6de09127f33fb01a24f83cca5d20ae350ecc3bab1a5cc5b326\n",
      "  Stored in directory: /root/.cache/pip/wheels/62/f2/10/1e606fd5f02395388f74e7462910fe851042f97238cbbd902f\n",
      "Successfully built sentence-transformers\n",
      "Installing collected packages: sentencepiece, safetensors, huggingface-hub, tokenizers, transformers, sentence-transformers\n",
      "Successfully installed huggingface-hub-0.17.3 safetensors-0.4.0 sentence-transformers-2.2.2 sentencepiece-0.1.99 tokenizers-0.14.1 transformers-4.34.0\n",
      "Collecting accelerate\n",
      "  Downloading accelerate-0.23.0-py3-none-any.whl (258 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m258.1/258.1 kB\u001B[0m \u001B[31m5.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.23.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.2)\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
      "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.0.1+cu118)\n",
      "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.17.3)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.12.4)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.5.0)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
      "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.0.0)\n",
      "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (3.27.6)\n",
      "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (17.0.2)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (2023.6.0)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (4.66.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2.0.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2023.7.22)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
      "Installing collected packages: accelerate\n",
      "Successfully installed accelerate-0.23.0\n",
      "Collecting einops\n",
      "  Downloading einops-0.7.0-py3-none-any.whl (44 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m44.6/44.6 kB\u001B[0m \u001B[31m1.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hInstalling collected packages: einops\n",
      "Successfully installed einops-0.7.0\n",
      "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.34.0)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.4)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.17.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.15,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.14.1)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (2023.6.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.5.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
      "Collecting xformers\n",
      "  Downloading xformers-0.0.22.post4-cp310-cp310-manylinux2014_x86_64.whl (211.8 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m211.8/211.8 MB\u001B[0m \u001B[31m2.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from xformers) (1.23.5)\n",
      "Collecting torch==2.1.0 (from xformers)\n",
      "  Downloading torch-2.1.0-cp310-cp310-manylinux1_x86_64.whl (670.2 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m670.2/670.2 MB\u001B[0m \u001B[31m2.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch==2.1.0->xformers) (3.12.4)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch==2.1.0->xformers) (4.5.0)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch==2.1.0->xformers) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch==2.1.0->xformers) (3.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch==2.1.0->xformers) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch==2.1.0->xformers) (2023.6.0)\n",
      "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m23.7/23.7 MB\u001B[0m \u001B[31m62.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m823.6/823.6 kB\u001B[0m \u001B[31m65.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m14.1/14.1 MB\u001B[0m \u001B[31m97.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m731.7/731.7 MB\u001B[0m \u001B[31m2.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m410.6/410.6 MB\u001B[0m \u001B[31m3.1 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m121.6/121.6 MB\u001B[0m \u001B[31m8.1 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m56.5/56.5 MB\u001B[0m \u001B[31m12.8 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m124.2/124.2 MB\u001B[0m \u001B[31m8.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m196.0/196.0 MB\u001B[0m \u001B[31m3.0 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-nccl-cu12==2.18.1 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m209.8/209.8 MB\u001B[0m \u001B[31m6.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch==2.1.0->xformers)\n",
      "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m99.1/99.1 kB\u001B[0m \u001B[31m12.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting triton==2.1.0 (from torch==2.1.0->xformers)\n",
      "  Downloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89.2 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m89.2/89.2 MB\u001B[0m \u001B[31m9.0 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.0->xformers)\n",
      "  Downloading nvidia_nvjitlink_cu12-12.2.140-py3-none-manylinux1_x86_64.whl (20.2 MB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m20.2/20.2 MB\u001B[0m \u001B[31m62.0 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch==2.1.0->xformers) (2.1.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch==2.1.0->xformers) (1.3.0)\n",
      "Installing collected packages: triton, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch, xformers\n",
      "  Attempting uninstall: triton\n",
      "    Found existing installation: triton 2.0.0\n",
      "    Uninstalling triton-2.0.0:\n",
      "      Successfully uninstalled triton-2.0.0\n",
      "  Attempting uninstall: torch\n",
      "    Found existing installation: torch 2.0.1+cu118\n",
      "    Uninstalling torch-2.0.1+cu118:\n",
      "      Successfully uninstalled torch-2.0.1+cu118\n",
      "\u001B[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "fastai 2.7.12 requires torch<2.1,>=1.7, but you have torch 2.1.0 which is incompatible.\n",
      "torchaudio 2.0.2+cu118 requires torch==2.0.1, but you have torch 2.1.0 which is incompatible.\n",
      "torchdata 0.6.1 requires torch==2.0.1, but you have torch 2.1.0 which is incompatible.\n",
      "torchtext 0.15.2 requires torch==2.0.1, but you have torch 2.1.0 which is incompatible.\n",
      "torchvision 0.15.2+cu118 requires torch==2.0.1, but you have torch 2.1.0 which is incompatible.\u001B[0m\u001B[31m\n",
      "\u001B[0mSuccessfully installed nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.18.1 nvidia-nvjitlink-cu12-12.2.140 nvidia-nvtx-cu12-12.1.105 torch-2.1.0 triton-2.1.0 xformers-0.0.22.post4\n",
      "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.10/dist-packages (0.17.3)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (3.12.4)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2023.6.0)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.66.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (6.0.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.5.0)\n",
      "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (23.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2.0.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2023.7.22)\n"
     ]
    }
   ],
   "source": [
    "!pip install openai\n",
    "!pip install tiktoken\n",
    "!pip install langchain\n",
    "!pip install -U sentence-transformers\n",
    "!pip install accelerate\n",
    "!pip install einops\n",
    "!pip install transformers\n",
    "!pip install xformers\n",
    "!pip install huggingface_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WkkFNJjnKrH5",
    "outputId": "04438ba8-346a-4b7b-e010-4d654b143ab5"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "import pickle\n",
    "from langchain import PromptTemplate, LLMChain\n",
    "from langchain.prompts.few_shot import FewShotPromptTemplate\n",
    "from langchain.prompts.prompt import PromptTemplate\n",
    "from langchain import HuggingFacePipeline, HuggingFaceHub\n",
    "import os\n",
    "import openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RnDSeyN3nJAb"
   },
   "outputs": [],
   "source": [
    "GPT_3_KEY = ''\n",
    "GPT_4_KEY = ''\n",
    "OPENAI_KEY = GPT_4_KEY\n",
    "HF_API_KEY = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "z8JkxCSVnwAM"
   },
   "outputs": [],
   "source": [
    "os.environ[\"OPENAI_API_KEY\"] = OPENAI_KEY\n",
    "os.environ[\"HUGGINGFACEHUB_API_TOKEN\"] = HF_API_KEY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aCkns9-YqPh7"
   },
   "outputs": [],
   "source": [
    "# Set openai.api_key to the OPENAI environment variable\n",
    "openai.api_key = os.environ[\"OPENAI_API_KEY\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "edAtkkBsqpLp"
   },
   "source": [
    "### API Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "p6YcrFj7nXqb",
    "outputId": "9a584775-b4ee-4c12-8771-0008b1867dff"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Hello! How can I assist you today?\n"
     ]
    }
   ],
   "source": [
    "# an example question about the 2022 Olympics\n",
    "query = 'Hi?'\n",
    "\n",
    "response = openai.ChatCompletion.create(\n",
    "    messages=[\n",
    "        {'role': 'system', 'content': ''},\n",
    "        {'role': 'user', 'content': query},\n",
    "    ],\n",
    "    model=\"gpt-3.5-turbo\",\n",
    "    temperature=0,\n",
    ")\n",
    "\n",
    "print(response['choices'][0]['message']['content'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wKub7RNWcfJi"
   },
   "source": [
    "## Environment Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "leVadleEL_c1"
   },
   "outputs": [],
   "source": [
    "# models\n",
    "EMBEDDING_MODEL = 'all-MiniLM-L6-v2'\n",
    "GPT_3 = \"gpt-3.5-turbo\"\n",
    "GPT_4 = \"gpt-4\"\n",
    "LOCAL_MODEL = \"DrugGPT\"\n",
    "# Use local embedding model?\n",
    "USELOCALEMBD = True\n",
    "# Use local model?\n",
    "USELOCALMODEL = False\n",
    "# Use langchain model or build your own from transformers library?\n",
    "USELANGCHAINMODEL = True\n",
    "# Use huggingface model? RECOMMNAD, OTHERWISE IT\"S TOO SLOW\n",
    "USEHFAPI = False\n",
    "# Use langchain openAI model or directly use openAI api\n",
    "USEOPENAI = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8x7-Qw5TL_wo"
   },
   "outputs": [],
   "source": [
    "# Define the slice size\n",
    "SLICE_THRESHOLD = 5000\n",
    "# Define the size of the evaluation set\n",
    "EVALUATION_SIZE = 1000\n",
    "# Define the smaller size of the evaluation set\n",
    "EVALUATION_SIZE_SMALL = 500"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_oWSdeO7kHzd"
   },
   "source": [
    "#### Data Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "F6Z0iZOICa1A"
   },
   "outputs": [],
   "source": [
    "data_paths_dict = {\n",
    "    'pubmedqa': {\n",
    "        'type': 'binary',\n",
    "        'data': '',\n",
    "        'answer': '',\n",
    "        'GPT_3_answer': '',\n",
    "        'GPT_4_answer': ''\n",
    "    },\n",
    "    'ade': {\n",
    "        'type': 'text',\n",
    "        'data': '',\n",
    "        'answer': '',\n",
    "        'GPT_3_answer': '',\n",
    "        'GPT_4_answer': ''\n",
    "    },\n",
    "    'chatDoctor': {\n",
    "        'type': 'text',\n",
    "        'data': '',\n",
    "        'answer': '',\n",
    "        'GPT_3_answer': '',\n",
    "        'GPT_4_answer': ''\n",
    "    },\n",
    "    'DDI_binary': {\n",
    "        'type': 'binary',\n",
    "        'data': '',\n",
    "        'answer': '',\n",
    "        'GPT_3_answer': '',\n",
    "        'GPT_4_answer': ''\n",
    "    },\n",
    "    'drug_usage': {\n",
    "        'type': 'text',\n",
    "        'data': '',\n",
    "        'answer': '',\n",
    "        'GPT_3_answer': '',\n",
    "        'GPT_4_answer': ''\n",
    "    },\n",
    "    'medmcqa': {\n",
    "        'type': 'mc',\n",
    "        'data': '',\n",
    "        'answer': '',\n",
    "        'GPT_3_answer': '',\n",
    "        'GPT_4_answer': ''\n",
    "    },\n",
    "    'mmlu_mc': {\n",
    "        'type': 'mc',\n",
    "        'data': '',\n",
    "        'answer': '',\n",
    "        'GPT_3_answer': '',\n",
    "        'GPT_4_answer': ''\n",
    "    },\n",
    "    'usmle_mc': {\n",
    "        'type': 'mc',\n",
    "        'data': '',\n",
    "        'answer': '',\n",
    "        'GPT_3_answer': '',\n",
    "        'GPT_4_answer': ''\n",
    "    },\n",
    "    'moderna_interactions': {\n",
    "        'type': 'binary',\n",
    "        'data': '',\n",
    "        'answer': '',\n",
    "        'GPT_3_answer': '',\n",
    "        'GPT_4_answer': ''\n",
    "    }\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_9HCYiAWulMb",
    "outputId": "a454420f-c204-439e-8c2a-96531abeb7da"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Device: cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f\"Device: {device}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lgfb_X6upCfa"
   },
   "source": [
    "## Local Model\n",
    "\n",
    "1. model: Specifies the model to be used for text generation. In this case, we will use 'DrugGPT' for DrugGPT evaluation.\n",
    "\n",
    "2. tokenizer = AutoTokenizer.from_pretrained(model): Loads the tokenizer associated with the model. A tokenizer is responsible for converting input text into a format (like a list of tokens) that the model can understand.\n",
    "\n",
    "4. torch_dtype=torch.bfloat16: This is the data type of the torch tensors used in the pipeline. Using bfloat16 instead of the default float32 can reduce memory usage and computation time, at the expense of some numerical precision.\n",
    "\n",
    "5. trust_remote_code=True: This allows execution of code from the remote model, which is necessary if the model includes custom post-processing.\n",
    "\n",
    "6. device_map=\"auto\": This allows the model to run on the best available device (either a GPU or CPU).\n",
    "\n",
    "7. max_length=200: This specifies the maximum length of the generated text.\n",
    "\n",
    "8. do_sample=True: This means that the generated text will be sampled from the model's output distribution rather than just taking the most probable token at each step. This makes the output more diverse and creative.\n",
    "\n",
    "9. top_k=10: This parameter is used for the \"top-k sampling\" strategy, which only considers the top k most probable tokens at each step.\n",
    "\n",
    "10. num_return_sequences=1: This means that the model will return one generated sequence.\n",
    "\n",
    "11. eos_token_id=tokenizer.eos_token_id: The id of the End of Sentence (EOS) token. The generation process will stop if the model generates this token."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VM0tN3QcY94Y"
   },
   "source": [
    "#### Save Local Model （We use Langchain instead）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mWmK-Od_h4CC"
   },
   "outputs": [],
   "source": [
    "if not USELANGCHAINMODEL:\n",
    "  print(\"We are using our own local model\")\n",
    "  # Save the pipeline\n",
    "  if os.path.isfile(local_model_path):\n",
    "    with open(local_model_path, 'rb') as f:\n",
    "      pipeline = pickle.load(f)\n",
    "  else:\n",
    "    tokenizer = AutoTokenizer.from_pretrained(LOCAL_MODEL)\n",
    "    model = AutoModelForCausalLM.from_pretrained(LOCAL_MODEL, trust_remote_code=True)\n",
    "    pipeline = transformers.pipeline(\n",
    "        \"text-generation\",\n",
    "        model=model,\n",
    "        tokenizer=tokenizer,\n",
    "        torch_dtype=torch.bfloat16,\n",
    "        trust_remote_code=True,\n",
    "        device_map=\"auto\",\n",
    "    )\n",
    "\n",
    "    with open(local_model_path, 'wb') as f:\n",
    "        pickle.dump(pipeline, f)\n",
    "\n",
    "  sequences = pipeline(\n",
    "    \"Tell me a joke\",\n",
    "      max_length=200,\n",
    "      do_sample=True,\n",
    "      top_k=10,\n",
    "      num_return_sequences=1,\n",
    "      eos_token_id=tokenizer.eos_token_id,\n",
    "  )\n",
    "  for seq in sequences:\n",
    "      print(f\"Result: {seq['generated_text']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ONONkNGpY5cx"
   },
   "source": [
    "### Langchain LLM Local Model or HF API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 355,
     "referenced_widgets": [
      "b78ab00b256e431f94113ac7f17a9f35",
      "3359b345f16a46f3993502eb61ed5348",
      "21507ac6372f42f6b7b194cc709bbc3e",
      "be25bd8f9c8843b98b79c25177f6bbbd",
      "1d5a445e405849bdad5e2faeddf8cbd6",
      "f48e1c96f9f7483d9b531c0ebe0ee9a0",
      "c709306d5725466ebbd4bf682e7aa06d",
      "8d57df5df31746eba7f7bdf0fcb4dd84",
      "79591c8fb8014aea94d33f1347a9915f",
      "346d121caddf4c3ea56c43c13ef73cd7",
      "f68f06f4e6374f0ea71b5fbb7c4e2951",
      "e90d653937e44e29a44c5e98e141f692",
      "de560f0edf5e40c0ac487bf22433ea91",
      "33038e684d3c47dfa9b9a278e9cde393",
      "4d14fafcbec74844a2f4f2111b7ced83",
      "e41bad1821844add8f0a2d342b2568dd",
      "216e741c53a64e65ad64cbbfb21e796a",
      "fad35c21739049f6b156e8ed9f0138dd",
      "67001f2daaf54e8eabbdc04ea04275bb",
      "87ec3d986f814458908757bd1449321b",
      "85fc6c67ce084234ba634bd2952e0015",
      "0a46be3677494945a84aa654de56d877",
      "963a1800cddc4da986620d0418573329",
      "f2aed2597c9547b0a3e6aad197445d49",
      "c28deeaca3d546a0bc9ed2acd912f01e",
      "c59e5979805a4c82982754de2b9619db",
      "768be564ad4d409da137a9ccc1d2f027",
      "9435c0d2f5ba4923a4d00eb3608c72a0",
      "95488ccb0e9e48af97bfbc7957281f51",
      "6e6a47ac7719476caeb5df399f8da236",
      "9310586957b849ddb3ab45e10892f858",
      "f0609213e40040629de73ddba66e6364",
      "49f643f587b441ae8b9ee779f86ae44f",
      "48600bf2d24a4aaab955e611e18e9952",
      "7ad3b17aca59487a91d2f5f59c638fda",
      "74037e81c9304243beda7e64f5c6100f",
      "c9ec7b904a45486aa96f8580d3aaac82",
      "b43ea46be5644303b1d2f22043627a05",
      "d1ce1f773a724fe399d1e95b6296a9b7",
      "771458772ce74f9ea6f596b621ede833",
      "40c0ad17f8264efe997d246e1616a58a",
      "4c4def410a41453c84ac94b551f928a8",
      "e18dae1140554388a5cec96d8a8f7065",
      "4a7cdceeb332497c80886da527ddfa2a",
      "fb6e10e4b20d4c75bb4675709c940acb",
      "9524696c0ce34053b7ee900761127282",
      "2418d04ed0d44f558a403d069cb3e1fe",
      "11ebb2f403924051956170ef5c7c2be4",
      "aeed353639234130a1c6d9ae0245f74d",
      "cc933fc06094453e97fe2d60b0308bac",
      "20bb0900a307434b9fb336f669da7fff",
      "f42522d00baa4d1eae90f0e4584da1b8",
      "8e683c5f82d74427a595cdfa56d0d124",
      "7152e9a8e75f4a5aa4b7d23f24eb0e91",
      "3ddda8a0f98241d79c47c8dff7c50d33",
      "dd3a1cd422be4868a7382710b4f46b21",
      "da78f46a75844b38828623726972ae5c",
      "cc1bd3c12ebd49b5adbf7fa855cc5711",
      "b6a0160136374c0b9c19dba9577b0e87",
      "b0502d359f1148a4b31e1c5e2412598e",
      "2dafdadad0114b59a165c4fe3d288dbd",
      "360efb60c21b4fcf9aa780496e641860",
      "4f52d67f052b4797b485c7096f86dbb6",
      "0e9498ce5ae14669be87a0fa80381d65",
      "a91211fb034c4b7595e680f15634946d",
      "c8d6c83083a7411991333a7ee26c4b8f",
      "4387ed1a4cef4c85b1992d055ab825b8",
      "b8e386774e7c4170bfa753e5dd9778a4",
      "6784a03f49fc4f5dacbe69859f87ddee",
      "de3e0b1a3445415bba5a9c5ff83a855f",
      "ee4286005ef24eecb2955e37942f3d90",
      "98490440e40f45d096ff5f09c643ddb7",
      "818fad01927d452dbd0d56f6dbd115e7",
      "86ccd77fc82b4553a494caa93fb71084",
      "3c902b51fc2e40af97b15de0abd54b9c",
      "65057a4b0ee8488ebc197dee7382a361",
      "bdf536c064064736b660e4ae0cf0fbd3",
      "fec6c8e41d794423bbd07fa6c46bca67",
      "64f8bd7f1dd64daa8a98ba1ae6ec1275",
      "3785400cbd7f4dab9ac58f74be8f692d",
      "15f6872f5dac4259b82b98194116f56c",
      "ed2bcffe5804483a81ba2f93d4fe05f1",
      "1f87c66caa37430c86e04d3fb4a8c020",
      "bcac5a4b41cf4498b7867f758a246e1a",
      "42c5c817a5584b12bb3a76611fc3936d",
      "523c71cd31114304a842fd3243e721e7",
      "bd5a150beb6d4d0a80a3cce82f07e520",
      "27003b7f49fd4ce4a10f8adcd819edac",
      "28d8bd3d9bc64246adc0f8cc70983034",
      "d37911bf83f340b79fe514b6bb4f4b01",
      "6643d3dfdbf045078f4d8e11a9de3469",
      "dc3553016e2c4b3caeab479f3e480f34",
      "9b20cfd864f14e4da4aaccbb40f2e292",
      "ec09d19a6b5040f8a43a58246978cf4a",
      "68a78f308c734052aa85d0a55e1f45a1",
      "bc97f164235d4d9a8c5ce6ddff9ec9fc",
      "826f4e459b414c97b7275b718071e485",
      "5d28a0f39a1a4bf9814b9492004243f6",
      "5041f6ec5d4449a782e7ff77a89c2cc8",
      "b82337e55b784988b4b8a0575d0f5ace",
      "b891ccf058094084a4c72f810d15ad14",
      "477ce1a400a74d898aef30253965ba6b",
      "e2f7fa000d274479bf4ddc2081b3ad28",
      "097ba6dcaaef4dba98c01d15ba25d6ba",
      "bc06c28ecfa54cdbb9b3b5aa9517dc28",
      "d4050c35ed20426e88439af593d2f3cf",
      "7a5ad88a07a64ad68044d25a5375e0ae",
      "793e7a5bfbf24ff6808bb6f96680de6d",
      "c343d2f13670422d908f58a57352e248",
      "5322ddb05d77460696d1807a33d6ff3e"
     ]
    },
    "id": "sfhF5USalJCw",
    "outputId": "dac9399e-b23b-4f75-f4be-1dc4d689ff8d"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b78ab00b256e431f94113ac7f17a9f35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/727 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e90d653937e44e29a44c5e98e141f692",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading tokenizer.model:   0%|          | 0.00/500k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "963a1800cddc4da986620d0418573329",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/411 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48600bf2d24a4aaab955e611e18e9952",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/554 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb6e10e4b20d4c75bb4675709c940acb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)model.bin.index.json:   0%|          | 0.00/26.8k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd3a1cd422be4868a7382710b4f46b21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4387ed1a4cef4c85b1992d055ab825b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00001-of-00002.bin:   0%|          | 0.00/9.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fec6c8e41d794423bbd07fa6c46bca67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00002-of-00002.bin:   0%|          | 0.00/3.50G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28d8bd3d9bc64246adc0f8cc70983034",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b82337e55b784988b4b8a0575d0f5ace",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/137 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We are using local mod\n"
     ]
    }
   ],
   "source": [
    "if USEHFAPI:\n",
    "  llm = HuggingFaceHub(repo_id=LOCAL_MODEL, task=\"text-generation\", model_kwargs={\"temperature\":0.5, \"max_length\":2000, \"trust_remote_code\": True})\n",
    "  print(f\"We are using HF API {llm}\")\n",
    "else:\n",
    "  llm = HuggingFacePipeline.from_model_id(model_id=LOCAL_MODEL, task=\"text-generation\", device=0, model_kwargs={\"temperature\":0.5, \"max_length\":2000, \"trust_remote_code\": True})\n",
    "  print(f\"We are using local mod\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uOzeKdtFR6eJ"
   },
   "source": [
    "## Chat Agent Prompts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CwMiGURCbZXc"
   },
   "source": [
    "### System Prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Qjq2SyH5bYoi"
   },
   "outputs": [],
   "source": [
    "SYS_PROMPT_INSTRUCTIONS = {\n",
    "    \"pubmedqa\": {\n",
    "        \"task\": \"You are tasked with answering question based on the provided content related to pubmed articles. You should thoroughly understand the question, extract relevant information from the content, analyze this information and finally provide an answer. BE CONCISE AND ACCURATE\",\n",
    "        \"answer_format\": \"\"\"\n",
    "Analysis: Provide an analysis that logically leads to the answer based on the relevant content.\n",
    "Final Answer: Provide the final answer, Must be a 'yes', 'no'.\n",
    "        \"\"\",\n",
    "        \"not_to_dos\": \"Do not make assumptions not supported by the content. Avoid providing personal opinions or interpretations. Stick to summarizing and interpreting the content as objectively and accurately as possible. Remember that you are providing an analysis based on the content and not diagnosing or treating medical conditions.\"\n",
    "    },\n",
    "    \"ade\": {\n",
    "        \"task\": \"Your task is to identify the adverse drug effect mentioned in the question based on the provided content . Thoroughly understand the question, extract relevant information, analyze it, and provide a concise and accurate answer.\",\n",
    "        \"answer_format\": \"\"\"\n",
    "Analysis: Provide an analysis that logically leads to the answer based on the relevant content.\n",
    "Final Answer: Provide the final answer, which should identify the adverse effect.\n",
    "        \"\"\",\n",
    "        \"not_to_dos\": \"Do not make assumptions not supported by the content. Avoid providing personal opinions or interpretations. Summarize and interpret the information as objectively and accurately as possible. You are providing an analysis, not diagnosing or treating medical conditions.\"\n",
    "    },\n",
    "\n",
    "    \"chatDoctor\": {\n",
    "        \"task\": \"Your task is to list the medications based on the provided content related to the symptom or disease mentioned in the question. Understand the question, extract relevant information, analyze it, and provide a concise and accurate answer.\",\n",
    "        \"answer_format\": \"\"\"\n",
    "Analysis: Provide an analysis that logically leads to the answer based on the relevant content.\n",
    "Final Answer: Provide the final answer, which should be a list of medications related to the symptom or disease.\n",
    "        \"\"\",\n",
    "        \"not_to_dos\": \"Do not make assumptions not supported by the content. Avoid providing personal opinions or interpretations. Summarize and interpret the information as objectively and accurately as possible. You are providing an analysis, not diagnosing or treating medical conditions.\"\n",
    "    },\n",
    "    \"DDI_binary\": {\n",
    "        \"task\": \"You are tasked to answer yes/no questions based on the provided content about whether there is a reaction between the drugs mentioned. Understand the question, extract relevant information, analyze it, and provide a concise and accurate answer.\",\n",
    "        \"answer_format\": \"\"\"\n",
    "Analysis: Provide an analysis that logically leads to the answer based on the relevant content.\n",
    "Final Answer: Provide the final answer. The answer should be 'yes' or 'no'.\n",
    "        \"\"\",\n",
    "        \"not_to_dos\": \"Do not make assumptions not supported by the content. Avoid providing personal opinions or interpretations. Summarize and interpret the information as objectively and accurately as possible. You are providing an analysis, not diagnosing or treating medical conditions.\"\n",
    "    },\n",
    "\n",
    "    \"drug_usage\": {\n",
    "        \"task\": \"Your task is to answer two yes/no questions based on the provided content related to the usage of the drug provided. Understand the question, extract relevant information, analyze it, and provide a concise and accurate answer.\",\n",
    "        \"answer_format\": \"\"\"\n",
    "Analysis: Provide an analysis that logically leads to the answer based on the relevant content.\n",
    "Final Answer: Provide the final answer, The answer should be two 'yes' or 'no' separated by comma.\n",
    "        \"\"\",\n",
    "        \"not_to_dos\": \"Do not make assumptions not supported by the content. Avoid providing personal opinions or interpretations. Summarize and interpret the information as objectively and accurately as possible. You are providing an analysis, not diagnosing or treating medical conditions.\"\n",
    "    },\n",
    "\n",
    "    \"medmcqa\": {\n",
    "        \"task\": \"Your task is to answer multiple choice questions based on the provided content about medication and pharmacology. Understand the question, extract relevant information, analyze it, and provide a concise and accurate answer.\",\n",
    "        \"answer_format\": \"\"\"\n",
    "Analysis: Provide an analysis that logically leads to the answer based on the relevant content.\n",
    "Final Answer: Provide the final answer, which should be a single letter in the alphabet representing the best option among the multiple choices provided in the question.\n",
    "        \"\"\",\n",
    "        \"not_to_dos\": \"Do not make assumptions not supported by the content. Avoid providing personal opinions or interpretations. Summarize and interpret the information as objectively and accurately as possible. You are providing an analysis, not diagnosing or treating medical conditions.\"\n",
    "    },\n",
    "\n",
    "    \"mmlu_mc\": {\n",
    "        \"task\": \"Your task is to answer multiple choice questions based on the provided content about drug recommendation. Understand the question, extract relevant information, analyze it, and provide a concise and accurate answer.\",\n",
    "        \"answer_format\": \"\"\"\n",
    "Analysis: Provide an analysis that logically leads to the answer based on the relevant content.\n",
    "Final Answer: Provide the final answer, which should be a single letter in the alphabet representing the best option among the multiple choices provided in the question.\n",
    "        \"\"\",\n",
    "        \"not_to_dos\": \"Do not make assumptions not supported by the content. Avoid providing personal opinions or interpretations. Summarize and interpret the information as objectively and accurately as possible. You are providing an analysis, not diagnosing or treating medical conditions.\"\n",
    "    },\n",
    "\n",
    "    \"usmle_mc\": {\n",
    "        \"task\": \"Your task is to answer multiple choice questions based on the provided content about drug recommendation. Understand the question, analyze it, and provide a concise and accurate answer.\",\n",
    "        \"answer_format\": \"\"\"\n",
    "Analysis: Provide an analysis that logically leads to the answer based on the relevant content.\n",
    "Final Answer: Provide the final answer, which should be a single letter in the alphabet representing the best option among the multiple choices provided in the question.\n",
    "        \"\"\",\n",
    "        \"not_to_dos\": \"Do not make assumptions not supported by the content. Avoid providing personal opinions or interpretations. Summarize and interpret the content as objectively and accurately as possible.\"\n",
    "    },\n",
    "    \"moderna_interactions\": {\n",
    "        \"task\": \"You are tasked to answer yes/no questions based on the provided content about whether the therapeutic efficacy of Moderna COVID-19 Vaccine decrease when used with another drug. Understand the question, extract relevant information, analyze it, and provide a concise and accurate answer.\",\n",
    "        \"answer_format\": \"\"\"\n",
    "Analysis: Provide an analysis that logically leads to the answer based on the relevant information.\n",
    "Final Answer: Provide the final answer. The answer should be 'yes' or 'no'.\n",
    "        \"\"\",\n",
    "        \"not_to_dos\": \"Do not make assumptions not supported by the content. Avoid providing personal opinions or interpretations. Summarize and interpret the information as objectively and accurately as possible. You are providing an analysis, not diagnosing or treating medical conditions.\"\n",
    "    },\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "o1RyEbDUi5Pu"
   },
   "outputs": [],
   "source": [
    "sys_template = PromptTemplate(\n",
    "    input_variables=[\"task\", \"answer_format\", \"not_to_dos\"],\n",
    "    template=\"Task: {task}\\nAnswer Format: {answer_format}\\nNot to dos: {not_to_dos}\\n----Below are some examples-----\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Beq5H8bhjAhM"
   },
   "outputs": [],
   "source": [
    "def generate_sys_prompt(knowledge_set):\n",
    "  instruction = SYS_PROMPT_INSTRUCTIONS[knowledge_set]\n",
    "  return sys_template.format(**instruction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yWqsTAkL_A2X"
   },
   "source": [
    "### Few Shots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uTo7b4NmnsRV"
   },
   "outputs": [],
   "source": [
    "FS_EXAMPLES = {\n",
    "    \"pubmedqa\": [\n",
    "        {\n",
    "            \"question\": \"Does intermittent warm blood cardioplegia provide adequate myocardial resuscitation after global ischaemia?\",\n",
    "            \"content\" : \"\",\n",
    "            \"analysis\": \"Based on the provided information from the study, intermittent warm blood cardioplegia does not seem to provide adequate myocardial resuscitation after global ischaemia.\",\n",
    "            \"final_answer\": \"no\"\n",
    "        },\n",
    "        ],\n",
    "    \"ade\": [\n",
    "        {\n",
    "            \"question\": \"Identify the adverse drug reaction related to azithromycin in this context: Intravenous azithromycin-induced ototoxicity.\",\n",
    "            \"content\": \"\",\n",
    "            \"analysis\": \"It can be concluded that the adverse drug reaction related to azithromycin is ototoxicity.\",\n",
    "            \"final_answer\": \"ototoxicity\"\n",
    "        },\n",
    "    ],\n",
    "    \"chatDoctor\": [\n",
    "        {\n",
    "            \"question\": \"What are the recommended medications for Panic disorder?\",\n",
    "            \"content\": \"\",\n",
    "            \"analysis\": \"The medications listed in the content are the recommended ones for treating Panic disorder.\",\n",
    "            \"final_answer\": \"lorazepam, alprazolam, clonazepam, paroxetine, venlafaxine, mirtazapine, buspirone, fluvoxamine, imipramine, desvenlafaxine, clomipramine, acamprosate\"\n",
    "        },\n",
    "    ],\n",
    "    \"DDI_binary\": [\n",
    "        {\n",
    "            \"question\": \"Is there a reaction between Sibutramine and Icatibant?\",\n",
    "            \"content\": \"\",\n",
    "            \"analysis\": \"From the given context, it can be concluded that there is a reaction effect identified between Sibutramine and Icatibant\",\n",
    "            \"final_answer\": \"yes\"\n",
    "        },\n",
    "    ],\n",
    "    \"drug_usage\": [\n",
    "        {\n",
    "            \"question\": \"Answer the following two questions about acetaminophen:\\n Have studies shown adverse effects on preganancy?\\n Have studies shown an interaction with alchohol?\",\n",
    "            \"content\": \"\",\n",
    "            \"analysis\": \"The content shows that acetaminophen has shwon adverse effects on preganancy demonstrated in animal studies. The content has shown that acetaminophen interacts with alchohol\",\n",
    "            \"final_answer\": \"yes, yes\"\n",
    "        },\n",
    "    ],\n",
    "    \"usmle_mc\": [\n",
    "        {\n",
    "            \"question\": \"A 62-year-old woman presents for a regular check-up. She complains of lightheadedness and palpitations which occur episodically. Past medical history is significant for a myocardial infarction 6 months ago and NYHA class II chronic heart failure. She also was diagnosed with grade I arterial hypertension 4 years ago. Current medications are aspirin 81 mg, atorvastatin 10 mg, enalapril 10 mg, and metoprolol 200 mg daily. Her vital signs are a blood pressure of 135/90 mm Hg, a heart rate of 125/min, a respiratory rate of 14/min, and a temperature of 36.5¬∞C (97.7¬∞F). Cardiopulmonary examination is significant for irregular heart rhythm and decreased S1 intensity. ECG is obtained and is shown in the picture (see image). Echocardiography shows a left ventricular ejection fraction of 39%. Which of the following drugs is the best choice for rate control in this patient? A:Atenolol, B:Diltiazem, C:Propafenone, D:Digoxin\",\n",
    "            \"content\": \"\",\n",
    "            \"analysis\": \"The 62-year-old woman's presentation of lightheadedness and palpitations, an irregular heart rhythm on cardiopulmonary examination, and her history of myocardial infarction and NYHA class II chronic heart failure suggest she may be suffering from a rate control issue related to her heart. The patient is already on a regimen that includes metoprolol, a beta-blocker used for rate control, and the dose is relatively high. This makes adding another beta-blocker like atenolol less effective. The use of calcium channel blockers, like diltiazem, might worsen her heart failure symptoms. Propafenone, a class IC antiarrhythmic, is mainly used to treat conditions that cause a fast heart rate, such as atrial fibrillation and atrial flutter, and ventricular arrhythmias, but may not be suitable for a patient with heart failure. On the other hand, digoxin has been traditionally used for rate control in patients with heart failure and atrial fibrillation, as it increases the strength of the heart's contractions and slows down the electrical impulses in the AV node.\",\n",
    "            \"final_answer\": \"D\",\n",
    "        },\n",
    "    ],\n",
    "    \"mmlu_mc\": [\n",
    "        {\n",
    "            \"question\": \"A 55-year-old man is brought to the emergency department by his wife because he told her he did not want to live anymore. During the past 6 weeks, he has experienced fatigue, loss of interest in usual activities, a 7-kg (15-lb) weight loss, and insomnia. He has no history of serious medical or psychiatric illness. Vital signs are temperature 37.0¬∞C (98.6¬∞F), pulse 80/min, respirations 16/min, and blood pressure 140/82 mm Hg. Physical examination discloses no abnormalities. Beck Depression Inventory score is 35 (severely depressed). He says he feels guilty that his investments have not done well and that he has ruined his family finances. He reports he hears voices at night telling him he has sinned. In addition to olanzapine, which of the following is the best treatment option for this patient? A:Divalproex, B:Fluoxetine, C:Lamotrigine, D:Lithium carbonate\",\n",
    "            \"content\": \"\",\n",
    "            \"analysis\": \"The patient presents with symptoms indicative of a major depressive episode with possible psychotic features, as indicated by his hearing voices. The severity of his depression is confirmed by his high Beck Depression Inventory score. As such, his treatment should address both the depressive symptoms and the psychotic symptoms.\",\n",
    "            \"final_answer\": \"B\",\n",
    "        },\n",
    "    ],\n",
    "    \"medmcqa\": [\n",
    "        {\n",
    "            \"question\": \"A 42year old female presents with diazepam and alcohol overdose. She is comatose. Temperature is 34.5degC. BP is 100/80 mmHg. Creatinine is 2.4mg/dL, AST -500, GGT- 35 IU. Urine dipstick showed 3+ for blood but urine analysis was normal. USG abdomen was normal. What is the most likely diagnosis? A:Hypothermia, B: Alcoholic hallucinosis, C: Rhabdomyolysis, D: Acute interstitial nephritis\",\n",
    "            \"content\": \"\",\n",
    "            \"analysis\": \"The patient has presented following an overdose of diazepam and alcohol. Given the symptoms and clinical findings, we are considering several potential diagnoses: Hypothermia, Alcoholic hallucinosis, Rhabdomyolysis, and Acute interstitial nephritis.Hypothermia (option A) could be suggested by the low body temperature, but it doesn't account for all of the patient's symptoms and test results.Alcoholic hallucinosis (option B) could be a possibility given the alcohol overdose, but the patient is comatose and not displaying signs of hallucinations.Acute interstitial nephritis (option D) might cause an increase in creatinine, but it generally does not result in blood in the urine without abnormality in urine analysis.Rhabdomyolysis (option C), on the other hand, can result from a drug overdose. This condition causes muscle breakdown, which releases myoglobin into the bloodstream. Myoglobin can cause renal failure, which could explain the increased creatinine. It also can show up as blood on a urine dipstick test while other aspects of a urine analysis remain normal, due to the fact that common urine dipstick tests cannot differentiate between myoglobin and hemoglobin.\",\n",
    "            \"final_answer\": \"C\",\n",
    "        },\n",
    "    ],\n",
    "    \"moderna_interactions\": [\n",
    "        {\n",
    "            \"question\": \"Does the therapeutic efficacy of Moderna COVID-19 Vaccine decrease when used in combination with abatacept?\",\n",
    "            \"content\": \"\",\n",
    "            \"analysis\": \"From the given context, it can be concluded that the therapeutic efficacy of Moderna COVID-19 Vaccine does decrease when used with abatacept\",\n",
    "            \"final_answer\": \"yes\"\n",
    "        },\n",
    "    ],\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Kkxv5jDB_N3X"
   },
   "outputs": [],
   "source": [
    "# Define Few-shots prompt\n",
    "fs_template = PromptTemplate(\n",
    "    input_variables=[\"question\", \"content\", \"analysis\", \"final_answer\"],\n",
    "    template=\"Question: {question}\\nContent: {content}\\nAnalysis: {analysis}\\nFinal Answer: {final_answer}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mHhWUVWZ_iAZ"
   },
   "outputs": [],
   "source": [
    "# Define few-shot prompt template\n",
    "def fs_prompt_temp(knowledge_set):\n",
    "  fs_p = FewShotPromptTemplate(\n",
    "      examples=FS_EXAMPLES[knowledge_set],\n",
    "      example_prompt=fs_template,\n",
    "      suffix=\"Question: {input_question}\\nContent: {input_content}\",\n",
    "      input_variables=[\"input_question\", \"input_content\"]\n",
    "  )\n",
    "  return fs_p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TlaN-hnxW4X6"
   },
   "source": [
    "#### Few shots generater for chat conversation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ma9YmZeqXYB_"
   },
   "outputs": [],
   "source": [
    "def generate_fs_prompt_chat(knowledge_set):\n",
    "  knowledge_set = FS_EXAMPLES[knowledge_set]\n",
    "  fs_messages = []\n",
    "  for example in knowledge_set:\n",
    "    user_message = {'role': 'user', 'content': f\"Question: {example['question']}\\nContent: {example['content']}\"}\n",
    "    assistant_message = {'role': 'assistant', 'content': f\"\\nAnalysis: {example['analysis']}\\nFinal Answer: {example['final_answer']}\"}\n",
    "    fs_messages.extend([user_message, assistant_message])\n",
    "  return fs_messages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hht2t8z5qop4"
   },
   "source": [
    "### Final Prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mKYR-kSYrWXq"
   },
   "outputs": [],
   "source": [
    "def generate_final_prompt(knowledge_set, input_question, input_content):\n",
    "  fs_prompt = fs_prompt_temp(knowledge_set)\n",
    "  fs_prompt_input = fs_prompt.format(input_question=input_question, input_content=input_content)\n",
    "  sys_prompt_input = generate_sys_prompt(knowledge_set)\n",
    "  return [sys_prompt_input, fs_prompt_input]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "HozQgjjI-_lB",
    "outputId": "7ee3a678-9315-4759-b4f6-86cc5836e5b1"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Task: You are tasked to answer yes/no questions based on the provided content about whether the therapeutic efficacy of Moderna COVID-19 Vaccine decrease when used with another drug. Understand the question, extract relevant information, analyze it, and provide a concise and accurate answer.\n",
      "Answer Format: \n",
      "Analysis: Provide an analysis that logically leads to the answer based on the relevant information.\n",
      "Final Answer: Provide the final answer. The answer should be 'yes' or 'no'.\n",
      "        \n",
      "Not to dos: Do not make assumptions not supported by the content. Avoid providing personal opinions or interpretations. Summarize and interpret the information as objectively and accurately as possible. You are providing an analysis, not diagnosing or treating medical conditions.\n",
      "----Below are some examples-----\n",
      "\n",
      "Question: Does the therapeutic efficacy of Moderna COVID-19 Vaccine decrease when used in combination with abatacept?\n",
      "Content: The therapeutic efficacy of Moderna COVID-19 Vaccine decrease when used in combination with abatacept?\n",
      "Analysis: From the given context, it can be concluded that the therapeutic efficacy of Moderna COVID-19 Vaccine does decrease when used with abatacept\n",
      "Final Answer: yes\n",
      "\n",
      "Question: hi\n",
      "Content: yo\n"
     ]
    }
   ],
   "source": [
    "prompt_template = prompt_input = PromptTemplate(\n",
    "    input_variables=[\"sys_prompt\", \"fs_prompt\"],\n",
    "    template=\"{sys_prompt}\\n{fs_prompt}\"\n",
    ")\n",
    "sys_prompt, fs_prompt = generate_final_prompt(knowledge_set='moderna_interactions', input_question='hi', input_content='yo')\n",
    "print(prompt_template.format(sys_prompt=sys_prompt, fs_prompt=fs_prompt))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UL411gCW_9wc"
   },
   "source": [
    "## Parsers\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "93ZqTi27jrB4"
   },
   "source": [
    "#### binary_parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GPwLKP7I__-z"
   },
   "outputs": [],
   "source": [
    "def binary_parser(output):\n",
    "    sections = output.split('\\n')\n",
    "    analysis = \"\"\n",
    "    final_answer = \"\"\n",
    "    for section in sections:\n",
    "        section_lower = section.lower()\n",
    "        if section_lower.startswith(\"analysis: \"):\n",
    "            analysis = section_lower.replace(\"analysis: \", \"\")\n",
    "        elif section_lower.startswith(\"final answer: \"):\n",
    "            final_answer = section_lower.replace(\"final answer: \", \"\").strip()\n",
    "    # Ensuring that we only get 'yes' or 'no'\n",
    "    if 'yes' in final_answer:\n",
    "        final_answer = 'yes'\n",
    "    elif 'no' in final_answer:\n",
    "        final_answer = 'no'\n",
    "    else:\n",
    "        final_answer = \"\"\n",
    "    return analysis, final_answer\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iiZGWHObjuAe"
   },
   "source": [
    "#### mc_parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Jp4ZkRlgjwFW"
   },
   "outputs": [],
   "source": [
    "def mc_parser(output):\n",
    "    sections = output.split('\\n')\n",
    "    analysis = \"\"\n",
    "    final_answer = \"\"\n",
    "    for section in sections:\n",
    "        section_lower = section.lower()\n",
    "        if section_lower.startswith(\"analysis: \"):\n",
    "            analysis = section_lower.replace(\"analysis: \", \"\")\n",
    "        elif section_lower.startswith(\"final answer: \"):\n",
    "            final_answer = section_lower.replace(\"final answer: \", \"\").strip()\n",
    "    # We only keep the first character of the final answer which should be the letter representing the chosen option\n",
    "    final_answer = final_answer[0] if final_answer and final_answer[0].isalpha() else \"\"\n",
    "    return analysis, final_answer\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ogDnKkfOjzld"
   },
   "source": [
    "#### text_parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tj2NYgP2jysp"
   },
   "outputs": [],
   "source": [
    "def text_parser(output):\n",
    "    sections = output.split('\\n')\n",
    "    analysis = \"\"\n",
    "    final_answer = \"\"\n",
    "    for section in sections:\n",
    "        section_lower = section.lower()\n",
    "        if section_lower.startswith(\"analysis: \"):\n",
    "            analysis = section_lower.replace(\"analysis: \", \"\")\n",
    "        elif section_lower.startswith(\"final answer: \"):\n",
    "            final_answer = section_lower.replace(\"final answer: \", \"\").strip()\n",
    "    return analysis, final_answer\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "T3pE4HWvlKVH"
   },
   "source": [
    "#### parser_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Vb_LwuuqiohS"
   },
   "outputs": [],
   "source": [
    "# knowledge_paths\n",
    "parser_dict = {\n",
    "    'pubmedqa': binary_parser,\n",
    "    'ade': text_parser,\n",
    "    'chatDoctor': text_parser,\n",
    "    'DDI_binary': binary_parser,\n",
    "    'drug_usage': text_parser,\n",
    "    'medmcqa': mc_parser,\n",
    "    'mmlu_mc': mc_parser,\n",
    "    'usmle_mc': mc_parser,\n",
    "    'moderna_interactions': binary_parser,\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Pm73fPy1Xg_5"
   },
   "source": [
    "## LLM Chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "lHSMtp2aXjj8",
    "outputId": "68c22276-a2b6-4f13-824e-04293666b2ce"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[0;32m<ipython-input-70-d032772de909>\u001B[0m in \u001B[0;36m<cell line: 1>\u001B[0;34m()\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mllm_chain\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mLLMChain\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mprompt\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mprompt_template\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mllm\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mllm\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[0;31mNameError\u001B[0m: name 'llm' is not defined"
     ]
    }
   ],
   "source": [
    "llm_chain = LLMChain(prompt=prompt_template, llm=llm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CUhu4HudEM9B"
   },
   "source": [
    "#### Sanity Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XcFmsjDvEKb3"
   },
   "outputs": [],
   "source": [
    "test_question = 'Is methotrexate an alternative to azathioprine in neuromyelitis optica spectrum disorders with aquaporin-4 antibodies?'\n",
    "test_content = ''\n",
    "test_sys_prompt, test_fs_prompt = generate_final_prompt(knowledge_set='pubmedqa', input_question=test_question, input_content=test_content)\n",
    "result = llm_chain.run({\"sys_prompt\": test_sys_prompt, \"fs_prompt\": test_fs_prompt})\n",
    "analysis, final_answer = parser(result)\n",
    "print(\"Analysis:\\n\", analysis)\n",
    "print(\"Final Answer:\\n\", final_answer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "luMUk-gKvOnW"
   },
   "source": [
    "## Data Pre-processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ldGr0plckp2R"
   },
   "source": [
    "### Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "o63yOPpeksWr"
   },
   "outputs": [],
   "source": [
    "def load_data(key, data_paths_dict, threshold):\n",
    "    # Get the path of the data based on the key\n",
    "    data_path = data_paths_dict[key]['data']\n",
    "\n",
    "    # Load the data into a DataFrame\n",
    "    df = pd.read_csv(data_path)\n",
    "    if len(df) > threshold:\n",
    "      df = df[:threshold]\n",
    "\n",
    "    # Convert DataFrame to dict\n",
    "    data_dict = {\n",
    "        'id': df['id'].tolist(),\n",
    "        'question': df['question'].tolist(),\n",
    "        'context': df['context'].tolist(),\n",
    "        'answer': df['answer'].tolist(),\n",
    "    }\n",
    "\n",
    "    n_samples = len(data_dict['question'])\n",
    "    print(f\"We have {n_samples} data samples in {key} dataset\")\n",
    "\n",
    "    return data_dict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dHxkS_82k6lJ",
    "outputId": "02041cba-1aaf-4b94-d6c8-b770dc30985b"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "We have 5000 data samples in pubmedqa dataset\n",
      "We have 5000 data samples in ade dataset\n",
      "We have 796 data samples in chatDoctor dataset\n",
      "We have 600 data samples in DDI_binary dataset\n",
      "We have 2888 data samples in drug_usage dataset\n",
      "We have 2000 data samples in medmcqa dataset\n",
      "We have 956 data samples in mmlu_mc dataset\n",
      "We have 1499 data samples in usmle_mc dataset\n",
      "We have 200 data samples in moderna_interactions dataset\n"
     ]
    }
   ],
   "source": [
    "data_df_dict = {}\n",
    "knowledge_df_dict = {\n",
    "    'text': {},\n",
    "    'embedding': {},\n",
    "}\n",
    "for key in data_paths_dict.keys():\n",
    "    if key in ['medmcqa', 'mmlu_mc', 'usmle_mc', 'DDI_binary', 'drug_usage', 'chatDoctor', 'moderna_interactions']:\n",
    "      knowledge_df_dict['text'][key] = pd.read_csv(knowledge_blocks_dict[key])\n",
    "    if not key in ['DDI_binary', 'drug_usage', 'moderna_interactions']:\n",
    "      data_df_dict[key] = load_data(key, data_paths_dict, SLICE_THRESHOLD)\n",
    "    else:\n",
    "      data_path = data_paths_dict[key]['data']\n",
    "      # Load the data into a DataFrame\n",
    "      df = pd.read_csv(data_path)\n",
    "      # Convert DataFrame to dict\n",
    "      data_dict = {\n",
    "          'question': df['question'].tolist(),\n",
    "          'answer': df['answer'].tolist(),\n",
    "      }\n",
    "      n_samples = len(data_dict['question'])\n",
    "      data_df_dict[key] = data_dict\n",
    "      print(f\"We have {n_samples} data samples in {key} dataset\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fS9FrcOAcmsP"
   },
   "source": [
    "### Dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dZuH2PemcmNe"
   },
   "outputs": [],
   "source": [
    "def dataloader(key, data_df_dict):\n",
    "    # Get the dictionary corresponding to the key\n",
    "    data = data_df_dict[key]\n",
    "\n",
    "    # Create a DataFrame from the dictionary\n",
    "    df = pd.DataFrame(data)\n",
    "\n",
    "    # Shuffle the DataFrame\n",
    "    df = df.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "    # Decide the sample size based on the size of the dataset\n",
    "    sample_size = EVALUATION_SIZE if len(df) > EVALUATION_SIZE else len(df)\n",
    "\n",
    "    if data_paths_dict[key]['type'] == 'binary':\n",
    "        min_samples = min(sample_size//2, min(df['answer'].value_counts().values))\n",
    "        # For binary, ensure equal number of 'yes' and 'no' samples\n",
    "        yes_df = df[df['answer'] == 'yes'].sample(n=min_samples, replace=False)\n",
    "        no_df = df[df['answer'] == 'no'].sample(n=min_samples, replace=False)\n",
    "        sample_df = pd.concat([yes_df, no_df])\n",
    "        print(f\"we have {len(yes_df)} of yes and {len(no_df)} of no in the loaded {key} evaluation set\")\n",
    "    elif data_paths_dict[key]['type'] == 'mc':\n",
    "        min_samples = min(sample_size//4, min(df['answer'].value_counts().values))\n",
    "        print(min_samples)\n",
    "        # For multiple choice, ensure equal number of each option\n",
    "        option_a_df = df[df['answer'] == 'A'].sample(n=min_samples, replace=False)\n",
    "        option_b_df = df[df['answer'] == 'B'].sample(n=min_samples, replace=False)\n",
    "        option_c_df = df[df['answer'] == 'C'].sample(n=min_samples, replace=False)\n",
    "        option_d_df = df[df['answer'] == 'D'].sample(n=min_samples, replace=False)\n",
    "        print(f\"we have {len(option_a_df)} of A and {len(option_b_df)} of B and {len(option_c_df)} of C and {len(option_d_df)} of D in the loaded {key} evaluation set\")\n",
    "        sample_df = pd.concat([option_a_df, option_b_df, option_c_df, option_d_df])\n",
    "    else:\n",
    "        # Select a random sample of rows from the DataFrame\n",
    "        sample_df = df.sample(n=sample_size)\n",
    "        print(f\"we have {len(sample_df)} of samples\")\n",
    "    # Convert the selected sample to a dict\n",
    "    sample_data = {\n",
    "        'sample': sample_df['question'].tolist(),\n",
    "        'label': sample_df['answer'].tolist(),\n",
    "    }\n",
    "\n",
    "    return sample_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ki9u5U8JtZHL",
    "outputId": "700779ea-33b3-40ac-e6ea-fb3e0a3752b8"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "we have 500 of yes and 500 of no in the loaded pubmedqa evaluation set\n",
      "250\n",
      "we have 250 of A and 250 of B and 250 of C and 250 of D in the loaded medmcqa evaluation set\n",
      "we have 100 of yes and 100 of no in the loaded moderna_interactions evaluation set\n"
     ]
    }
   ],
   "source": [
    "pubmed_eval = dataloader('pubmedqa', data_df_dict)\n",
    "medmcqa_eval = dataloader('medmcqa', data_df_dict)\n",
    "moderna_eval = dataloader('moderna_interactions', data_df_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MGI8VNh7udcl",
    "outputId": "1cd94277-fc1e-46ae-bf08-926702e77d56"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Does vasopressin improve survival compared with epinephrine in a neonatal piglet model of asphyxial cardiac arrest?\n",
      "yes\n"
     ]
    }
   ],
   "source": [
    "print(pubmed_eval['sample'][0])\n",
    "print(pubmed_eval['label'][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nSxiXaqxe63G"
   },
   "source": [
    "## OPENAI API Handlers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Rja-YT-Ge9iT"
   },
   "outputs": [],
   "source": [
    "# Function to create a chat message and get a response\n",
    "def chat_handler(query, prompt, few_shots, model):\n",
    "    if USEOPENAI:\n",
    "      while True:\n",
    "        try:\n",
    "            response = openai.ChatCompletion.create(\n",
    "                messages = [\n",
    "                    {'role': 'system', 'content': prompt},\n",
    "                    *few_shots,\n",
    "                    {'role': 'user', 'content': query},\n",
    "                ],\n",
    "                model=model,\n",
    "                temperature=0.1,\n",
    "            )\n",
    "            return response['choices'][0]['message']['content']\n",
    "        except Exception as e:\n",
    "            print(f\"Chat API error: {e}\")\n",
    "            continue\n",
    "\n",
    "# Function to create a embedding message and get a response\n",
    "def embedding_handler(text):\n",
    "    while True:\n",
    "      try:\n",
    "          embedding_response = openai.Embedding.create(\n",
    "              model='text-embedding-ada-002',\n",
    "              input=text,\n",
    "          )\n",
    "          return embedding_response[\"data\"][0][\"embedding\"]\n",
    "      except Exception as e:\n",
    "          print(f\"Embedding API error: {e}\")\n",
    "          continue\n",
    "\n",
    "\n",
    "# Function to get knowledge about a specific medication\n",
    "def knowledge_handler(data):\n",
    "    return data['context'];\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qYeHw_ows6a2"
   },
   "source": [
    "## Chat Implementation (set local to true to evaluate other models, current evaluating openAI models)\n",
    "\n",
    "1. query: user input\n",
    "2. model: LLM model\n",
    "3. knowledge: text based knowledge\n",
    "4. knowledge_embedding: embedding based knowledge\n",
    "5. embedding_process: the process to convert text to embedding\n",
    "6. useknowledge: useknowledge or not\n",
    "7. local: use local model or openAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Jq0vjVc1DwAB"
   },
   "outputs": [],
   "source": [
    "def ask(query, model, knowledge_key, embedding_process, useknowledge, local):\n",
    "    parser = parser_dict[knowledge_key]\n",
    "\n",
    "    CoT = ''\n",
    "    chat_response = ''\n",
    "\n",
    "    # we can switch from using local open-source model or openAI api\n",
    "    if local:\n",
    "\n",
    "      # generate system prompt, few-shot prompt with user query and knowledge\n",
    "      sys_prompt, fs_prompt = generate_final_prompt(knowledge_set=knowledge_key, input_question=query, input_content='')\n",
    "      response = model.run({\"sys_prompt\": sys_prompt, \"fs_prompt\": fs_prompt})\n",
    "\n",
    "      # parse the final result\n",
    "      analysis, final_answer = parser(response)\n",
    "      CoT = analysis\n",
    "      chat_response = final_answer\n",
    "\n",
    "    else:\n",
    "      sys_prompt = generate_sys_prompt(knowledge_key)\n",
    "      fs_prompt = generate_fs_prompt_chat(knowledge_key)\n",
    "      # construct user query with domain-specific knowledge\n",
    "      query = f\"\\nQuestion: {query} \\nContent: {''}\"\n",
    "      # Initiate conversation\n",
    "      response = chat_handler(query, sys_prompt, fs_prompt, model)\n",
    "\n",
    "      analysis, final_answer = parser(response)\n",
    "      CoT = analysis\n",
    "      chat_response = final_answer\n",
    "\n",
    "    return [chat_response, CoT, response]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "K1VE3B39DRkD"
   },
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d7PkKLQHX3Rg",
    "outputId": "113d5a41-f0cc-480e-bb6b-1d09075bfd64"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "190\n",
      "we have 190 of A and 190 of B and 190 of C and 190 of D in the loaded mmlu_mc evaluation set\n"
     ]
    }
   ],
   "source": [
    "dataset_name = 'mmlu_mc'\n",
    "evaluation_set = dataloader(dataset_name, data_df_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pLBo_lTeQczH",
    "outputId": "426b08c1-605d-4822-f3b7-f492016154d8"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "A 35-year-old man comes to the office because of 1-week history of mid low back pain that radiates down his right leg. The pain began after the patient lifted a heavy box onto his truck. He rates his current pain as an 8 on a 10-point scale. He has been unable to find a comfortable position and has been sleeping in a recliner. Medical history is unremarkable and he takes no medications. He has smoked one pack of cigarettes daily for the past 25 years, and he drinks a six-pack of beer on Friday and Saturday nights. BMI is 27 kg/m2 . He appears uncomfortable and stands during the physical examination. Vital signs are normal. Straight-leg raise test is positive on the right, with loss of right ankle reflex. The remainder of the physical examination discloses no abnormalities. Which of the following is the most likely explanation for this patient’s symptoms? A:Displacement of the nucleus pulposus, B:Hypertrophy of the facet joints, C:Osteophyte formation, D:Spondylolisthesis\n",
      "A\n"
     ]
    }
   ],
   "source": [
    "print(evaluation_set['sample'][0])\n",
    "print(evaluation_set['label'][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### metrics"
   ],
   "metadata": {
    "id": "AjvGeNnhW9Ok"
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aEyXA8nUb4OQ"
   },
   "outputs": [],
   "source": [
    "def check_text_accuracy(prediction, actual):\n",
    "    # Remove any trailing periods from the prediction\n",
    "    prediction = prediction.rstrip('.')\n",
    "\n",
    "    # Generate the initials from the prediction\n",
    "    prediction_initials = ''.join(word[0] for word in prediction.split())\n",
    "\n",
    "    actual_words = actual.split()\n",
    "    prediction_contains_all_words = all(word in prediction for word in actual_words)\n",
    "\n",
    "    if prediction_contains_all_words or prediction_initials == actual:\n",
    "        # Prediction contains all words in actual or initials match\n",
    "        return True\n",
    "\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ciPE_sygmdQg"
   },
   "outputs": [],
   "source": [
    "def calculate_f1_metrics(prediction, label):\n",
    "    # Convert predictions and labels to sets for easy comparison\n",
    "    prediction_set = set(prediction.split(', '))\n",
    "    label_set = set(label.split(', '))\n",
    "    print(prediction_set)\n",
    "    print(label_set)\n",
    "    # Compute true positives, false positives, and false negatives\n",
    "    true_positives = len(prediction_set & label_set)\n",
    "    false_positives = len(prediction_set - label_set)\n",
    "    false_negatives = len(label_set - prediction_set)\n",
    "\n",
    "    # Compute precision, recall, and F1 score\n",
    "    precision = true_positives / (true_positives + false_positives) if true_positives + false_positives > 0 else 0\n",
    "    recall = true_positives / (true_positives + false_negatives) if true_positives + false_negatives > 0 else 0\n",
    "    f1_score = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0\n",
    "\n",
    "    return precision, recall, f1_score\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "84181z1RI_op"
   },
   "source": [
    "### GPT-3.5 Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "un1t76vGd4Od"
   },
   "source": [
    "#### Without CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0krY4-bRvO2K"
   },
   "outputs": [],
   "source": [
    "accurate_predictions = 0;\n",
    "print_process = True;\n",
    "basic_mc_prompt = 'Provide the final answer, which should be a single letter in the alphabet representing the best option among the multiple choices provided in the question. Follow this format:\\nFinal Answer: A'\n",
    "basic_binary_prompt = 'Provide the final answer, which should be a yes or no. Never say you dont know, always provide a yes or no answer regardless. If you dont have enough information, just guess. Follow this format:\\nFinal Answer: yes'\n",
    "for i in tqdm(range(len(evaluation_set['sample']) ), desc=\"Processing\"):\n",
    "  chat_response = chat_handler(evaluation_set['sample'][i], basic_binary_prompt, [], GPT_3)\n",
    "  analysis, prediction = binary_parser(chat_response)\n",
    "  if dataset_name == 'ade':\n",
    "    if check_text_accuracy(prediction, evaluation_set['label'][i].lower()):\n",
    "      accurate_predictions += 1\n",
    "    else:\n",
    "      print(f\"\\nWrong Answer Index: {i} Question: {evaluation_set['sample'][i]} Chat Response: {prediction}, Answer: {evaluation_set['label'][i]}\")\n",
    "  else:\n",
    "    prediction = chat_response.split(\".\")[0].lower()\n",
    "    if prediction == evaluation_set['label'][i].lower():\n",
    "      accurate_predictions += 1\n",
    "    else:\n",
    "      print(f\"\\nWrong Answer Index: {i} Chat Response: {chat_response}, Answer: {evaluation_set['label'][i]}\")\n",
    "print(f\"\\nAccuracy: {accurate_predictions}/{len(evaluation_set['sample']) }\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8gmAmiHoeCj4"
   },
   "source": [
    "#### With CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AddFd4gBeB-J",
    "outputId": "4bcd656e-0e3c-4e88-d8a5-ad4f7a057a81"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:   1%|          | 2/200 [00:05<09:05,  2.75s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 1 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:   2%|▎         | 5/200 [00:15<09:41,  2.98s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 4 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   3%|▎         | 6/200 [00:18<09:42,  3.00s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 5 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:   4%|▍         | 9/200 [00:25<08:01,  2.52s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 8 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:   6%|▌         | 11/200 [00:30<07:52,  2.50s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 10 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   6%|▌         | 12/200 [00:32<07:21,  2.35s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 11 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:   7%|▋         | 14/200 [00:38<08:34,  2.77s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 13 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   8%|▊         | 15/200 [00:40<07:56,  2.58s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 14 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:   8%|▊         | 17/200 [00:46<08:16,  2.71s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 16 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  10%|█         | 21/200 [00:56<08:22,  2.81s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 20 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  12%|█▏        | 24/200 [01:06<08:49,  3.01s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 23 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  14%|█▍        | 29/200 [01:20<08:33,  3.00s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 28 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  17%|█▋        | 34/200 [01:32<07:01,  2.54s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 33 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  18%|█▊        | 36/200 [01:37<07:05,  2.59s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 35 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  18%|█▊        | 37/200 [01:40<07:03,  2.60s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 36 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  20%|██        | 40/200 [01:47<06:34,  2.46s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 39 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  22%|██▏       | 44/200 [01:58<06:44,  2.60s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 43 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  24%|██▍       | 48/200 [02:10<07:21,  2.91s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 47 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  24%|██▍       | 49/200 [02:13<07:32,  3.00s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 48 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  26%|██▌       | 51/200 [02:18<07:01,  2.83s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 50 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  26%|██▌       | 52/200 [02:21<06:38,  2.69s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 51 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  28%|██▊       | 56/200 [02:32<06:59,  2.91s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 55 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  30%|███       | 60/200 [02:41<05:32,  2.38s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 59 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  31%|███       | 62/200 [02:46<05:46,  2.51s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 61 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  32%|███▏      | 63/200 [02:48<05:22,  2.35s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 62 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  32%|███▏      | 64/200 [02:50<05:14,  2.31s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 63 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  34%|███▎      | 67/200 [02:59<06:08,  2.77s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 66 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  36%|███▌      | 71/200 [03:09<05:19,  2.48s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 70 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  36%|███▌      | 72/200 [03:11<05:26,  2.55s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 71 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  38%|███▊      | 77/200 [03:24<05:17,  2.58s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 76 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  39%|███▉      | 78/200 [03:27<05:19,  2.62s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Chat API error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 23 Jun 2023 03:02:25 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7db976589a924aa5-TPE', 'alt-svc': 'h3=\":443\"; ma=86400'}\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  42%|████▏     | 83/200 [08:54<49:13, 25.24s/it]  "
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 82 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  42%|████▎     | 85/200 [08:58<25:59, 13.56s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 84 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  44%|████▎     | 87/200 [09:04<15:02,  7.99s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 86 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  45%|████▌     | 90/200 [09:13<08:54,  4.86s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 89 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  46%|████▋     | 93/200 [09:21<06:09,  3.46s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 92 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  49%|████▉     | 98/200 [09:35<04:33,  2.68s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 97 YO Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  50%|█████     | 101/200 [09:43<04:22,  2.65s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 100 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  52%|█████▎    | 105/200 [09:52<03:34,  2.26s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 104 YO Chat Response: yes, Answer: no\n",
      "Chat API error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 23 Jun 2023 03:08:50 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7db97fbd3a3e4a6d-TPE', 'alt-svc': 'h3=\":443\"; ma=86400'}\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  55%|█████▍    | 109/200 [15:15<52:41, 34.74s/it]  "
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 108 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  56%|█████▋    | 113/200 [15:26<15:15, 10.53s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 112 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  57%|█████▋    | 114/200 [15:29<11:36,  8.10s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 113 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  58%|█████▊    | 117/200 [15:36<06:09,  4.45s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 116 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  60%|██████    | 120/200 [15:45<04:32,  3.40s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Chat API error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 23 Jun 2023 03:14:43 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7db9885df9566a81-TPE', 'alt-svc': 'h3=\":443\"; ma=86400'}\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  61%|██████    | 122/200 [21:02<1:29:02, 68.50s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 121 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  62%|██████▏   | 124/200 [21:08<44:15, 34.94s/it]  "
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 123 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  63%|██████▎   | 126/200 [21:15<23:13, 18.83s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 125 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  64%|██████▍   | 128/200 [21:21<13:00, 10.85s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Chat API error: The server is overloaded or not ready yet.\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  66%|██████▌   | 131/200 [22:00<11:30, 10.00s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 130 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  69%|██████▉   | 138/200 [22:18<03:31,  3.41s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 137 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  70%|██████▉   | 139/200 [22:21<03:18,  3.26s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 138 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  70%|███████   | 141/200 [22:25<02:37,  2.66s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 140 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  71%|███████   | 142/200 [22:27<02:23,  2.48s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 141 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  72%|███████▏  | 143/200 [22:31<02:42,  2.84s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 142 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  73%|███████▎  | 146/200 [22:39<02:22,  2.63s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 145 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  74%|███████▍  | 148/200 [22:44<02:22,  2.73s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 147 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  76%|███████▌  | 151/200 [22:51<01:53,  2.32s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 150 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  78%|███████▊  | 157/200 [23:07<01:55,  2.70s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 156 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  80%|████████  | 160/200 [23:15<01:55,  2.88s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 159 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  80%|████████  | 161/200 [23:18<01:47,  2.76s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 160 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  82%|████████▎ | 165/200 [23:28<01:35,  2.73s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 164 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  83%|████████▎ | 166/200 [23:31<01:29,  2.62s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 165 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  84%|████████▎ | 167/200 [23:34<01:30,  2.74s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 166 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  84%|████████▍ | 168/200 [23:37<01:33,  2.91s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 167 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  86%|████████▌ | 172/200 [23:47<01:08,  2.43s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 171 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  86%|████████▋ | 173/200 [23:49<01:03,  2.35s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Chat API error: The server is overloaded or not ready yet.\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  88%|████████▊ | 175/200 [24:24<03:39,  8.78s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 174 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  94%|█████████▎| 187/200 [24:55<00:37,  2.89s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 186 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  94%|█████████▍| 188/200 [24:58<00:33,  2.78s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 187 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  94%|█████████▍| 189/200 [25:01<00:32,  2.91s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 188 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  98%|█████████▊| 197/200 [25:22<00:08,  2.69s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 196 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing: 100%|█████████▉| 199/200 [25:28<00:02,  2.95s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 198 YO Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing: 100%|██████████| 200/200 [25:31<00:00,  7.66s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Accuracy: 132/200\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "accurate_predictions = 0;\n",
    "print_process = True;\n",
    "basic_binary_prompt = 'Analyze the question first, Provide the final answer, which should be a yes or no. Never say you dont know, always provide a yes or no answer regardless. If you dont have enough information, just guess. You must Follow this format:\\nAnalysis: your analysis\\nFinal Answer: no'\n",
    "basic_mc_prompt = 'Analyze the question first, then provide the final answer, which should be a single letter in the alphabet representing the best option among the multiple choices provided in the question. Follow this format:\\nAnalysis: your analysis\\nFinal Answer: A'\n",
    "basic_text_prompt = 'Analyze the question first, then provide the final answer, which should be a single term consisting of english words. Follow this format:\\nAnalysis: your analysis\\nFinal Answer: the adverse drug reaction you identified'\n",
    "basic_double_binary_prompt = 'Your task is to answer to questions. Analyze the questions first, then provide the final answer, which should consist of two terms, each being either \"yes\" or \"no\" corresponding to the questions. Never say you dont know, always provide a yes or no answer regardless. If you dont have enough information, just guess. Follow this format:\\n\\nAnalysis: your analysis\\nFinal Answer: no, no'\n",
    "basic_text_list_prompt = '''Your task is to identify a list of medications. Analyze the question first, then provide the final answer, which should consist of drug names, each separated by a comma. Never say you don't know, always provide an answer regardless. If you don't have enough information, just guess. Follow this format:\n",
    "Analysis: your analysis\n",
    "Final Answer: DrugA, DrugB, DrugC, ...\n",
    "'''\n",
    "CoT_list = []\n",
    "QA_list = []\n",
    "precision_list = []\n",
    "recall_list = []\n",
    "f1_list = []\n",
    "for i in tqdm(range(len(evaluation_set['sample'])), desc=\"Processing\"):\n",
    "  chat_response = chat_handler(evaluation_set['sample'][i], basic_binary_prompt, [], GPT_3)\n",
    "  analysis, prediction = binary_parser(chat_response)\n",
    "  CoT_list.append(chat_response)\n",
    "  QA_list.append(prediction)\n",
    "  if dataset_name == 'ade':\n",
    "    if check_text_accuracy(prediction, evaluation_set['label'][i].lower()):\n",
    "      accurate_predictions += 1\n",
    "    else:\n",
    "      print(f\"\\nWrong Answer Index: {i} Question: {evaluation_set['sample'][i]} Chat Response: {prediction}, Answer: {evaluation_set['label'][i]}\")\n",
    "  elif dataset_name == 'drug_usage':\n",
    "      # Extract the binary yes/no answers from the chat response\n",
    "      answer1, answer2 = prediction.split(\", \")\n",
    "      label1, label2 = evaluation_set['label'][i].lower().split(\", \")\n",
    "      # Check the accuracy of each part of the answer\n",
    "      if answer1.lower() == label1 and answer2.lower() == label2:\n",
    "        accurate_predictions += 1\n",
    "      else:\n",
    "        print(f\"\\nWrong Answer Index: {i} Question: {evaluation_set['sample'][i]} Chat Response: {chat_response}, Answer: {evaluation_set['label'][i]}\")\n",
    "  elif dataset_name == 'chatDoctor':\n",
    "      # Extract the binary yes/no answers from the chat response\n",
    "        precision, recall, f1 = calculate_f1_metrics(prediction, evaluation_set['label'][i].lower())\n",
    "        precision_list.append(precision)\n",
    "        recall_list.append(recall)\n",
    "        f1_list.append(f1)\n",
    "  else:\n",
    "    if prediction == evaluation_set['label'][i].lower():\n",
    "      accurate_predictions += 1\n",
    "    else:\n",
    "      print(f\"\\nWrong Answer Index: {i} YO Chat Response: {prediction}, Answer: {evaluation_set['label'][i]}\")\n",
    "\n",
    "data = {\n",
    "      'Questions': evaluation_set['sample'],\n",
    "      'Long Answer': CoT_list,\n",
    "      'Short Answer': QA_list,\n",
    "      'Official Answer': evaluation_set['label']\n",
    "    }\n",
    "# Convert the dictionary to a DataFrame\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# Save the DataFrame to a csv file\n",
    "df.to_csv(data_paths_dict[dataset_name]['GPT_3_answer'], index=False)\n",
    "print(f\"\\nAccuracy: {accurate_predictions}/{len(evaluation_set['sample']) }\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_5NslBfgJIAf"
   },
   "source": [
    "### GPT-4 Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6tqF5IMeeP21"
   },
   "source": [
    "#### Without CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZMZafy8oJDW8",
    "outputId": "ffec3567-9382-496b-a221-b050c10b7a30"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250\n",
      "we have 250 of A and 250 of B and 250 of C and 250 of D in the loaded medmcqa evaluation set\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:   1%|          | 8/1000 [00:11<23:18,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 7 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:   3%|▎         | 27/1000 [00:37<20:06,  1.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 26 Chat Response: Final Answer: C, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:   3%|▎         | 28/1000 [00:39<20:39,  1.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 27 Chat Response: Final Answer: C, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:   4%|▎         | 35/1000 [00:50<23:56,  1.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 34 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:   4%|▍         | 38/1000 [00:54<25:42,  1.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 37 Chat Response: Final Answer: C, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:   5%|▌         | 50/1000 [01:08<19:48,  1.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 49 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:   5%|▌         | 51/1000 [01:10<20:33,  1.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 50 Chat Response: Final Answer: C, Answer: A\n",
      "Chat API error: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID a0616f7b2be72004dc12c2e9de6ba781 in your message.)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:   6%|▌         | 57/1000 [02:53<1:51:09,  7.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 56 Chat Response: As an AI, I am not able to view images or interpret ECG results. Please provide the information from the ECG, and I will try to help answer the question., Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:   8%|▊         | 82/1000 [03:25<18:50,  1.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 81 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:   8%|▊         | 83/1000 [03:28<27:10,  1.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 82 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:   8%|▊         | 84/1000 [03:29<24:48,  1.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 83 Chat Response: Final Answer: C, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:   9%|▉         | 91/1000 [03:37<16:21,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 90 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:   9%|▉         | 94/1000 [03:40<16:30,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 93 Chat Response: Final Answer: C, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  10%|▉         | 97/1000 [03:43<15:31,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 96 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  10%|▉         | 99/1000 [03:46<17:07,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chat API error: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 49cbfa450388d8ee5271b18d437b9a0c in your message.)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  10%|█         | 104/1000 [05:23<1:55:19,  7.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chat API error: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 869b2372eb7db511d8bbdee1d7ec6986 in your message.)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  11%|█         | 111/1000 [07:02<1:13:22,  4.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 110 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  12%|█▏        | 123/1000 [07:15<16:11,  1.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chat API error: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID d4dfb132fdc6f5d31bc3344503fdd6b4 in your message.)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  13%|█▎        | 129/1000 [08:52<1:22:47,  5.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 128 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  15%|█▍        | 149/1000 [09:15<17:46,  1.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 148 Chat Response: Final Answer: D, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  15%|█▌        | 154/1000 [09:20<14:01,  1.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 153 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  16%|█▌        | 156/1000 [09:22<14:42,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 155 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  17%|█▋        | 166/1000 [09:32<15:42,  1.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 165 Chat Response: Final Answer: C, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  18%|█▊        | 176/1000 [09:43<14:48,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 175 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  18%|█▊        | 178/1000 [09:45<16:06,  1.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 177 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  18%|█▊        | 180/1000 [09:48<15:49,  1.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 179 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  20%|█▉        | 196/1000 [10:07<14:55,  1.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 195 Chat Response: Final Answer: D, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  20%|█▉        | 197/1000 [10:08<14:36,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 196 Chat Response: Final Answer: D, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  20%|█▉        | 199/1000 [10:10<14:04,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 198 Chat Response: Final Answer: D, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  21%|██        | 210/1000 [10:24<14:57,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chat API error: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 90d8873c11481c2bdeb165cd4dfebbe1 in your message.)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  21%|██▏       | 213/1000 [11:57<3:07:13, 14.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 212 Chat Response: Final Answer: D, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  23%|██▎       | 226/1000 [12:12<15:31,  1.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 225 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  23%|██▎       | 231/1000 [12:17<13:24,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 230 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  24%|██▍       | 243/1000 [12:30<12:25,  1.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 242 Chat Response: Final Answer: C, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  25%|██▍       | 249/1000 [12:38<15:03,  1.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 248 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  25%|██▌       | 250/1000 [12:39<14:17,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 249 Chat Response: Final Answer: B, Answer: A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  26%|██▌       | 262/1000 [12:51<13:48,  1.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 261 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  26%|██▋       | 263/1000 [12:52<12:52,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 262 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  26%|██▋       | 264/1000 [12:53<12:40,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 263 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  28%|██▊       | 275/1000 [13:11<29:48,  2.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 274 Chat Response: It is not possible for me to view the ECG image, as I am an AI language model and cannot process images. Please provide a description of the ECG or the condition it represents to help me provide an accurate answer., Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  28%|██▊       | 284/1000 [13:20<12:53,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 283 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  29%|██▊       | 286/1000 [13:23<13:02,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 285 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  30%|██▉       | 295/1000 [13:32<13:08,  1.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 294 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  30%|██▉       | 296/1000 [13:33<13:17,  1.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 295 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  31%|███       | 311/1000 [13:52<13:54,  1.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 310 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  32%|███▏      | 321/1000 [14:05<14:15,  1.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 320 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  32%|███▏      | 324/1000 [14:08<13:20,  1.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chat API error: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID bce23e348ba2455c6a2b4a0544add7f8 in your message.)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  33%|███▎      | 328/1000 [15:43<1:56:14, 10.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 327 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  33%|███▎      | 332/1000 [15:49<42:50,  3.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 331 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  34%|███▎      | 335/1000 [15:52<22:51,  2.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 334 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  34%|███▎      | 336/1000 [15:53<18:56,  1.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 335 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  34%|███▍      | 338/1000 [15:55<15:07,  1.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 337 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  34%|███▍      | 341/1000 [15:58<12:36,  1.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 340 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  34%|███▍      | 344/1000 [16:01<11:37,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 343 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  34%|███▍      | 345/1000 [16:02<11:16,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 344 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  35%|███▌      | 351/1000 [16:08<10:43,  1.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 350 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  35%|███▌      | 352/1000 [16:09<10:35,  1.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 351 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  36%|███▋      | 364/1000 [16:30<13:42,  1.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 363 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  37%|███▋      | 368/1000 [16:34<11:24,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 367 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  37%|███▋      | 373/1000 [16:39<10:37,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chat API error: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID f289232da84144581648551bc46b8c36 in your message.)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  38%|███▊      | 378/1000 [18:15<1:19:04,  7.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 377 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  38%|███▊      | 381/1000 [18:19<34:53,  3.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 380 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  38%|███▊      | 384/1000 [18:24<21:24,  2.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chat API error: Request failed due to server shutdown {\n",
      "  \"error\": {\n",
      "    \"message\": \"Request failed due to server shutdown\",\n",
      "    \"type\": \"server_error\",\n",
      "    \"param\": null,\n",
      "    \"code\": null\n",
      "  }\n",
      "}\n",
      " 500 {'error': {'message': 'Request failed due to server shutdown', 'type': 'server_error', 'param': None, 'code': None}} {'Date': 'Sat, 10 Jun 2023 11:38:33 GMT', 'Content-Type': 'application/json', 'Content-Length': '141', 'Connection': 'keep-alive', 'access-control-allow-origin': '*', 'openai-model': 'gpt-4-0314', 'openai-organization': 'user-so5pfryxritwlv4sc3ijfnpe', 'openai-processing-ms': '18904', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15724800; includeSubDomains', 'x-ratelimit-limit-requests': '200', 'x-ratelimit-limit-tokens': '40000', 'x-ratelimit-remaining-requests': '199', 'x-ratelimit-remaining-tokens': '39873', 'x-ratelimit-reset-requests': '300ms', 'x-ratelimit-reset-tokens': '190ms', 'x-request-id': '53f35b955e222f0391260a8c0cd7a3f4', 'CF-Cache-Status': 'DYNAMIC', 'Server': 'cloudflare', 'CF-RAY': '7d5153ac9c21a3ac-TPE', 'alt-svc': 'h3=\":443\"; ma=86400'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  38%|███▊      | 385/1000 [18:44<1:16:43,  7.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 384 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  39%|███▉      | 391/1000 [18:50<18:35,  1.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 390 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  39%|███▉      | 392/1000 [18:51<16:05,  1.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 391 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  40%|███▉      | 396/1000 [18:56<11:57,  1.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 395 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  40%|████      | 405/1000 [19:06<11:08,  1.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 404 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  41%|████      | 409/1000 [19:10<12:07,  1.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 408 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  41%|████▏     | 414/1000 [19:16<11:49,  1.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 413 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  42%|████▏     | 415/1000 [19:18<12:55,  1.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 414 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  43%|████▎     | 429/1000 [19:34<10:19,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 428 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  43%|████▎     | 432/1000 [19:37<10:00,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 431 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  44%|████▎     | 435/1000 [19:40<09:42,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 434 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  44%|████▍     | 444/1000 [19:51<10:11,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 443 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  45%|████▍     | 449/1000 [19:57<10:34,  1.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 448 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  45%|████▌     | 450/1000 [19:58<10:39,  1.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 449 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  45%|████▌     | 451/1000 [19:59<09:56,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 450 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  45%|████▌     | 452/1000 [20:00<09:43,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 451 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  46%|████▌     | 457/1000 [20:07<13:42,  1.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 456 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  47%|████▋     | 466/1000 [20:18<10:07,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 465 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  47%|████▋     | 467/1000 [20:19<10:25,  1.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 466 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  47%|████▋     | 471/1000 [20:23<09:45,  1.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 470 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  49%|████▊     | 486/1000 [20:44<17:19,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 485 Chat Response: I am an AI language model and I cannot see images or interpret visual information. Please provide a description of the rhythm disorder so that I can help you with the best drug choice., Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  49%|████▊     | 487/1000 [20:46<18:06,  2.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 486 Chat Response: Final Answer: C, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  50%|████▉     | 495/1000 [20:56<10:57,  1.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 494 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  50%|████▉     | 497/1000 [20:58<09:31,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 496 Chat Response: Final Answer: D, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  50%|█████     | 500/1000 [21:01<08:57,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 499 Chat Response: Final Answer: A, Answer: B\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  50%|█████     | 501/1000 [21:02<08:50,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 500 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  50%|█████     | 502/1000 [21:03<08:46,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 501 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  50%|█████     | 504/1000 [21:06<10:40,  1.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 503 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  51%|█████     | 506/1000 [21:08<09:35,  1.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 505 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  51%|█████     | 507/1000 [21:09<09:33,  1.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 506 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  51%|█████     | 510/1000 [21:13<09:15,  1.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 509 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  52%|█████▏    | 520/1000 [21:24<08:55,  1.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 519 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  52%|█████▏    | 522/1000 [21:27<09:35,  1.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 521 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  52%|█████▏    | 524/1000 [21:29<09:07,  1.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 523 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  53%|█████▎    | 528/1000 [21:33<08:30,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 527 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  53%|█████▎    | 534/1000 [21:42<12:43,  1.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 533 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  54%|█████▎    | 536/1000 [21:44<10:04,  1.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 535 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  54%|█████▍    | 540/1000 [21:49<08:58,  1.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chat API error: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 6f10d4af62ba0c05184674aeac8111c6 in your message.)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  55%|█████▌    | 550/1000 [23:29<15:29,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 549 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  55%|█████▌    | 551/1000 [23:30<13:49,  1.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 550 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  55%|█████▌    | 554/1000 [23:34<10:15,  1.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 553 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  56%|█████▌    | 557/1000 [23:38<09:03,  1.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 556 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  56%|█████▌    | 559/1000 [23:39<08:01,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 558 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  56%|█████▋    | 565/1000 [23:45<07:22,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 564 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  57%|█████▋    | 567/1000 [23:48<07:52,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 566 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  57%|█████▋    | 570/1000 [23:51<08:09,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 569 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  57%|█████▋    | 574/1000 [23:56<09:07,  1.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 573 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  58%|█████▊    | 576/1000 [23:58<08:06,  1.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 575 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  58%|█████▊    | 579/1000 [24:02<08:07,  1.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 578 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  58%|█████▊    | 582/1000 [24:05<07:52,  1.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 581 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  59%|█████▉    | 588/1000 [24:12<07:34,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 587 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  59%|█████▉    | 589/1000 [24:13<07:34,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 588 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  60%|█████▉    | 597/1000 [24:23<07:21,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 596 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  60%|██████    | 605/1000 [24:31<07:16,  1.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 604 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  61%|██████    | 609/1000 [24:36<07:14,  1.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 608 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  61%|██████    | 612/1000 [24:39<07:17,  1.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 611 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  63%|██████▎   | 627/1000 [24:56<06:19,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 626 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  63%|██████▎   | 628/1000 [24:57<06:09,  1.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 627 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  64%|██████▎   | 636/1000 [25:05<06:10,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 635 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  64%|██████▍   | 640/1000 [25:10<06:50,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 639 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  65%|██████▍   | 647/1000 [25:18<07:02,  1.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 646 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  66%|██████▌   | 658/1000 [25:30<05:47,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 657 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  66%|██████▋   | 664/1000 [25:36<05:52,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 663 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  67%|██████▋   | 673/1000 [25:57<08:21,  1.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 672 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  68%|██████▊   | 682/1000 [26:09<07:56,  1.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 681 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  68%|██████▊   | 684/1000 [26:12<07:14,  1.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 683 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  69%|██████▉   | 688/1000 [26:16<06:22,  1.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 687 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  69%|██████▉   | 694/1000 [26:22<05:18,  1.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 693 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  70%|██████▉   | 695/1000 [26:24<05:58,  1.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 694 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  70%|██████▉   | 696/1000 [26:25<05:28,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 695 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  70%|███████   | 702/1000 [26:31<05:17,  1.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 701 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  70%|███████   | 703/1000 [26:33<05:56,  1.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 702 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  71%|███████   | 709/1000 [26:40<06:14,  1.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 708 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  71%|███████   | 712/1000 [26:43<05:18,  1.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 711 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  71%|███████▏  | 714/1000 [26:46<05:18,  1.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 713 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  72%|███████▏  | 718/1000 [26:50<04:55,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 717 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  72%|███████▏  | 722/1000 [26:54<04:51,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 721 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  72%|███████▎  | 725/1000 [26:57<04:48,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 724 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  73%|███████▎  | 730/1000 [27:03<04:54,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 729 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  73%|███████▎  | 731/1000 [27:04<04:44,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 730 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  74%|███████▎  | 735/1000 [27:13<09:50,  2.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 734 Chat Response: As an AI language model, I am not able to diagnose a patient without detailed information and a proper medical examination. It is recommended to consult a healthcare professional for an accurate diagnosis., Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  74%|███████▎  | 737/1000 [27:15<07:17,  1.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 736 Chat Response: Final Answer: D, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  74%|███████▍  | 739/1000 [27:17<05:53,  1.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 738 Chat Response: Final Answer: B, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  74%|███████▍  | 741/1000 [27:19<05:00,  1.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 740 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  74%|███████▍  | 743/1000 [27:22<06:02,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 742 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  74%|███████▍  | 744/1000 [27:24<05:36,  1.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 743 Chat Response: Final Answer: A, Answer: C\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  75%|███████▌  | 751/1000 [27:31<04:49,  1.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 750 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  76%|███████▌  | 761/1000 [27:41<04:10,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 760 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  77%|███████▋  | 772/1000 [27:53<04:20,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 771 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  77%|███████▋  | 774/1000 [27:57<05:29,  1.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 773 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  80%|███████▉  | 797/1000 [28:23<03:42,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 796 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  80%|███████▉  | 799/1000 [28:25<03:26,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chat API error: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID f270a90be5430ddadd0bd01c458874fd in your message.)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  80%|████████  | 801/1000 [29:58<1:06:23, 20.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 800 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  80%|████████  | 802/1000 [29:59<47:17, 14.33s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 801 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  80%|████████  | 804/1000 [30:02<25:10,  7.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 803 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  81%|████████  | 806/1000 [30:04<14:13,  4.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 805 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  81%|████████  | 811/1000 [30:10<05:36,  1.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 810 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  82%|████████▏ | 818/1000 [30:18<03:27,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 817 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  82%|████████▎ | 825/1000 [30:28<03:31,  1.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 824 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  83%|████████▎ | 826/1000 [30:29<03:23,  1.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 825 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  83%|████████▎ | 827/1000 [30:30<03:06,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 826 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  83%|████████▎ | 829/1000 [30:32<03:01,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 828 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  83%|████████▎ | 834/1000 [30:37<02:59,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 833 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  84%|████████▎ | 837/1000 [30:40<02:41,  1.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 836 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  84%|████████▍ | 840/1000 [30:43<02:37,  1.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 839 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  84%|████████▍ | 843/1000 [30:46<02:31,  1.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 842 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  84%|████████▍ | 845/1000 [30:48<02:29,  1.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 844 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  86%|████████▌ | 861/1000 [31:05<02:28,  1.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 860 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  86%|████████▌ | 862/1000 [31:06<02:31,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 861 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  86%|████████▋ | 865/1000 [31:10<02:30,  1.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 864 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  87%|████████▋ | 870/1000 [31:15<02:19,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 869 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  87%|████████▋ | 872/1000 [31:17<02:16,  1.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 871 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  88%|████████▊ | 876/1000 [31:21<02:13,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 875 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  88%|████████▊ | 884/1000 [31:30<02:02,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 883 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  88%|████████▊ | 885/1000 [31:31<01:58,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 884 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  89%|████████▊ | 886/1000 [31:32<01:56,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 885 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  89%|████████▉ | 891/1000 [31:38<01:55,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 890 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  89%|████████▉ | 892/1000 [31:39<01:50,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 891 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  89%|████████▉ | 893/1000 [31:40<01:46,  1.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 892 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  90%|████████▉ | 896/1000 [31:43<01:49,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 895 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  90%|████████▉ | 897/1000 [31:44<01:46,  1.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 896 Chat Response: Final Answer: B, Answer: D\n",
      "Chat API error: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID ff8adba9188d805888ba22b46cd2c888 in your message.)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  90%|█████████ | 900/1000 [33:18<24:03, 14.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 899 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  90%|█████████ | 904/1000 [33:22<06:52,  4.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 903 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  91%|█████████ | 909/1000 [33:28<02:26,  1.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 908 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  91%|█████████ | 910/1000 [33:30<02:33,  1.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 909 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  92%|█████████▏| 915/1000 [33:35<01:34,  1.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 914 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  92%|█████████▏| 916/1000 [33:36<01:31,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 915 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  92%|█████████▏| 922/1000 [33:42<01:18,  1.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 921 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  93%|█████████▎| 927/1000 [33:49<01:42,  1.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 926 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  93%|█████████▎| 929/1000 [33:51<01:31,  1.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 928 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  93%|█████████▎| 931/1000 [33:53<01:15,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 930 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  94%|█████████▍| 944/1000 [34:07<01:01,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 943 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  95%|█████████▍| 946/1000 [34:09<00:56,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 945 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  95%|█████████▍| 947/1000 [34:13<01:48,  2.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 946 Chat Response: I cannot provide a final answer without more context or information about the patient's symptoms, medical history, and test results. Please provide more details to help me determine the most likely diagnosis., Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  95%|█████████▌| 950/1000 [34:17<01:10,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 949 Chat Response: Final Answer: A, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  96%|█████████▌| 956/1000 [34:23<00:46,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 955 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  96%|█████████▌| 960/1000 [34:27<00:48,  1.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 959 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  97%|█████████▋| 968/1000 [34:38<00:40,  1.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 967 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  98%|█████████▊| 976/1000 [34:52<00:34,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 975 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  98%|█████████▊| 977/1000 [34:53<00:29,  1.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 976 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  98%|█████████▊| 978/1000 [34:54<00:25,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 977 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  98%|█████████▊| 979/1000 [34:55<00:21,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 978 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  99%|█████████▉| 992/1000 [35:09<00:08,  1.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 991 Chat Response: Final Answer: C, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\rProcessing:  99%|█████████▉| 993/1000 [35:10<00:07,  1.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wrong Answer Index: 992 Chat Response: Final Answer: B, Answer: D\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing: 100%|██████████| 1000/1000 [35:18<00:00,  2.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Accuracy: 802/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "accurate_predictions = 0;\n",
    "print_process = True;\n",
    "basic_mc_prompt = 'Provide the final answer, which should be a single letter in the alphabet representing the best option among the multiple choices provided in the question. Follow this format:\\nFinal Answer: A'\n",
    "basic_binary_prompt = 'Provide the final answer, which should be a yes or no. Never say you dont know, always provide a yes or no answer regardless. If you dont have enough information, just guess. Follow this format:\\nFinal Answer: yes'\n",
    "for i in tqdm(range(len(evaluation_set['sample']) ), desc=\"Processing\"):\n",
    "  chat_response = chat_handler(evaluation_set['sample'][i], basic_binary_prompt, [], GPT_4)\n",
    "  analysis, prediction = binary_parser(chat_response)\n",
    "  if dataset_name == 'ade':\n",
    "    if check_text_accuracy(prediction, evaluation_set['label'][i].lower()):\n",
    "      accurate_predictions += 1\n",
    "    else:\n",
    "      print(f\"\\nWrong Answer Index: {i} Question: {evaluation_set['sample'][i]} Chat Response: {prediction}, Answer: {evaluation_set['label'][i]}\")\n",
    "  else:\n",
    "    prediction = chat_response.split(\".\")[0].lower()\n",
    "    if prediction == evaluation_set['label'][i].lower():\n",
    "      accurate_predictions += 1\n",
    "    else:\n",
    "      print(f\"\\nWrong Answer Index: {i} Chat Response: {chat_response}, Answer: {evaluation_set['label'][i]}\")\n",
    "print(f\"\\nAccuracy: {accurate_predictions}/{len(evaluation_set['sample']) }\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pu8W6edVeSo7"
   },
   "source": [
    "#### With CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JbO93nrJeV-D",
    "outputId": "e20c9c97-1a11-4c6f-9d9d-dbbe2c25bd97"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:   1%|          | 2/200 [00:14<24:41,  7.48s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 1 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   2%|▏         | 3/200 [00:20<21:13,  6.46s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 2 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   2%|▏         | 4/200 [00:25<20:07,  6.16s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 3 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   2%|▎         | 5/200 [00:31<18:57,  5.83s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 4 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   3%|▎         | 6/200 [00:36<18:34,  5.74s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 5 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   4%|▎         | 7/200 [00:41<17:22,  5.40s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 6 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   4%|▍         | 8/200 [00:46<17:28,  5.46s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 7 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   4%|▍         | 9/200 [00:52<17:55,  5.63s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 8 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   5%|▌         | 10/200 [00:59<18:35,  5.87s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 9 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   6%|▌         | 11/200 [01:03<17:13,  5.47s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 10 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   6%|▌         | 12/200 [01:13<20:47,  6.64s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 11 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   6%|▋         | 13/200 [01:21<22:32,  7.23s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 12 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   7%|▋         | 14/200 [01:27<21:20,  6.88s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 13 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   8%|▊         | 15/200 [01:32<19:13,  6.24s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 14 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   8%|▊         | 16/200 [01:37<17:30,  5.71s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 15 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   8%|▊         | 17/200 [01:43<17:59,  5.90s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 16 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:   9%|▉         | 18/200 [01:51<20:06,  6.63s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 17 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  10%|▉         | 19/200 [01:56<18:09,  6.02s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 18 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  10%|█         | 21/200 [02:08<17:38,  5.91s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 20 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  11%|█         | 22/200 [02:13<16:49,  5.67s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 21 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  12%|█▏        | 23/200 [02:19<17:09,  5.82s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 22 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  12%|█▏        | 24/200 [02:25<17:27,  5.95s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 23 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  12%|█▎        | 25/200 [02:31<17:00,  5.83s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 24 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  13%|█▎        | 26/200 [02:38<17:50,  6.15s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 25 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  14%|█▎        | 27/200 [02:44<18:12,  6.31s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 26 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  14%|█▍        | 28/200 [02:50<17:35,  6.13s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 27 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  14%|█▍        | 29/200 [02:57<17:57,  6.30s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 28 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  15%|█▌        | 30/200 [03:01<16:31,  5.83s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 29 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  16%|█▌        | 31/200 [03:07<15:46,  5.60s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 30 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  16%|█▌        | 32/200 [03:12<15:43,  5.61s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 31 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  16%|█▋        | 33/200 [03:20<17:39,  6.34s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 32 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  17%|█▋        | 34/200 [03:25<16:41,  6.03s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 33 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  18%|█▊        | 35/200 [03:32<16:57,  6.17s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 34 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  18%|█▊        | 36/200 [03:36<15:10,  5.55s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 35 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  18%|█▊        | 37/200 [03:42<15:34,  5.73s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 36 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  19%|█▉        | 38/200 [03:47<14:59,  5.55s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 37 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  20%|█▉        | 39/200 [03:56<17:21,  6.47s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 38 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  20%|██        | 41/200 [04:10<17:29,  6.60s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 40 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  21%|██        | 42/200 [04:18<18:22,  6.98s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 41 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  22%|██▏       | 43/200 [04:25<18:08,  6.93s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 42 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  22%|██▏       | 44/200 [04:31<17:15,  6.64s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 43 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  23%|██▎       | 46/200 [04:46<17:35,  6.85s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 45 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  24%|██▎       | 47/200 [04:51<16:17,  6.39s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 46 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  24%|██▍       | 48/200 [04:58<16:57,  6.70s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 47 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  24%|██▍       | 49/200 [05:03<15:36,  6.20s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 48 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  25%|██▌       | 50/200 [05:12<17:06,  6.84s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 49 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  26%|██▌       | 51/200 [05:19<17:15,  6.95s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 50 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  26%|██▌       | 52/200 [05:25<16:26,  6.67s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 51 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  26%|██▋       | 53/200 [05:30<15:26,  6.30s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 52 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  27%|██▋       | 54/200 [05:36<14:33,  5.98s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 53 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  28%|██▊       | 55/200 [05:41<14:16,  5.91s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 54 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  28%|██▊       | 57/200 [05:59<17:22,  7.29s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 56 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  29%|██▉       | 58/200 [06:03<15:19,  6.48s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 57 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  30%|██▉       | 59/200 [06:09<14:26,  6.15s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 58 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  30%|███       | 60/200 [06:14<13:32,  5.80s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 59 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  30%|███       | 61/200 [06:19<13:18,  5.74s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 60 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  31%|███       | 62/200 [06:25<12:53,  5.61s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 61 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  32%|███▏      | 64/200 [06:36<12:58,  5.72s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 63 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  32%|███▎      | 65/200 [06:41<12:17,  5.46s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 64 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  33%|███▎      | 66/200 [06:53<16:19,  7.31s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 65 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  34%|███▎      | 67/200 [07:00<15:43,  7.09s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 66 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  34%|███▍      | 68/200 [07:13<19:36,  8.91s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 67 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  34%|███▍      | 69/200 [07:18<17:17,  7.92s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 68 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  35%|███▌      | 70/200 [07:24<15:41,  7.24s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 69 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  36%|███▌      | 72/200 [07:38<14:45,  6.92s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 71 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  36%|███▋      | 73/200 [07:44<14:27,  6.83s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 72 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  37%|███▋      | 74/200 [07:50<13:26,  6.40s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 73 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  38%|███▊      | 75/200 [07:57<13:59,  6.72s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 74 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  38%|███▊      | 77/200 [08:07<12:00,  5.85s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 76 Chat Response: no, Answer: yes\n",
      "Chat API error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 23 Jun 2023 03:38:11 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7db9aaa1985c4a68-TPE', 'alt-svc': 'h3=\":443\"; ma=86400'}\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  39%|███▉      | 78/200 [13:32<3:26:26, 101.53s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 77 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  40%|███▉      | 79/200 [13:38<2:26:50, 72.81s/it] "
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 78 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  40%|████      | 80/200 [13:44<1:45:43, 52.87s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 79 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  41%|████      | 82/200 [13:56<56:39, 28.81s/it]  "
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 81 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  42%|████▏     | 83/200 [14:05<44:25, 22.78s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 82 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  43%|████▎     | 86/200 [14:28<23:41, 12.47s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 85 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  44%|████▎     | 87/200 [14:33<19:18, 10.25s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 86 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  44%|████▍     | 89/200 [14:44<14:11,  7.67s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 88 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  45%|████▌     | 90/200 [14:51<14:04,  7.67s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 89 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  47%|████▋     | 94/200 [15:18<11:50,  6.70s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 93 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  48%|████▊     | 95/200 [15:26<12:20,  7.05s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 94 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  48%|████▊     | 96/200 [15:33<12:09,  7.01s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 95 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  48%|████▊     | 97/200 [15:40<12:10,  7.09s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 96 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  50%|████▉     | 99/200 [15:53<11:11,  6.65s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 98 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\rProcessing:  50%|█████     | 100/200 [16:02<11:55,  7.16s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 99 Chat Response: no, Answer: yes\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  66%|██████▌   | 131/200 [19:26<08:05,  7.04s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 130 Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  90%|█████████ | 180/200 [24:49<02:29,  7.47s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 179 Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing:  94%|█████████▍| 189/200 [25:48<01:13,  6.69s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Wrong Answer Index: 188 Chat Response: yes, Answer: no\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Processing: 100%|██████████| 200/200 [26:56<00:00,  8.08s/it]"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Accuracy: 113/200\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "accurate_predictions = 0;\n",
    "print_process = True;\n",
    "basic_binary_prompt = 'Analyze the question first, Provide the final answer, which should be a yes or no. Never say you dont know, always provide a yes or no answer regardless. If you dont have enough information, just guess. Follow this format:\\n Analysis: your analysis\\nFinal Answer: no'\n",
    "basic_mc_prompt = 'Analyze the question first, then provide the final answer, which should be a single letter in the alphabet representing the best option among the multiple choices provided in the question. Follow this format:\\nAnalysis: your analysis\\nFinal Answer: A'\n",
    "basic_text_prompt = 'Analyze the question first, then provide the final answer, which should be a single term consisting of english words. Follow this format:\\nAnalysis: your analysis\\nFinal Answer: the adverse drug reaction you identified'\n",
    "basic_double_binary_prompt = 'Your task is to answer to questions. Analyze the questions first, then provide the final answer, which should consist of two terms, each being either \"yes\" or \"no\" corresponding to the questions. Never say you dont know, always provide a yes or no answer regardless. If you dont have enough information, just guess. Follow this format:\\n\\nAnalysis: your analysis\\nFinal Answer: no, no'\n",
    "basic_text_list_prompt = '''Your task is to identify a list of medications. Analyze the question first, then provide the final answer, which should consist of drug names, each separated by a comma. Never say you don't know, always provide an answer regardless. If you don't have enough information, just guess. Follow this format:\n",
    "Analysis: your analysis\n",
    "Final Answer: DrugA, DrugB, DrugC, ...\n",
    "'''\n",
    "CoT_list = []\n",
    "QA_list = []\n",
    "precision_list = []\n",
    "recall_list = []\n",
    "f1_list = []\n",
    "for i in tqdm(range(len(evaluation_set['sample'])), desc=\"Processing\"):\n",
    "  chat_response = chat_handler(evaluation_set['sample'][i], basic_binary_prompt, [], GPT_4)\n",
    "  analysis, prediction = text_parser(chat_response)\n",
    "  CoT_list.append(chat_response)\n",
    "  QA_list.append(prediction)\n",
    "  if dataset_name == 'ade':\n",
    "    if check_text_accuracy(prediction, evaluation_set['label'][i].lower()):\n",
    "      accurate_predictions += 1\n",
    "    else:\n",
    "      print(f\"\\nWrong Answer Index: {i} Question: {evaluation_set['sample'][i]} Chat Response: {prediction}, Answer: {evaluation_set['label'][i]}\")\n",
    "  elif dataset_name == 'drug_usage':\n",
    "      # Extract the binary yes/no answers from the chat response\n",
    "      answer1, answer2 = prediction.split(\", \")\n",
    "      label1, label2 = evaluation_set['label'][i].lower().split(\", \")\n",
    "      # Check the accuracy of each part of the answer\n",
    "      if answer1.lower() == label1 and answer2.lower() == label2:\n",
    "        accurate_predictions += 1\n",
    "      else:\n",
    "        print(f\"\\nWrong Answer Index: {i} Question: {evaluation_set['sample'][i]} Chat Response: {chat_response}, Answer: {evaluation_set['label'][i]}\")\n",
    "  elif dataset_name == 'chatDoctor':\n",
    "      # Extract the binary yes/no answers from the chat response\n",
    "        precision, recall, f1 = calculate_f1_metrics(prediction, evaluation_set['label'][i].lower())\n",
    "        precision_list.append(precision)\n",
    "        recall_list.append(recall)\n",
    "        f1_list.append(f1)\n",
    "  else:\n",
    "    if prediction == evaluation_set['label'][i].lower():\n",
    "      accurate_predictions += 1\n",
    "    else:\n",
    "      print(f\"\\nWrong Answer Index: {i} Chat Response: {prediction}, Answer: {evaluation_set['label'][i]}\")\n",
    "\n",
    "data = {\n",
    "      'Questions': evaluation_set['sample'],\n",
    "      'Long Answer': CoT_list,\n",
    "      'Short Answer': QA_list,\n",
    "      'Official Answer': evaluation_set['label']\n",
    "    }\n",
    "# Convert the dictionary to a DataFrame\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# Save the DataFrame to a csv file\n",
    "df.to_csv(data_paths_dict[dataset_name]['GPT_4_answer'], index=False)\n",
    "print(f\"\\nAccuracy: {accurate_predictions}/{len(evaluation_set['sample']) }\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "Pm73fPy1Xg_5"
   ],
   "machine_shape": "hm",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "097ba6dcaaef4dba98c01d15ba25d6ba": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0a46be3677494945a84aa654de56d877": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0e9498ce5ae14669be87a0fa80381d65": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "11ebb2f403924051956170ef5c7c2be4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7152e9a8e75f4a5aa4b7d23f24eb0e91",
      "placeholder": "​",
      "style": "IPY_MODEL_3ddda8a0f98241d79c47c8dff7c50d33",
      "value": " 26.8k/26.8k [00:00&lt;00:00, 2.19MB/s]"
     }
    },
    "15f6872f5dac4259b82b98194116f56c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bd5a150beb6d4d0a80a3cce82f07e520",
      "placeholder": "​",
      "style": "IPY_MODEL_27003b7f49fd4ce4a10f8adcd819edac",
      "value": " 3.50G/3.50G [00:10&lt;00:00, 330MB/s]"
     }
    },
    "1d5a445e405849bdad5e2faeddf8cbd6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1f87c66caa37430c86e04d3fb4a8c020": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "20bb0900a307434b9fb336f669da7fff": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "21507ac6372f42f6b7b194cc709bbc3e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8d57df5df31746eba7f7bdf0fcb4dd84",
      "max": 727,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_79591c8fb8014aea94d33f1347a9915f",
      "value": 727
     }
    },
    "216e741c53a64e65ad64cbbfb21e796a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2418d04ed0d44f558a403d069cb3e1fe": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f42522d00baa4d1eae90f0e4584da1b8",
      "max": 26788,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8e683c5f82d74427a595cdfa56d0d124",
      "value": 26788
     }
    },
    "27003b7f49fd4ce4a10f8adcd819edac": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "28d8bd3d9bc64246adc0f8cc70983034": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d37911bf83f340b79fe514b6bb4f4b01",
       "IPY_MODEL_6643d3dfdbf045078f4d8e11a9de3469",
       "IPY_MODEL_dc3553016e2c4b3caeab479f3e480f34"
      ],
      "layout": "IPY_MODEL_9b20cfd864f14e4da4aaccbb40f2e292"
     }
    },
    "2dafdadad0114b59a165c4fe3d288dbd": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "33038e684d3c47dfa9b9a278e9cde393": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_67001f2daaf54e8eabbdc04ea04275bb",
      "max": 499723,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_87ec3d986f814458908757bd1449321b",
      "value": 499723
     }
    },
    "3359b345f16a46f3993502eb61ed5348": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f48e1c96f9f7483d9b531c0ebe0ee9a0",
      "placeholder": "​",
      "style": "IPY_MODEL_c709306d5725466ebbd4bf682e7aa06d",
      "value": "Downloading (…)okenizer_config.json: 100%"
     }
    },
    "346d121caddf4c3ea56c43c13ef73cd7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "360efb60c21b4fcf9aa780496e641860": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3785400cbd7f4dab9ac58f74be8f692d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_42c5c817a5584b12bb3a76611fc3936d",
      "max": 3500315539,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_523c71cd31114304a842fd3243e721e7",
      "value": 3500315539
     }
    },
    "3c902b51fc2e40af97b15de0abd54b9c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "3ddda8a0f98241d79c47c8dff7c50d33": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "40c0ad17f8264efe997d246e1616a58a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "42c5c817a5584b12bb3a76611fc3936d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4387ed1a4cef4c85b1992d055ab825b8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b8e386774e7c4170bfa753e5dd9778a4",
       "IPY_MODEL_6784a03f49fc4f5dacbe69859f87ddee",
       "IPY_MODEL_de3e0b1a3445415bba5a9c5ff83a855f"
      ],
      "layout": "IPY_MODEL_ee4286005ef24eecb2955e37942f3d90"
     }
    },
    "477ce1a400a74d898aef30253965ba6b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7a5ad88a07a64ad68044d25a5375e0ae",
      "max": 137,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_793e7a5bfbf24ff6808bb6f96680de6d",
      "value": 137
     }
    },
    "48600bf2d24a4aaab955e611e18e9952": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_7ad3b17aca59487a91d2f5f59c638fda",
       "IPY_MODEL_74037e81c9304243beda7e64f5c6100f",
       "IPY_MODEL_c9ec7b904a45486aa96f8580d3aaac82"
      ],
      "layout": "IPY_MODEL_b43ea46be5644303b1d2f22043627a05"
     }
    },
    "49f643f587b441ae8b9ee779f86ae44f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4a7cdceeb332497c80886da527ddfa2a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4c4def410a41453c84ac94b551f928a8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "4d14fafcbec74844a2f4f2111b7ced83": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_85fc6c67ce084234ba634bd2952e0015",
      "placeholder": "​",
      "style": "IPY_MODEL_0a46be3677494945a84aa654de56d877",
      "value": " 500k/500k [00:00&lt;00:00, 31.4MB/s]"
     }
    },
    "4f52d67f052b4797b485c7096f86dbb6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5041f6ec5d4449a782e7ff77a89c2cc8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "523c71cd31114304a842fd3243e721e7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "5322ddb05d77460696d1807a33d6ff3e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5d28a0f39a1a4bf9814b9492004243f6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "64f8bd7f1dd64daa8a98ba1ae6ec1275": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1f87c66caa37430c86e04d3fb4a8c020",
      "placeholder": "​",
      "style": "IPY_MODEL_bcac5a4b41cf4498b7867f758a246e1a",
      "value": "Downloading (…)l-00002-of-00002.bin: 100%"
     }
    },
    "65057a4b0ee8488ebc197dee7382a361": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6643d3dfdbf045078f4d8e11a9de3469": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bc97f164235d4d9a8c5ce6ddff9ec9fc",
      "max": 2,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_826f4e459b414c97b7275b718071e485",
      "value": 2
     }
    },
    "67001f2daaf54e8eabbdc04ea04275bb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6784a03f49fc4f5dacbe69859f87ddee": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_86ccd77fc82b4553a494caa93fb71084",
      "max": 9976634558,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3c902b51fc2e40af97b15de0abd54b9c",
      "value": 9976634558
     }
    },
    "68a78f308c734052aa85d0a55e1f45a1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6e6a47ac7719476caeb5df399f8da236": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7152e9a8e75f4a5aa4b7d23f24eb0e91": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "74037e81c9304243beda7e64f5c6100f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_40c0ad17f8264efe997d246e1616a58a",
      "max": 554,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_4c4def410a41453c84ac94b551f928a8",
      "value": 554
     }
    },
    "768be564ad4d409da137a9ccc1d2f027": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "771458772ce74f9ea6f596b621ede833": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "793e7a5bfbf24ff6808bb6f96680de6d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "79591c8fb8014aea94d33f1347a9915f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "7a5ad88a07a64ad68044d25a5375e0ae": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7ad3b17aca59487a91d2f5f59c638fda": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d1ce1f773a724fe399d1e95b6296a9b7",
      "placeholder": "​",
      "style": "IPY_MODEL_771458772ce74f9ea6f596b621ede833",
      "value": "Downloading (…)lve/main/config.json: 100%"
     }
    },
    "818fad01927d452dbd0d56f6dbd115e7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "826f4e459b414c97b7275b718071e485": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "85fc6c67ce084234ba634bd2952e0015": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "86ccd77fc82b4553a494caa93fb71084": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "87ec3d986f814458908757bd1449321b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "8d57df5df31746eba7f7bdf0fcb4dd84": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8e683c5f82d74427a595cdfa56d0d124": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "9310586957b849ddb3ab45e10892f858": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "9435c0d2f5ba4923a4d00eb3608c72a0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9524696c0ce34053b7ee900761127282": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_cc933fc06094453e97fe2d60b0308bac",
      "placeholder": "​",
      "style": "IPY_MODEL_20bb0900a307434b9fb336f669da7fff",
      "value": "Downloading (…)model.bin.index.json: 100%"
     }
    },
    "95488ccb0e9e48af97bfbc7957281f51": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "963a1800cddc4da986620d0418573329": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f2aed2597c9547b0a3e6aad197445d49",
       "IPY_MODEL_c28deeaca3d546a0bc9ed2acd912f01e",
       "IPY_MODEL_c59e5979805a4c82982754de2b9619db"
      ],
      "layout": "IPY_MODEL_768be564ad4d409da137a9ccc1d2f027"
     }
    },
    "98490440e40f45d096ff5f09c643ddb7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9b20cfd864f14e4da4aaccbb40f2e292": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a91211fb034c4b7595e680f15634946d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "aeed353639234130a1c6d9ae0245f74d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b0502d359f1148a4b31e1c5e2412598e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b43ea46be5644303b1d2f22043627a05": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b6a0160136374c0b9c19dba9577b0e87": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a91211fb034c4b7595e680f15634946d",
      "placeholder": "​",
      "style": "IPY_MODEL_c8d6c83083a7411991333a7ee26c4b8f",
      "value": " 2/2 [00:39&lt;00:00, 18.12s/it]"
     }
    },
    "b78ab00b256e431f94113ac7f17a9f35": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3359b345f16a46f3993502eb61ed5348",
       "IPY_MODEL_21507ac6372f42f6b7b194cc709bbc3e",
       "IPY_MODEL_be25bd8f9c8843b98b79c25177f6bbbd"
      ],
      "layout": "IPY_MODEL_1d5a445e405849bdad5e2faeddf8cbd6"
     }
    },
    "b82337e55b784988b4b8a0575d0f5ace": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b891ccf058094084a4c72f810d15ad14",
       "IPY_MODEL_477ce1a400a74d898aef30253965ba6b",
       "IPY_MODEL_e2f7fa000d274479bf4ddc2081b3ad28"
      ],
      "layout": "IPY_MODEL_097ba6dcaaef4dba98c01d15ba25d6ba"
     }
    },
    "b891ccf058094084a4c72f810d15ad14": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bc06c28ecfa54cdbb9b3b5aa9517dc28",
      "placeholder": "​",
      "style": "IPY_MODEL_d4050c35ed20426e88439af593d2f3cf",
      "value": "Downloading (…)neration_config.json: 100%"
     }
    },
    "b8e386774e7c4170bfa753e5dd9778a4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_98490440e40f45d096ff5f09c643ddb7",
      "placeholder": "​",
      "style": "IPY_MODEL_818fad01927d452dbd0d56f6dbd115e7",
      "value": "Downloading (…)l-00001-of-00002.bin: 100%"
     }
    },
    "bc06c28ecfa54cdbb9b3b5aa9517dc28": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bc97f164235d4d9a8c5ce6ddff9ec9fc": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bcac5a4b41cf4498b7867f758a246e1a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bd5a150beb6d4d0a80a3cce82f07e520": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bdf536c064064736b660e4ae0cf0fbd3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "be25bd8f9c8843b98b79c25177f6bbbd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_346d121caddf4c3ea56c43c13ef73cd7",
      "placeholder": "​",
      "style": "IPY_MODEL_f68f06f4e6374f0ea71b5fbb7c4e2951",
      "value": " 727/727 [00:00&lt;00:00, 58.2kB/s]"
     }
    },
    "c28deeaca3d546a0bc9ed2acd912f01e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6e6a47ac7719476caeb5df399f8da236",
      "max": 411,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_9310586957b849ddb3ab45e10892f858",
      "value": 411
     }
    },
    "c343d2f13670422d908f58a57352e248": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c59e5979805a4c82982754de2b9619db": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f0609213e40040629de73ddba66e6364",
      "placeholder": "​",
      "style": "IPY_MODEL_49f643f587b441ae8b9ee779f86ae44f",
      "value": " 411/411 [00:00&lt;00:00, 36.2kB/s]"
     }
    },
    "c709306d5725466ebbd4bf682e7aa06d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c8d6c83083a7411991333a7ee26c4b8f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c9ec7b904a45486aa96f8580d3aaac82": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e18dae1140554388a5cec96d8a8f7065",
      "placeholder": "​",
      "style": "IPY_MODEL_4a7cdceeb332497c80886da527ddfa2a",
      "value": " 554/554 [00:00&lt;00:00, 51.5kB/s]"
     }
    },
    "cc1bd3c12ebd49b5adbf7fa855cc5711": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4f52d67f052b4797b485c7096f86dbb6",
      "max": 2,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0e9498ce5ae14669be87a0fa80381d65",
      "value": 2
     }
    },
    "cc933fc06094453e97fe2d60b0308bac": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d1ce1f773a724fe399d1e95b6296a9b7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d37911bf83f340b79fe514b6bb4f4b01": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ec09d19a6b5040f8a43a58246978cf4a",
      "placeholder": "​",
      "style": "IPY_MODEL_68a78f308c734052aa85d0a55e1f45a1",
      "value": "Loading checkpoint shards: 100%"
     }
    },
    "d4050c35ed20426e88439af593d2f3cf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "da78f46a75844b38828623726972ae5c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2dafdadad0114b59a165c4fe3d288dbd",
      "placeholder": "​",
      "style": "IPY_MODEL_360efb60c21b4fcf9aa780496e641860",
      "value": "Downloading shards: 100%"
     }
    },
    "dc3553016e2c4b3caeab479f3e480f34": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5d28a0f39a1a4bf9814b9492004243f6",
      "placeholder": "​",
      "style": "IPY_MODEL_5041f6ec5d4449a782e7ff77a89c2cc8",
      "value": " 2/2 [00:09&lt;00:00,  4.16s/it]"
     }
    },
    "dd3a1cd422be4868a7382710b4f46b21": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_da78f46a75844b38828623726972ae5c",
       "IPY_MODEL_cc1bd3c12ebd49b5adbf7fa855cc5711",
       "IPY_MODEL_b6a0160136374c0b9c19dba9577b0e87"
      ],
      "layout": "IPY_MODEL_b0502d359f1148a4b31e1c5e2412598e"
     }
    },
    "de3e0b1a3445415bba5a9c5ff83a855f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_65057a4b0ee8488ebc197dee7382a361",
      "placeholder": "​",
      "style": "IPY_MODEL_bdf536c064064736b660e4ae0cf0fbd3",
      "value": " 9.98G/9.98G [00:29&lt;00:00, 350MB/s]"
     }
    },
    "de560f0edf5e40c0ac487bf22433ea91": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_216e741c53a64e65ad64cbbfb21e796a",
      "placeholder": "​",
      "style": "IPY_MODEL_fad35c21739049f6b156e8ed9f0138dd",
      "value": "Downloading tokenizer.model: 100%"
     }
    },
    "e18dae1140554388a5cec96d8a8f7065": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e2f7fa000d274479bf4ddc2081b3ad28": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c343d2f13670422d908f58a57352e248",
      "placeholder": "​",
      "style": "IPY_MODEL_5322ddb05d77460696d1807a33d6ff3e",
      "value": " 137/137 [00:00&lt;00:00, 12.5kB/s]"
     }
    },
    "e41bad1821844add8f0a2d342b2568dd": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e90d653937e44e29a44c5e98e141f692": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_de560f0edf5e40c0ac487bf22433ea91",
       "IPY_MODEL_33038e684d3c47dfa9b9a278e9cde393",
       "IPY_MODEL_4d14fafcbec74844a2f4f2111b7ced83"
      ],
      "layout": "IPY_MODEL_e41bad1821844add8f0a2d342b2568dd"
     }
    },
    "ec09d19a6b5040f8a43a58246978cf4a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ed2bcffe5804483a81ba2f93d4fe05f1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ee4286005ef24eecb2955e37942f3d90": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f0609213e40040629de73ddba66e6364": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f2aed2597c9547b0a3e6aad197445d49": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9435c0d2f5ba4923a4d00eb3608c72a0",
      "placeholder": "​",
      "style": "IPY_MODEL_95488ccb0e9e48af97bfbc7957281f51",
      "value": "Downloading (…)cial_tokens_map.json: 100%"
     }
    },
    "f42522d00baa4d1eae90f0e4584da1b8": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f48e1c96f9f7483d9b531c0ebe0ee9a0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f68f06f4e6374f0ea71b5fbb7c4e2951": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fad35c21739049f6b156e8ed9f0138dd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fb6e10e4b20d4c75bb4675709c940acb": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_9524696c0ce34053b7ee900761127282",
       "IPY_MODEL_2418d04ed0d44f558a403d069cb3e1fe",
       "IPY_MODEL_11ebb2f403924051956170ef5c7c2be4"
      ],
      "layout": "IPY_MODEL_aeed353639234130a1c6d9ae0245f74d"
     }
    },
    "fec6c8e41d794423bbd07fa6c46bca67": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_64f8bd7f1dd64daa8a98ba1ae6ec1275",
       "IPY_MODEL_3785400cbd7f4dab9ac58f74be8f692d",
       "IPY_MODEL_15f6872f5dac4259b82b98194116f56c"
      ],
      "layout": "IPY_MODEL_ed2bcffe5804483a81ba2f93d4fe05f1"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}