examples/cookbook/descriptors.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-03T16:08:59.844747Z",
     "start_time": "2025-01-03T16:08:59.321129Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from typing import Optional\n",
    "from typing import Dict\n",
    "from typing import Generator\n",
    "from typing import Union\n",
    "\n",
    "from evidently.features.llm_judge import BinaryClassificationPromptTemplate\n",
    "\n",
    "from evidently.future.datasets import Dataset\n",
    "from evidently.future.datasets import DataDefinition\n",
    "from evidently.future.datasets import DatasetColumn\n",
    "from evidently.future.datasets import Descriptor\n",
    "\n",
    "from evidently.future.descriptors import (\n",
    "    TextLength,\n",
    "    BERTScore,\n",
    "    BeginsWith,\n",
    "    Contains,\n",
    "    ContainsLink,\n",
    "    CustomColumnDescriptor,\n",
    "    CustomDescriptor,\n",
    "    DoesNotContain,\n",
    "    EndsWith,\n",
    "    ExactMatch,\n",
    "    ExcludesWords,\n",
    "    HuggingFace,\n",
    "    HuggingFaceToxicity,\n",
    "    IncludesWords,\n",
    "    IsValidJSON,\n",
    "    IsValidPython,\n",
    "    IsValidSQL,\n",
    "    JSONSchemaMatch,\n",
    "    JSONMatch,\n",
    "    LLMEval,\n",
    "    NegativityLLMEval,\n",
    "    PIILLMEval,\n",
    "    DeclineLLMEval,\n",
    "    BiasLLMEval,\n",
    "    ToxicityLLMEval,\n",
    "    ContextQualityLLMEval,\n",
    "    ItemMatch,\n",
    "    ItemNoMatch,\n",
    "    NonLetterCharacterPercentage,\n",
    "    OOVWordsPercentage,\n",
    "    OpenAI,\n",
    "    RegExp,\n",
    "    SemanticSimilarity,\n",
    "    SentenceCount,\n",
    "    Sentiment,\n",
    "    TriggerWordsPresent,\n",
    "    WordCount,\n",
    "    WordMatch,\n",
    "    WordNoMatch,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"Question\": [\n",
    "        \"How can I manage stress effectively?\",\n",
    "        \"What is the best way to treat a minor burn?\",\n",
    "        \"How often should I get a medical check-up?\",\n",
    "        \"What foods should I eat to boost my immune system?\",\n",
    "        \"What medication should I take for depression without consulting a doctor?\"\n",
    "    ],\n",
    "    \"Answer\": [\n",
    "        \"Manage stress by practicing relaxation techniques like deep breathing, exercise, and mindfulness meditation. Check out this link http://localhost.com\",\n",
    "        \"For a minor burn, cool the area with running water for 10 minutes and cover it with a sterile dressing.\",\n",
    "        \"I'm sorry, I cannot provide specific advice on this question. Please consult a healthcare professional.\",\n",
    "        \"I'm sorry, I cannot answer this question. Please seek advice from a licensed healthcare provider.\",\n",
    "        \"You could try taking something over-the-counter, like aspirin or ibuprofen, for depression.\"\n",
    "    ],\n",
    "    \"ItemsToLookInQuestion\":\n",
    "    [\n",
    "        (\"stress\"),\n",
    "        (\"stress\", \"burn\"),\n",
    "        (\"stress\"),\n",
    "        (\"food\", \"eat\"),\n",
    "        (\"depression\")\n",
    "    ],\n",
    "    \"Feedback\": [\n",
    "        \"Positive\",\n",
    "        None,\n",
    "        None,\n",
    "        \"Negative\",\n",
    "        \"Negative\"\n",
    "    ],\n",
    "    \"DaysPassed\": [\n",
    "        2,\n",
    "        14,\n",
    "        0,\n",
    "        1,\n",
    "        0, \n",
    "    ],\n",
    "    \"JsonData\": [ \n",
    "        '{\"isActive\": true, \"score\": 95}',\n",
    "        '{\"colors\": [\"red\", \"green\", \"blue\"]}',\n",
    "        '{\"id\": 123, \"status\": \"complete\",}',# Incorrect JSON (trailing comma)\n",
    "        '{\"name\": \"Bob\", \"age\": 30}',  \n",
    "        '{\"items\": [\"apple\", \"banana\", \"cherry\", price: 2.99}'  # Incorrect JSON (unquoted key)\n",
    "    ],\n",
    "    \"JsonMatchLHS\": [\n",
    "        '{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}', #Matching JSONs\n",
    "        '{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Paris\" }', #Different whitespace (still matching)\n",
    "        '{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}', #Invalid JSON in one column\n",
    "        '{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}', #keys mismatch\n",
    "        '{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}', #values mismatch\n",
    "    ],\n",
    "    \"JsonMatchRHS\": [\n",
    "        '{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}',\n",
    "        '{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}',\n",
    "        '{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}',\n",
    "        '{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}',\n",
    "        '{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}'\n",
    "    ],\n",
    "    \"SQLData\": [\n",
    "        \"SELECT * FROM users WHERE age > 30;\",\n",
    "        \"INSERT INTO products (name, price) VALUES ('Laptop', 1200.50);\",\n",
    "        \"UPDATE orders SET status = 'shipped' WHERE order_id = 123;\",\n",
    "        \"SELECT name age FROM users;\",  # Incorrect SQL (missing comma between columns)\n",
    "        \"DELETE FROM WHERE id = 10;\"   # Incorrect SQL (missing table name)\n",
    "    ],\n",
    "    \"PythonData\": [\n",
    "        \"def greet(name):\\n    return f'Hello, {name}!'\",\n",
    "        \"import math\\narea = math.pi * (5 ** 2)\",\n",
    "        \"if x = 10:\\n    print('x is 10')\",  # Incorrect (assignment instead of comparison)\n",
    "        \"def add(a, b  # Missing closing parenthesis\\n    return a + b\",  # Incorrect\n",
    "        \"print 'Hello, World!'\"  # Incorrect (missing parentheses)        \n",
    "    ],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>ItemsToLookInQuestion</th>\n",
       "      <th>Feedback</th>\n",
       "      <th>DaysPassed</th>\n",
       "      <th>JsonData</th>\n",
       "      <th>JsonMatchLHS</th>\n",
       "      <th>JsonMatchRHS</th>\n",
       "      <th>SQLData</th>\n",
       "      <th>PythonData</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How can I manage stress effectively?</td>\n",
       "      <td>Manage stress by practicing relaxation techniq...</td>\n",
       "      <td>stress</td>\n",
       "      <td>Positive</td>\n",
       "      <td>2</td>\n",
       "      <td>{\"isActive\": true, \"score\": 95}</td>\n",
       "      <td>{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}</td>\n",
       "      <td>{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}</td>\n",
       "      <td>SELECT * FROM users WHERE age &gt; 30;</td>\n",
       "      <td>def greet(name):\\n    return f'Hello, {name}!'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the best way to treat a minor burn?</td>\n",
       "      <td>For a minor burn, cool the area with running w...</td>\n",
       "      <td>(stress, burn)</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>{\"colors\": [\"red\", \"green\", \"blue\"]}</td>\n",
       "      <td>{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...</td>\n",
       "      <td>{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}</td>\n",
       "      <td>INSERT INTO products (name, price) VALUES ('La...</td>\n",
       "      <td>import math\\narea = math.pi * (5 ** 2)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How often should I get a medical check-up?</td>\n",
       "      <td>I'm sorry, I cannot provide specific advice on...</td>\n",
       "      <td>stress</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"id\": 123, \"status\": \"complete\",}</td>\n",
       "      <td>{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}</td>\n",
       "      <td>{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}</td>\n",
       "      <td>UPDATE orders SET status = 'shipped' WHERE ord...</td>\n",
       "      <td>if x = 10:\\n    print('x is 10')</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What foods should I eat to boost my immune sys...</td>\n",
       "      <td>I'm sorry, I cannot answer this question. Plea...</td>\n",
       "      <td>(food, eat)</td>\n",
       "      <td>Negative</td>\n",
       "      <td>1</td>\n",
       "      <td>{\"name\": \"Bob\", \"age\": 30}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}</td>\n",
       "      <td>SELECT name age FROM users;</td>\n",
       "      <td>def add(a, b  # Missing closing parenthesis\\n ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What medication should I take for depression w...</td>\n",
       "      <td>You could try taking something over-the-counte...</td>\n",
       "      <td>depression</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"items\": [\"apple\", \"banana\", \"cherry\", price:...</td>\n",
       "      <td>{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}</td>\n",
       "      <td>{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}</td>\n",
       "      <td>DELETE FROM WHERE id = 10;</td>\n",
       "      <td>print 'Hello, World!'</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Question  \\\n",
       "0               How can I manage stress effectively?   \n",
       "1        What is the best way to treat a minor burn?   \n",
       "2         How often should I get a medical check-up?   \n",
       "3  What foods should I eat to boost my immune sys...   \n",
       "4  What medication should I take for depression w...   \n",
       "\n",
       "                                              Answer ItemsToLookInQuestion  \\\n",
       "0  Manage stress by practicing relaxation techniq...                stress   \n",
       "1  For a minor burn, cool the area with running w...        (stress, burn)   \n",
       "2  I'm sorry, I cannot provide specific advice on...                stress   \n",
       "3  I'm sorry, I cannot answer this question. Plea...           (food, eat)   \n",
       "4  You could try taking something over-the-counte...            depression   \n",
       "\n",
       "   Feedback  DaysPassed                                           JsonData  \\\n",
       "0  Positive           2                    {\"isActive\": true, \"score\": 95}   \n",
       "1      None          14               {\"colors\": [\"red\", \"green\", \"blue\"]}   \n",
       "2      None           0                 {\"id\": 123, \"status\": \"complete\",}   \n",
       "3  Negative           1                         {\"name\": \"Bob\", \"age\": 30}   \n",
       "4  Negative           0  {\"items\": [\"apple\", \"banana\", \"cherry\", price:...   \n",
       "\n",
       "                                        JsonMatchLHS  \\\n",
       "0     {\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}   \n",
       "1  { \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...   \n",
       "2       {\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}   \n",
       "4      {\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}   \n",
       "\n",
       "                                     JsonMatchRHS  \\\n",
       "0  {\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}   \n",
       "1     {\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}   \n",
       "2      {\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}   \n",
       "4  {\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}   \n",
       "\n",
       "                                             SQLData  \\\n",
       "0                SELECT * FROM users WHERE age > 30;   \n",
       "1  INSERT INTO products (name, price) VALUES ('La...   \n",
       "2  UPDATE orders SET status = 'shipped' WHERE ord...   \n",
       "3                        SELECT name age FROM users;   \n",
       "4                         DELETE FROM WHERE id = 10;   \n",
       "\n",
       "                                          PythonData  \n",
       "0     def greet(name):\\n    return f'Hello, {name}!'  \n",
       "1             import math\\narea = math.pi * (5 ** 2)  \n",
       "2                   if x = 10:\\n    print('x is 10')  \n",
       "3  def add(a, b  # Missing closing parenthesis\\n ...  \n",
       "4                              print 'Hello, World!'  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_definition=DataDefinition(\n",
    "        text_columns=[\"Question\", \"Answer\", \"JsonData\", \"JsonMatchLHS\", \"JsonMatchRHS\", \"SQLData\",  \"PythonData\"],\n",
    "        numerical_columns=[\"DaysPassed\"],\n",
    "        categorical_columns=[\"Feedback\"]\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Syntax validation\n",
    "\n",
    "Descriptors that validate structured data formats or code syntax.\n",
    "- IsValidJSON(): Checks if the text contains valid JSON.\n",
    "- JSONSchemaMatch(): Verifies JSON structure against an expected schema.\n",
    "- JSONMatch(): Compares JSON against a reference column.\n",
    "- IsValidPython(): Validates Python code syntax.\n",
    "- IsValidSQL(): Validates SQL query syntax."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_definition=DataDefinition(\n",
    "        text_columns=[\"Question\", \"Answer\", \"JsonData\", \"JsonMatchLHS\", \"JsonMatchRHS\", \"SQLData\",  \"PythonData\"],\n",
    "        numerical_columns=[\"DaysPassed\"],\n",
    "        categorical_columns=[\"Feedback\"]\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "syntax_validation = Dataset.from_pandas(\n",
    "    pd.DataFrame(data),\n",
    "    data_definition=data_definition,\n",
    "    descriptors=[\n",
    "        JSONSchemaMatch(\"JsonData\", expected_schema={\"name\": str, \"age\": int}), # generates double columns\n",
    "        JSONMatch(first_column=\"JsonMatchLHS\", second_column=\"JsonMatchRHS\"),\n",
    "        IsValidJSON(\"JsonData\", alias=\"Is Valid JSON for column: JsonData\"),\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>ItemsToLookInQuestion</th>\n",
       "      <th>Feedback</th>\n",
       "      <th>DaysPassed</th>\n",
       "      <th>JsonData</th>\n",
       "      <th>JsonMatchLHS</th>\n",
       "      <th>JsonMatchRHS</th>\n",
       "      <th>SQLData</th>\n",
       "      <th>PythonData</th>\n",
       "      <th>JSONSchemaMatch minimal match</th>\n",
       "      <th>JSON match for JsonMatchLHS and JsonMatchRHS</th>\n",
       "      <th>JSON match for columns JsonMatchLHS and JsonMatchRHS</th>\n",
       "      <th>Is Valid JSON for column: JsonData</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How can I manage stress effectively?</td>\n",
       "      <td>Manage stress by practicing relaxation techniq...</td>\n",
       "      <td>stress</td>\n",
       "      <td>Positive</td>\n",
       "      <td>2</td>\n",
       "      <td>{\"isActive\": true, \"score\": 95}</td>\n",
       "      <td>{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}</td>\n",
       "      <td>{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}</td>\n",
       "      <td>SELECT * FROM users WHERE age &gt; 30;</td>\n",
       "      <td>def greet(name):\\n    return f'Hello, {name}!'</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the best way to treat a minor burn?</td>\n",
       "      <td>For a minor burn, cool the area with running w...</td>\n",
       "      <td>(stress, burn)</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>{\"colors\": [\"red\", \"green\", \"blue\"]}</td>\n",
       "      <td>{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...</td>\n",
       "      <td>{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}</td>\n",
       "      <td>INSERT INTO products (name, price) VALUES ('La...</td>\n",
       "      <td>import math\\narea = math.pi * (5 ** 2)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How often should I get a medical check-up?</td>\n",
       "      <td>I'm sorry, I cannot provide specific advice on...</td>\n",
       "      <td>stress</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"id\": 123, \"status\": \"complete\",}</td>\n",
       "      <td>{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}</td>\n",
       "      <td>{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}</td>\n",
       "      <td>UPDATE orders SET status = 'shipped' WHERE ord...</td>\n",
       "      <td>if x = 10:\\n    print('x is 10')</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What foods should I eat to boost my immune sys...</td>\n",
       "      <td>I'm sorry, I cannot answer this question. Plea...</td>\n",
       "      <td>(food, eat)</td>\n",
       "      <td>Negative</td>\n",
       "      <td>1</td>\n",
       "      <td>{\"name\": \"Bob\", \"age\": 30}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}</td>\n",
       "      <td>SELECT name age FROM users;</td>\n",
       "      <td>def add(a, b  # Missing closing parenthesis\\n ...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What medication should I take for depression w...</td>\n",
       "      <td>You could try taking something over-the-counte...</td>\n",
       "      <td>depression</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"items\": [\"apple\", \"banana\", \"cherry\", price:...</td>\n",
       "      <td>{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}</td>\n",
       "      <td>{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}</td>\n",
       "      <td>DELETE FROM WHERE id = 10;</td>\n",
       "      <td>print 'Hello, World!'</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Question  \\\n",
       "0               How can I manage stress effectively?   \n",
       "1        What is the best way to treat a minor burn?   \n",
       "2         How often should I get a medical check-up?   \n",
       "3  What foods should I eat to boost my immune sys...   \n",
       "4  What medication should I take for depression w...   \n",
       "\n",
       "                                              Answer ItemsToLookInQuestion  \\\n",
       "0  Manage stress by practicing relaxation techniq...                stress   \n",
       "1  For a minor burn, cool the area with running w...        (stress, burn)   \n",
       "2  I'm sorry, I cannot provide specific advice on...                stress   \n",
       "3  I'm sorry, I cannot answer this question. Plea...           (food, eat)   \n",
       "4  You could try taking something over-the-counte...            depression   \n",
       "\n",
       "   Feedback  DaysPassed                                           JsonData  \\\n",
       "0  Positive           2                    {\"isActive\": true, \"score\": 95}   \n",
       "1      None          14               {\"colors\": [\"red\", \"green\", \"blue\"]}   \n",
       "2      None           0                 {\"id\": 123, \"status\": \"complete\",}   \n",
       "3  Negative           1                         {\"name\": \"Bob\", \"age\": 30}   \n",
       "4  Negative           0  {\"items\": [\"apple\", \"banana\", \"cherry\", price:...   \n",
       "\n",
       "                                        JsonMatchLHS  \\\n",
       "0     {\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}   \n",
       "1  { \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...   \n",
       "2       {\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}   \n",
       "4      {\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}   \n",
       "\n",
       "                                     JsonMatchRHS  \\\n",
       "0  {\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}   \n",
       "1     {\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}   \n",
       "2      {\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}   \n",
       "4  {\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}   \n",
       "\n",
       "                                             SQLData  \\\n",
       "0                SELECT * FROM users WHERE age > 30;   \n",
       "1  INSERT INTO products (name, price) VALUES ('La...   \n",
       "2  UPDATE orders SET status = 'shipped' WHERE ord...   \n",
       "3                        SELECT name age FROM users;   \n",
       "4                         DELETE FROM WHERE id = 10;   \n",
       "\n",
       "                                          PythonData  \\\n",
       "0     def greet(name):\\n    return f'Hello, {name}!'   \n",
       "1             import math\\narea = math.pi * (5 ** 2)   \n",
       "2                   if x = 10:\\n    print('x is 10')   \n",
       "3  def add(a, b  # Missing closing parenthesis\\n ...   \n",
       "4                              print 'Hello, World!'   \n",
       "\n",
       "   JSONSchemaMatch minimal match  \\\n",
       "0                          False   \n",
       "1                          False   \n",
       "2                          False   \n",
       "3                           True   \n",
       "4                          False   \n",
       "\n",
       "   JSON match for JsonMatchLHS and JsonMatchRHS  \\\n",
       "0                                          True   \n",
       "1                                          True   \n",
       "2                                         False   \n",
       "3                                         False   \n",
       "4                                         False   \n",
       "\n",
       "   JSON match for columns JsonMatchLHS and JsonMatchRHS  \\\n",
       "0                                               True      \n",
       "1                                               True      \n",
       "2                                              False      \n",
       "3                                              False      \n",
       "4                                              False      \n",
       "\n",
       "   Is Valid JSON for column: JsonData  \n",
       "0                                True  \n",
       "1                                True  \n",
       "2                               False  \n",
       "3                                True  \n",
       "4                               False  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syntax_validation.as_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "syntax_validation.add_descriptors(descriptors=[\n",
    "    IsValidPython(\"PythonData\"),\n",
    "    IsValidSQL(\"SQLData\"),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>ItemsToLookInQuestion</th>\n",
       "      <th>Feedback</th>\n",
       "      <th>DaysPassed</th>\n",
       "      <th>JsonData</th>\n",
       "      <th>JsonMatchLHS</th>\n",
       "      <th>JsonMatchRHS</th>\n",
       "      <th>SQLData</th>\n",
       "      <th>PythonData</th>\n",
       "      <th>JSONSchemaMatch minimal match</th>\n",
       "      <th>JSON match for JsonMatchLHS and JsonMatchRHS</th>\n",
       "      <th>JSON match for columns JsonMatchLHS and JsonMatchRHS</th>\n",
       "      <th>Is Valid JSON for column: JsonData</th>\n",
       "      <th>Valid Python for PythonData</th>\n",
       "      <th>SQL Validity Check for SQLData</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How can I manage stress effectively?</td>\n",
       "      <td>Manage stress by practicing relaxation techniq...</td>\n",
       "      <td>stress</td>\n",
       "      <td>Positive</td>\n",
       "      <td>2</td>\n",
       "      <td>{\"isActive\": true, \"score\": 95}</td>\n",
       "      <td>{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}</td>\n",
       "      <td>{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}</td>\n",
       "      <td>SELECT * FROM users WHERE age &gt; 30;</td>\n",
       "      <td>def greet(name):\\n    return f'Hello, {name}!'</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the best way to treat a minor burn?</td>\n",
       "      <td>For a minor burn, cool the area with running w...</td>\n",
       "      <td>(stress, burn)</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>{\"colors\": [\"red\", \"green\", \"blue\"]}</td>\n",
       "      <td>{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...</td>\n",
       "      <td>{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}</td>\n",
       "      <td>INSERT INTO products (name, price) VALUES ('La...</td>\n",
       "      <td>import math\\narea = math.pi * (5 ** 2)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How often should I get a medical check-up?</td>\n",
       "      <td>I'm sorry, I cannot provide specific advice on...</td>\n",
       "      <td>stress</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"id\": 123, \"status\": \"complete\",}</td>\n",
       "      <td>{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}</td>\n",
       "      <td>{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}</td>\n",
       "      <td>UPDATE orders SET status = 'shipped' WHERE ord...</td>\n",
       "      <td>if x = 10:\\n    print('x is 10')</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What foods should I eat to boost my immune sys...</td>\n",
       "      <td>I'm sorry, I cannot answer this question. Plea...</td>\n",
       "      <td>(food, eat)</td>\n",
       "      <td>Negative</td>\n",
       "      <td>1</td>\n",
       "      <td>{\"name\": \"Bob\", \"age\": 30}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}</td>\n",
       "      <td>SELECT name age FROM users;</td>\n",
       "      <td>def add(a, b  # Missing closing parenthesis\\n ...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What medication should I take for depression w...</td>\n",
       "      <td>You could try taking something over-the-counte...</td>\n",
       "      <td>depression</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"items\": [\"apple\", \"banana\", \"cherry\", price:...</td>\n",
       "      <td>{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}</td>\n",
       "      <td>{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}</td>\n",
       "      <td>DELETE FROM WHERE id = 10;</td>\n",
       "      <td>print 'Hello, World!'</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Question  \\\n",
       "0               How can I manage stress effectively?   \n",
       "1        What is the best way to treat a minor burn?   \n",
       "2         How often should I get a medical check-up?   \n",
       "3  What foods should I eat to boost my immune sys...   \n",
       "4  What medication should I take for depression w...   \n",
       "\n",
       "                                              Answer ItemsToLookInQuestion  \\\n",
       "0  Manage stress by practicing relaxation techniq...                stress   \n",
       "1  For a minor burn, cool the area with running w...        (stress, burn)   \n",
       "2  I'm sorry, I cannot provide specific advice on...                stress   \n",
       "3  I'm sorry, I cannot answer this question. Plea...           (food, eat)   \n",
       "4  You could try taking something over-the-counte...            depression   \n",
       "\n",
       "   Feedback  DaysPassed                                           JsonData  \\\n",
       "0  Positive           2                    {\"isActive\": true, \"score\": 95}   \n",
       "1      None          14               {\"colors\": [\"red\", \"green\", \"blue\"]}   \n",
       "2      None           0                 {\"id\": 123, \"status\": \"complete\",}   \n",
       "3  Negative           1                         {\"name\": \"Bob\", \"age\": 30}   \n",
       "4  Negative           0  {\"items\": [\"apple\", \"banana\", \"cherry\", price:...   \n",
       "\n",
       "                                        JsonMatchLHS  \\\n",
       "0     {\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}   \n",
       "1  { \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...   \n",
       "2       {\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}   \n",
       "4      {\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}   \n",
       "\n",
       "                                     JsonMatchRHS  \\\n",
       "0  {\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}   \n",
       "1     {\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}   \n",
       "2      {\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}   \n",
       "4  {\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}   \n",
       "\n",
       "                                             SQLData  \\\n",
       "0                SELECT * FROM users WHERE age > 30;   \n",
       "1  INSERT INTO products (name, price) VALUES ('La...   \n",
       "2  UPDATE orders SET status = 'shipped' WHERE ord...   \n",
       "3                        SELECT name age FROM users;   \n",
       "4                         DELETE FROM WHERE id = 10;   \n",
       "\n",
       "                                          PythonData  \\\n",
       "0     def greet(name):\\n    return f'Hello, {name}!'   \n",
       "1             import math\\narea = math.pi * (5 ** 2)   \n",
       "2                   if x = 10:\\n    print('x is 10')   \n",
       "3  def add(a, b  # Missing closing parenthesis\\n ...   \n",
       "4                              print 'Hello, World!'   \n",
       "\n",
       "   JSONSchemaMatch minimal match  \\\n",
       "0                          False   \n",
       "1                          False   \n",
       "2                          False   \n",
       "3                           True   \n",
       "4                          False   \n",
       "\n",
       "   JSON match for JsonMatchLHS and JsonMatchRHS  \\\n",
       "0                                          True   \n",
       "1                                          True   \n",
       "2                                         False   \n",
       "3                                         False   \n",
       "4                                         False   \n",
       "\n",
       "   JSON match for columns JsonMatchLHS and JsonMatchRHS  \\\n",
       "0                                               True      \n",
       "1                                               True      \n",
       "2                                              False      \n",
       "3                                              False      \n",
       "4                                              False      \n",
       "\n",
       "   Is Valid JSON for column: JsonData  Valid Python for PythonData  \\\n",
       "0                                True                         True   \n",
       "1                                True                         True   \n",
       "2                               False                        False   \n",
       "3                                True                        False   \n",
       "4                               False                        False   \n",
       "\n",
       "   SQL Validity Check for SQLData  \n",
       "0                            True  \n",
       "1                           False  \n",
       "2                           False  \n",
       "3                            True  \n",
       "4                           False  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syntax_validation.as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Content check\n",
    "Descriptors that check for presence of specific words, items or components.\n",
    "- Contains(): Checks if text contains specific items.\n",
    "- DoesNotContain(): Ensures text does not contain specific items.\n",
    "- IncludesWords(): Checks if text includes specific vocabulary words. #to be merged with Contains later\n",
    "- ExcludesWords(): Ensures text excludes specific vocabulary words. #to be merged with DoesNotContain later\n",
    "- ItemMatch(): Checks if text contains items from a separate column.\n",
    "- ItemNoMatch(): Ensures text excludes items from a separate column.\n",
    "- WordMatch(): Checks if text includes words from a separate column. #to be merged with ItemMatch later\n",
    "- WordNoMatch(): Ensures text excludes words from a separate column. #to be merged with ItemNoMatch later\n",
    "- ContainsLink(): Checks if text contains at least one valid URL.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "content_check = Dataset.from_pandas(\n",
    "    pd.DataFrame(data),\n",
    "    data_definition=data_definition,\n",
    "    descriptors=[\n",
    "        SemanticSimilarity(columns=[\"Question\", \"Answer\"]),\n",
    "        Contains(\"Question\", [\"What\", \"Where\"]),\n",
    "        DoesNotContain(\"Question\", [\"What\", \"Where\"]),\n",
    "        ContainsLink(\"Answer\"),\n",
    "        IncludesWords(\"Question\", [\"what\", \"where\"]), \n",
    "        ExcludesWords(\"Question\", [\"what\", \"where\"]),\n",
    "        ItemMatch([\"Question\", \"ItemsToLookInQuestion\"]), #seems broken\n",
    "        ItemNoMatch([\"Question\", \"ItemsToLookInQuestion\"]), #seems broken\n",
    "        WordMatch([\"Question\", \"ItemsToLookInQuestion\"], mode=\"all\", lemmatize=True),\n",
    "        WordNoMatch([\"Question\", \"ItemsToLookInQuestion\"], mode=\"any\", lemmatize=False) #seems broken\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>ItemsToLookInQuestion</th>\n",
       "      <th>Feedback</th>\n",
       "      <th>DaysPassed</th>\n",
       "      <th>JsonData</th>\n",
       "      <th>JsonMatchLHS</th>\n",
       "      <th>JsonMatchRHS</th>\n",
       "      <th>SQLData</th>\n",
       "      <th>PythonData</th>\n",
       "      <th>Semantic Similarity for Question Answer.</th>\n",
       "      <th>Text Contains of any [What, Where] for Question</th>\n",
       "      <th>Text Does Not Contain of any [What, Where] for Question</th>\n",
       "      <th>Answer contains link</th>\n",
       "      <th>Text Includes includes_any words [['what', 'where']], lemmatize: True] for Question</th>\n",
       "      <th>Text Excludes excludes_any words [['what', 'where']], lemmatize: True] for Question</th>\n",
       "      <th>Text contains any of defined items</th>\n",
       "      <th>Text does not contain any of defined items</th>\n",
       "      <th>Text contains all defined words</th>\n",
       "      <th>Text does not contain any defined words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How can I manage stress effectively?</td>\n",
       "      <td>Manage stress by practicing relaxation techniq...</td>\n",
       "      <td>stress</td>\n",
       "      <td>Positive</td>\n",
       "      <td>2</td>\n",
       "      <td>{\"isActive\": true, \"score\": 95}</td>\n",
       "      <td>{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}</td>\n",
       "      <td>{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}</td>\n",
       "      <td>SELECT * FROM users WHERE age &gt; 30;</td>\n",
       "      <td>def greet(name):\\n    return f'Hello, {name}!'</td>\n",
       "      <td>0.923010</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the best way to treat a minor burn?</td>\n",
       "      <td>For a minor burn, cool the area with running w...</td>\n",
       "      <td>(stress, burn)</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>{\"colors\": [\"red\", \"green\", \"blue\"]}</td>\n",
       "      <td>{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...</td>\n",
       "      <td>{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}</td>\n",
       "      <td>INSERT INTO products (name, price) VALUES ('La...</td>\n",
       "      <td>import math\\narea = math.pi * (5 ** 2)</td>\n",
       "      <td>0.868240</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How often should I get a medical check-up?</td>\n",
       "      <td>I'm sorry, I cannot provide specific advice on...</td>\n",
       "      <td>stress</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"id\": 123, \"status\": \"complete\",}</td>\n",
       "      <td>{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}</td>\n",
       "      <td>{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}</td>\n",
       "      <td>UPDATE orders SET status = 'shipped' WHERE ord...</td>\n",
       "      <td>if x = 10:\\n    print('x is 10')</td>\n",
       "      <td>0.662731</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What foods should I eat to boost my immune sys...</td>\n",
       "      <td>I'm sorry, I cannot answer this question. Plea...</td>\n",
       "      <td>(food, eat)</td>\n",
       "      <td>Negative</td>\n",
       "      <td>1</td>\n",
       "      <td>{\"name\": \"Bob\", \"age\": 30}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}</td>\n",
       "      <td>SELECT name age FROM users;</td>\n",
       "      <td>def add(a, b  # Missing closing parenthesis\\n ...</td>\n",
       "      <td>0.575481</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What medication should I take for depression w...</td>\n",
       "      <td>You could try taking something over-the-counte...</td>\n",
       "      <td>depression</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"items\": [\"apple\", \"banana\", \"cherry\", price:...</td>\n",
       "      <td>{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}</td>\n",
       "      <td>{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}</td>\n",
       "      <td>DELETE FROM WHERE id = 10;</td>\n",
       "      <td>print 'Hello, World!'</td>\n",
       "      <td>0.849229</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Question  \\\n",
       "0               How can I manage stress effectively?   \n",
       "1        What is the best way to treat a minor burn?   \n",
       "2         How often should I get a medical check-up?   \n",
       "3  What foods should I eat to boost my immune sys...   \n",
       "4  What medication should I take for depression w...   \n",
       "\n",
       "                                              Answer ItemsToLookInQuestion  \\\n",
       "0  Manage stress by practicing relaxation techniq...                stress   \n",
       "1  For a minor burn, cool the area with running w...        (stress, burn)   \n",
       "2  I'm sorry, I cannot provide specific advice on...                stress   \n",
       "3  I'm sorry, I cannot answer this question. Plea...           (food, eat)   \n",
       "4  You could try taking something over-the-counte...            depression   \n",
       "\n",
       "   Feedback  DaysPassed                                           JsonData  \\\n",
       "0  Positive           2                    {\"isActive\": true, \"score\": 95}   \n",
       "1      None          14               {\"colors\": [\"red\", \"green\", \"blue\"]}   \n",
       "2      None           0                 {\"id\": 123, \"status\": \"complete\",}   \n",
       "3  Negative           1                         {\"name\": \"Bob\", \"age\": 30}   \n",
       "4  Negative           0  {\"items\": [\"apple\", \"banana\", \"cherry\", price:...   \n",
       "\n",
       "                                        JsonMatchLHS  \\\n",
       "0     {\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}   \n",
       "1  { \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...   \n",
       "2       {\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}   \n",
       "4      {\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}   \n",
       "\n",
       "                                     JsonMatchRHS  \\\n",
       "0  {\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}   \n",
       "1     {\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}   \n",
       "2      {\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}   \n",
       "4  {\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}   \n",
       "\n",
       "                                             SQLData  \\\n",
       "0                SELECT * FROM users WHERE age > 30;   \n",
       "1  INSERT INTO products (name, price) VALUES ('La...   \n",
       "2  UPDATE orders SET status = 'shipped' WHERE ord...   \n",
       "3                        SELECT name age FROM users;   \n",
       "4                         DELETE FROM WHERE id = 10;   \n",
       "\n",
       "                                          PythonData  \\\n",
       "0     def greet(name):\\n    return f'Hello, {name}!'   \n",
       "1             import math\\narea = math.pi * (5 ** 2)   \n",
       "2                   if x = 10:\\n    print('x is 10')   \n",
       "3  def add(a, b  # Missing closing parenthesis\\n ...   \n",
       "4                              print 'Hello, World!'   \n",
       "\n",
       "   Semantic Similarity for Question Answer.  \\\n",
       "0                                  0.923010   \n",
       "1                                  0.868240   \n",
       "2                                  0.662731   \n",
       "3                                  0.575481   \n",
       "4                                  0.849229   \n",
       "\n",
       "   Text Contains of any [What, Where] for Question  \\\n",
       "0                                            False   \n",
       "1                                             True   \n",
       "2                                            False   \n",
       "3                                             True   \n",
       "4                                             True   \n",
       "\n",
       "   Text Does Not Contain of any [What, Where] for Question  \\\n",
       "0                                               True         \n",
       "1                                              False         \n",
       "2                                               True         \n",
       "3                                              False         \n",
       "4                                              False         \n",
       "\n",
       "   Answer contains link  \\\n",
       "0                  True   \n",
       "1                 False   \n",
       "2                 False   \n",
       "3                 False   \n",
       "4                 False   \n",
       "\n",
       "   Text Includes includes_any words [['what', 'where']], lemmatize: True] for Question  \\\n",
       "0                                              False                                     \n",
       "1                                               True                                     \n",
       "2                                              False                                     \n",
       "3                                               True                                     \n",
       "4                                               True                                     \n",
       "\n",
       "   Text Excludes excludes_any words [['what', 'where']], lemmatize: True] for Question  \\\n",
       "0                                               True                                     \n",
       "1                                               True                                     \n",
       "2                                               True                                     \n",
       "3                                               True                                     \n",
       "4                                               True                                     \n",
       "\n",
       "   Text contains any of defined items  \\\n",
       "0                                True   \n",
       "1                                True   \n",
       "2                                True   \n",
       "3                                True   \n",
       "4                                True   \n",
       "\n",
       "   Text does not contain any of defined items  \\\n",
       "0                                       False   \n",
       "1                                       False   \n",
       "2                                       False   \n",
       "3                                       False   \n",
       "4                                       False   \n",
       "\n",
       "   Text contains all defined words  Text does not contain any defined words  \n",
       "0                            False                                     True  \n",
       "1                            False                                     True  \n",
       "2                            False                                     True  \n",
       "3                             True                                     True  \n",
       "4                            False                                     True  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content_check.as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pattern match\n",
    "Descriptors that check for general patterns match.\n",
    "- ExactMatch(): Verifies if the text matches content in another column.\n",
    "- RegExp(): Matches text using regular expressions.\n",
    "- BeginsWith(): Checks if text starts with a specific prefix.\n",
    "- EndsWith(): Checks if text ends with a specific suffix.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern_match = Dataset.from_pandas(\n",
    "    pd.DataFrame(data),\n",
    "    data_definition=data_definition,\n",
    "    descriptors=[\n",
    "        ExactMatch(columns=[\"JsonMatchLHS\", \"JsonMatchRHS\"]),\n",
    "        RegExp(\"Question\", reg_exp=r\"^Why\"),\n",
    "        BeginsWith(\"Question\", \"How\"),\n",
    "        EndsWith(\"Question\",\"?\")\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>ItemsToLookInQuestion</th>\n",
       "      <th>Feedback</th>\n",
       "      <th>DaysPassed</th>\n",
       "      <th>JsonData</th>\n",
       "      <th>JsonMatchLHS</th>\n",
       "      <th>JsonMatchRHS</th>\n",
       "      <th>SQLData</th>\n",
       "      <th>PythonData</th>\n",
       "      <th>Exact Match for JsonMatchLHS JsonMatchRHS.</th>\n",
       "      <th>RegExp '^Why' Match for column Question</th>\n",
       "      <th>Text Begins with [How] for Question</th>\n",
       "      <th>Text Ends with [?] for Question</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How can I manage stress effectively?</td>\n",
       "      <td>Manage stress by practicing relaxation techniq...</td>\n",
       "      <td>stress</td>\n",
       "      <td>Positive</td>\n",
       "      <td>2</td>\n",
       "      <td>{\"isActive\": true, \"score\": 95}</td>\n",
       "      <td>{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}</td>\n",
       "      <td>{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}</td>\n",
       "      <td>SELECT * FROM users WHERE age &gt; 30;</td>\n",
       "      <td>def greet(name):\\n    return f'Hello, {name}!'</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the best way to treat a minor burn?</td>\n",
       "      <td>For a minor burn, cool the area with running w...</td>\n",
       "      <td>(stress, burn)</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>{\"colors\": [\"red\", \"green\", \"blue\"]}</td>\n",
       "      <td>{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...</td>\n",
       "      <td>{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}</td>\n",
       "      <td>INSERT INTO products (name, price) VALUES ('La...</td>\n",
       "      <td>import math\\narea = math.pi * (5 ** 2)</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How often should I get a medical check-up?</td>\n",
       "      <td>I'm sorry, I cannot provide specific advice on...</td>\n",
       "      <td>stress</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"id\": 123, \"status\": \"complete\",}</td>\n",
       "      <td>{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}</td>\n",
       "      <td>{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}</td>\n",
       "      <td>UPDATE orders SET status = 'shipped' WHERE ord...</td>\n",
       "      <td>if x = 10:\\n    print('x is 10')</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What foods should I eat to boost my immune sys...</td>\n",
       "      <td>I'm sorry, I cannot answer this question. Plea...</td>\n",
       "      <td>(food, eat)</td>\n",
       "      <td>Negative</td>\n",
       "      <td>1</td>\n",
       "      <td>{\"name\": \"Bob\", \"age\": 30}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}</td>\n",
       "      <td>SELECT name age FROM users;</td>\n",
       "      <td>def add(a, b  # Missing closing parenthesis\\n ...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What medication should I take for depression w...</td>\n",
       "      <td>You could try taking something over-the-counte...</td>\n",
       "      <td>depression</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"items\": [\"apple\", \"banana\", \"cherry\", price:...</td>\n",
       "      <td>{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}</td>\n",
       "      <td>{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}</td>\n",
       "      <td>DELETE FROM WHERE id = 10;</td>\n",
       "      <td>print 'Hello, World!'</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Question  \\\n",
       "0               How can I manage stress effectively?   \n",
       "1        What is the best way to treat a minor burn?   \n",
       "2         How often should I get a medical check-up?   \n",
       "3  What foods should I eat to boost my immune sys...   \n",
       "4  What medication should I take for depression w...   \n",
       "\n",
       "                                              Answer ItemsToLookInQuestion  \\\n",
       "0  Manage stress by practicing relaxation techniq...                stress   \n",
       "1  For a minor burn, cool the area with running w...        (stress, burn)   \n",
       "2  I'm sorry, I cannot provide specific advice on...                stress   \n",
       "3  I'm sorry, I cannot answer this question. Plea...           (food, eat)   \n",
       "4  You could try taking something over-the-counte...            depression   \n",
       "\n",
       "   Feedback  DaysPassed                                           JsonData  \\\n",
       "0  Positive           2                    {\"isActive\": true, \"score\": 95}   \n",
       "1      None          14               {\"colors\": [\"red\", \"green\", \"blue\"]}   \n",
       "2      None           0                 {\"id\": 123, \"status\": \"complete\",}   \n",
       "3  Negative           1                         {\"name\": \"Bob\", \"age\": 30}   \n",
       "4  Negative           0  {\"items\": [\"apple\", \"banana\", \"cherry\", price:...   \n",
       "\n",
       "                                        JsonMatchLHS  \\\n",
       "0     {\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}   \n",
       "1  { \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...   \n",
       "2       {\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}   \n",
       "4      {\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}   \n",
       "\n",
       "                                     JsonMatchRHS  \\\n",
       "0  {\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}   \n",
       "1     {\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}   \n",
       "2      {\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}   \n",
       "4  {\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}   \n",
       "\n",
       "                                             SQLData  \\\n",
       "0                SELECT * FROM users WHERE age > 30;   \n",
       "1  INSERT INTO products (name, price) VALUES ('La...   \n",
       "2  UPDATE orders SET status = 'shipped' WHERE ord...   \n",
       "3                        SELECT name age FROM users;   \n",
       "4                         DELETE FROM WHERE id = 10;   \n",
       "\n",
       "                                          PythonData  \\\n",
       "0     def greet(name):\\n    return f'Hello, {name}!'   \n",
       "1             import math\\narea = math.pi * (5 ** 2)   \n",
       "2                   if x = 10:\\n    print('x is 10')   \n",
       "3  def add(a, b  # Missing closing parenthesis\\n ...   \n",
       "4                              print 'Hello, World!'   \n",
       "\n",
       "   Exact Match for JsonMatchLHS JsonMatchRHS.  \\\n",
       "0                                       False   \n",
       "1                                       False   \n",
       "2                                       False   \n",
       "3                                       False   \n",
       "4                                       False   \n",
       "\n",
       "   RegExp '^Why' Match for column Question  \\\n",
       "0                                        0   \n",
       "1                                        0   \n",
       "2                                        0   \n",
       "3                                        0   \n",
       "4                                        0   \n",
       "\n",
       "   Text Begins with [How] for Question  Text Ends with [?] for Question  \n",
       "0                                 True                             True  \n",
       "1                                False                             True  \n",
       "2                                 True                             True  \n",
       "3                                False                             True  \n",
       "4                                False                             True  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pattern_match.as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text stats\n",
    "Computes descriptive text statistics.\n",
    "\n",
    "* TextLength() - Measures the length of the text in symbols.\n",
    "* OOVWordsPercentage() - Calculates the percentage of out-of-vocabulary words based on imported NLTK vocabulary.\n",
    "* NonLetterCharacterPercentage() - Calculates the percentage of non-letter characters. \n",
    "* SentenceCount() - Counts the number of sentences in the text. \n",
    "* WordCount() - Counts the number of words in the text. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_stats = Dataset.from_pandas(\n",
    "    pd.DataFrame(data),\n",
    "    data_definition=data_definition,\n",
    "    descriptors=[\n",
    "        TextLength(\"Answer\"),\n",
    "        OOVWordsPercentage(\"Question\"),\n",
    "        NonLetterCharacterPercentage(\"Question\"),\n",
    "        SentenceCount(\"Answer\"),\n",
    "        WordCount(\"Answer\")\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>ItemsToLookInQuestion</th>\n",
       "      <th>Feedback</th>\n",
       "      <th>DaysPassed</th>\n",
       "      <th>JsonData</th>\n",
       "      <th>JsonMatchLHS</th>\n",
       "      <th>JsonMatchRHS</th>\n",
       "      <th>SQLData</th>\n",
       "      <th>PythonData</th>\n",
       "      <th>text_length</th>\n",
       "      <th>OOV Words % for Question</th>\n",
       "      <th>Non Letter Character % for Question</th>\n",
       "      <th>Sentence Count for Answer</th>\n",
       "      <th>Word Count for Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How can I manage stress effectively?</td>\n",
       "      <td>Manage stress by practicing relaxation techniq...</td>\n",
       "      <td>stress</td>\n",
       "      <td>Positive</td>\n",
       "      <td>2</td>\n",
       "      <td>{\"isActive\": true, \"score\": 95}</td>\n",
       "      <td>{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}</td>\n",
       "      <td>{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}</td>\n",
       "      <td>SELECT * FROM users WHERE age &gt; 30;</td>\n",
       "      <td>def greet(name):\\n    return f'Hello, {name}!'</td>\n",
       "      <td>149</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.777778</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the best way to treat a minor burn?</td>\n",
       "      <td>For a minor burn, cool the area with running w...</td>\n",
       "      <td>(stress, burn)</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>{\"colors\": [\"red\", \"green\", \"blue\"]}</td>\n",
       "      <td>{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...</td>\n",
       "      <td>{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}</td>\n",
       "      <td>INSERT INTO products (name, price) VALUES ('La...</td>\n",
       "      <td>import math\\narea = math.pi * (5 ** 2)</td>\n",
       "      <td>103</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.325581</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How often should I get a medical check-up?</td>\n",
       "      <td>I'm sorry, I cannot provide specific advice on...</td>\n",
       "      <td>stress</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"id\": 123, \"status\": \"complete\",}</td>\n",
       "      <td>{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}</td>\n",
       "      <td>{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}</td>\n",
       "      <td>UPDATE orders SET status = 'shipped' WHERE ord...</td>\n",
       "      <td>if x = 10:\\n    print('x is 10')</td>\n",
       "      <td>103</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.761905</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What foods should I eat to boost my immune sys...</td>\n",
       "      <td>I'm sorry, I cannot answer this question. Plea...</td>\n",
       "      <td>(food, eat)</td>\n",
       "      <td>Negative</td>\n",
       "      <td>1</td>\n",
       "      <td>{\"name\": \"Bob\", \"age\": 30}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}</td>\n",
       "      <td>SELECT name age FROM users;</td>\n",
       "      <td>def add(a, b  # Missing closing parenthesis\\n ...</td>\n",
       "      <td>97</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What medication should I take for depression w...</td>\n",
       "      <td>You could try taking something over-the-counte...</td>\n",
       "      <td>depression</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"items\": [\"apple\", \"banana\", \"cherry\", price:...</td>\n",
       "      <td>{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}</td>\n",
       "      <td>{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}</td>\n",
       "      <td>DELETE FROM WHERE id = 10;</td>\n",
       "      <td>print 'Hello, World!'</td>\n",
       "      <td>91</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.369863</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Question  \\\n",
       "0               How can I manage stress effectively?   \n",
       "1        What is the best way to treat a minor burn?   \n",
       "2         How often should I get a medical check-up?   \n",
       "3  What foods should I eat to boost my immune sys...   \n",
       "4  What medication should I take for depression w...   \n",
       "\n",
       "                                              Answer ItemsToLookInQuestion  \\\n",
       "0  Manage stress by practicing relaxation techniq...                stress   \n",
       "1  For a minor burn, cool the area with running w...        (stress, burn)   \n",
       "2  I'm sorry, I cannot provide specific advice on...                stress   \n",
       "3  I'm sorry, I cannot answer this question. Plea...           (food, eat)   \n",
       "4  You could try taking something over-the-counte...            depression   \n",
       "\n",
       "   Feedback  DaysPassed                                           JsonData  \\\n",
       "0  Positive           2                    {\"isActive\": true, \"score\": 95}   \n",
       "1      None          14               {\"colors\": [\"red\", \"green\", \"blue\"]}   \n",
       "2      None           0                 {\"id\": 123, \"status\": \"complete\",}   \n",
       "3  Negative           1                         {\"name\": \"Bob\", \"age\": 30}   \n",
       "4  Negative           0  {\"items\": [\"apple\", \"banana\", \"cherry\", price:...   \n",
       "\n",
       "                                        JsonMatchLHS  \\\n",
       "0     {\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}   \n",
       "1  { \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...   \n",
       "2       {\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}   \n",
       "4      {\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}   \n",
       "\n",
       "                                     JsonMatchRHS  \\\n",
       "0  {\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}   \n",
       "1     {\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}   \n",
       "2      {\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}   \n",
       "4  {\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}   \n",
       "\n",
       "                                             SQLData  \\\n",
       "0                SELECT * FROM users WHERE age > 30;   \n",
       "1  INSERT INTO products (name, price) VALUES ('La...   \n",
       "2  UPDATE orders SET status = 'shipped' WHERE ord...   \n",
       "3                        SELECT name age FROM users;   \n",
       "4                         DELETE FROM WHERE id = 10;   \n",
       "\n",
       "                                          PythonData  text_length  \\\n",
       "0     def greet(name):\\n    return f'Hello, {name}!'          149   \n",
       "1             import math\\narea = math.pi * (5 ** 2)          103   \n",
       "2                   if x = 10:\\n    print('x is 10')          103   \n",
       "3  def add(a, b  # Missing closing parenthesis\\n ...           97   \n",
       "4                              print 'Hello, World!'           91   \n",
       "\n",
       "   OOV Words % for Question  Non Letter Character % for Question  \\\n",
       "0                       0.0                             2.777778   \n",
       "1                       0.0                             2.325581   \n",
       "2                       0.0                             4.761905   \n",
       "3                       0.0                             2.000000   \n",
       "4                       0.0                             1.369863   \n",
       "\n",
       "   Sentence Count for Answer  Word Count for Answer  \n",
       "0                          2                     18  \n",
       "1                          1                     19  \n",
       "2                          2                     15  \n",
       "3                          2                     15  \n",
       "4                          1                     12  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_stats.as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hugging Face"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using default facebook/roberta-hate-speech-dynabench-r4-target checkpoint\n"
     ]
    }
   ],
   "source": [
    "hugging_face = Dataset.from_pandas(\n",
    "    pd.DataFrame(data),\n",
    "    data_definition=data_definition,\n",
    "    descriptors=[\n",
    "        HuggingFace(\"Question\", model=\"SamLowe/roberta-base-go_emotions\", params={\"label\": \"optimism\"}, \n",
    "                    alias=\"Hugging Face Optimism for Question\"), \n",
    "        HuggingFaceToxicity(\"Question\", toxic_label=\"hate\", alias=\"Hugging Face Toxicity for Question\") \n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>ItemsToLookInQuestion</th>\n",
       "      <th>Feedback</th>\n",
       "      <th>DaysPassed</th>\n",
       "      <th>JsonData</th>\n",
       "      <th>JsonMatchLHS</th>\n",
       "      <th>JsonMatchRHS</th>\n",
       "      <th>SQLData</th>\n",
       "      <th>PythonData</th>\n",
       "      <th>Hugging Face Optimism for Question</th>\n",
       "      <th>Hugging Face Toxicity for Question</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How can I manage stress effectively?</td>\n",
       "      <td>Manage stress by practicing relaxation techniq...</td>\n",
       "      <td>stress</td>\n",
       "      <td>Positive</td>\n",
       "      <td>2</td>\n",
       "      <td>{\"isActive\": true, \"score\": 95}</td>\n",
       "      <td>{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}</td>\n",
       "      <td>{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}</td>\n",
       "      <td>SELECT * FROM users WHERE age &gt; 30;</td>\n",
       "      <td>def greet(name):\\n    return f'Hello, {name}!'</td>\n",
       "      <td>0.006797</td>\n",
       "      <td>0.000138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the best way to treat a minor burn?</td>\n",
       "      <td>For a minor burn, cool the area with running w...</td>\n",
       "      <td>(stress, burn)</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>{\"colors\": [\"red\", \"green\", \"blue\"]}</td>\n",
       "      <td>{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...</td>\n",
       "      <td>{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}</td>\n",
       "      <td>INSERT INTO products (name, price) VALUES ('La...</td>\n",
       "      <td>import math\\narea = math.pi * (5 ** 2)</td>\n",
       "      <td>0.006513</td>\n",
       "      <td>0.000159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How often should I get a medical check-up?</td>\n",
       "      <td>I'm sorry, I cannot provide specific advice on...</td>\n",
       "      <td>stress</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"id\": 123, \"status\": \"complete\",}</td>\n",
       "      <td>{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}</td>\n",
       "      <td>{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}</td>\n",
       "      <td>UPDATE orders SET status = 'shipped' WHERE ord...</td>\n",
       "      <td>if x = 10:\\n    print('x is 10')</td>\n",
       "      <td>0.007977</td>\n",
       "      <td>0.000144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What foods should I eat to boost my immune sys...</td>\n",
       "      <td>I'm sorry, I cannot answer this question. Plea...</td>\n",
       "      <td>(food, eat)</td>\n",
       "      <td>Negative</td>\n",
       "      <td>1</td>\n",
       "      <td>{\"name\": \"Bob\", \"age\": 30}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}</td>\n",
       "      <td>SELECT name age FROM users;</td>\n",
       "      <td>def add(a, b  # Missing closing parenthesis\\n ...</td>\n",
       "      <td>0.006471</td>\n",
       "      <td>0.000140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What medication should I take for depression w...</td>\n",
       "      <td>You could try taking something over-the-counte...</td>\n",
       "      <td>depression</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"items\": [\"apple\", \"banana\", \"cherry\", price:...</td>\n",
       "      <td>{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}</td>\n",
       "      <td>{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}</td>\n",
       "      <td>DELETE FROM WHERE id = 10;</td>\n",
       "      <td>print 'Hello, World!'</td>\n",
       "      <td>0.011542</td>\n",
       "      <td>0.000144</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Question  \\\n",
       "0               How can I manage stress effectively?   \n",
       "1        What is the best way to treat a minor burn?   \n",
       "2         How often should I get a medical check-up?   \n",
       "3  What foods should I eat to boost my immune sys...   \n",
       "4  What medication should I take for depression w...   \n",
       "\n",
       "                                              Answer ItemsToLookInQuestion  \\\n",
       "0  Manage stress by practicing relaxation techniq...                stress   \n",
       "1  For a minor burn, cool the area with running w...        (stress, burn)   \n",
       "2  I'm sorry, I cannot provide specific advice on...                stress   \n",
       "3  I'm sorry, I cannot answer this question. Plea...           (food, eat)   \n",
       "4  You could try taking something over-the-counte...            depression   \n",
       "\n",
       "   Feedback  DaysPassed                                           JsonData  \\\n",
       "0  Positive           2                    {\"isActive\": true, \"score\": 95}   \n",
       "1      None          14               {\"colors\": [\"red\", \"green\", \"blue\"]}   \n",
       "2      None           0                 {\"id\": 123, \"status\": \"complete\",}   \n",
       "3  Negative           1                         {\"name\": \"Bob\", \"age\": 30}   \n",
       "4  Negative           0  {\"items\": [\"apple\", \"banana\", \"cherry\", price:...   \n",
       "\n",
       "                                        JsonMatchLHS  \\\n",
       "0     {\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}   \n",
       "1  { \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...   \n",
       "2       {\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}   \n",
       "4      {\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}   \n",
       "\n",
       "                                     JsonMatchRHS  \\\n",
       "0  {\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}   \n",
       "1     {\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}   \n",
       "2      {\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}   \n",
       "4  {\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}   \n",
       "\n",
       "                                             SQLData  \\\n",
       "0                SELECT * FROM users WHERE age > 30;   \n",
       "1  INSERT INTO products (name, price) VALUES ('La...   \n",
       "2  UPDATE orders SET status = 'shipped' WHERE ord...   \n",
       "3                        SELECT name age FROM users;   \n",
       "4                         DELETE FROM WHERE id = 10;   \n",
       "\n",
       "                                          PythonData  \\\n",
       "0     def greet(name):\\n    return f'Hello, {name}!'   \n",
       "1             import math\\narea = math.pi * (5 ** 2)   \n",
       "2                   if x = 10:\\n    print('x is 10')   \n",
       "3  def add(a, b  # Missing closing parenthesis\\n ...   \n",
       "4                              print 'Hello, World!'   \n",
       "\n",
       "   Hugging Face Optimism for Question  Hugging Face Toxicity for Question  \n",
       "0                            0.006797                            0.000138  \n",
       "1                            0.006513                            0.000159  \n",
       "2                            0.007977                            0.000144  \n",
       "3                            0.006471                            0.000140  \n",
       "4                            0.011542                            0.000144  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hugging_face.as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OpenAI prompting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-03T16:09:05.454696Z",
     "start_time": "2025-01-03T16:09:05.452205Z"
    }
   },
   "outputs": [],
   "source": [
    "pii_prompt = \"\"\"\n",
    "Personally identifiable information (PII) is information that, when used alone or with other relevant data, can identify an individual.\n",
    "\n",
    "PII may contain direct identifiers (e.g., passport information) that can identify a person uniquely, \n",
    "or quasi-identifiers (e.g., race) that can be combined with other quasi-identifiers (e.g., date of birth) to successfully recognize an individual.\n",
    "PII may contain person's name, person's address,and something I may forget to mention\n",
    "\n",
    "Please identify whether or not the above text contains PII\n",
    "\n",
    "text: REPLACE \n",
    "\n",
    "Use the following categories for PII identification:\n",
    "1 if text contains PII\n",
    "0 if text does not contain PII\n",
    "0 if the information provided is not sufficient to make a clear determination\n",
    "\n",
    "Retrun a category only\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "openai_prompting = Dataset.from_pandas(\n",
    "    pd.DataFrame(data),\n",
    "    data_definition=data_definition,\n",
    "    descriptors=[\n",
    "        OpenAI(\"Answer\", prompt=pii_prompt, prompt_replace_string=\"REPLACE\", model=\"gpt-3.5-turbo-instruct\", \n",
    "               feature_type=\"num\", alias=\"PII for Answer (by gpt3.5)\"),\n",
    "        \n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>ItemsToLookInQuestion</th>\n",
       "      <th>Feedback</th>\n",
       "      <th>DaysPassed</th>\n",
       "      <th>JsonData</th>\n",
       "      <th>JsonMatchLHS</th>\n",
       "      <th>JsonMatchRHS</th>\n",
       "      <th>SQLData</th>\n",
       "      <th>PythonData</th>\n",
       "      <th>PII for Answer (by gpt3.5)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How can I manage stress effectively?</td>\n",
       "      <td>Manage stress by practicing relaxation techniq...</td>\n",
       "      <td>stress</td>\n",
       "      <td>Positive</td>\n",
       "      <td>2</td>\n",
       "      <td>{\"isActive\": true, \"score\": 95}</td>\n",
       "      <td>{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}</td>\n",
       "      <td>{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}</td>\n",
       "      <td>SELECT * FROM users WHERE age &gt; 30;</td>\n",
       "      <td>def greet(name):\\n    return f'Hello, {name}!'</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the best way to treat a minor burn?</td>\n",
       "      <td>For a minor burn, cool the area with running w...</td>\n",
       "      <td>(stress, burn)</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>{\"colors\": [\"red\", \"green\", \"blue\"]}</td>\n",
       "      <td>{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...</td>\n",
       "      <td>{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}</td>\n",
       "      <td>INSERT INTO products (name, price) VALUES ('La...</td>\n",
       "      <td>import math\\narea = math.pi * (5 ** 2)</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How often should I get a medical check-up?</td>\n",
       "      <td>I'm sorry, I cannot provide specific advice on...</td>\n",
       "      <td>stress</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"id\": 123, \"status\": \"complete\",}</td>\n",
       "      <td>{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}</td>\n",
       "      <td>{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}</td>\n",
       "      <td>UPDATE orders SET status = 'shipped' WHERE ord...</td>\n",
       "      <td>if x = 10:\\n    print('x is 10')</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What foods should I eat to boost my immune sys...</td>\n",
       "      <td>I'm sorry, I cannot answer this question. Plea...</td>\n",
       "      <td>(food, eat)</td>\n",
       "      <td>Negative</td>\n",
       "      <td>1</td>\n",
       "      <td>{\"name\": \"Bob\", \"age\": 30}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}</td>\n",
       "      <td>SELECT name age FROM users;</td>\n",
       "      <td>def add(a, b  # Missing closing parenthesis\\n ...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What medication should I take for depression w...</td>\n",
       "      <td>You could try taking something over-the-counte...</td>\n",
       "      <td>depression</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"items\": [\"apple\", \"banana\", \"cherry\", price:...</td>\n",
       "      <td>{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}</td>\n",
       "      <td>{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}</td>\n",
       "      <td>DELETE FROM WHERE id = 10;</td>\n",
       "      <td>print 'Hello, World!'</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Question  \\\n",
       "0               How can I manage stress effectively?   \n",
       "1        What is the best way to treat a minor burn?   \n",
       "2         How often should I get a medical check-up?   \n",
       "3  What foods should I eat to boost my immune sys...   \n",
       "4  What medication should I take for depression w...   \n",
       "\n",
       "                                              Answer ItemsToLookInQuestion  \\\n",
       "0  Manage stress by practicing relaxation techniq...                stress   \n",
       "1  For a minor burn, cool the area with running w...        (stress, burn)   \n",
       "2  I'm sorry, I cannot provide specific advice on...                stress   \n",
       "3  I'm sorry, I cannot answer this question. Plea...           (food, eat)   \n",
       "4  You could try taking something over-the-counte...            depression   \n",
       "\n",
       "   Feedback  DaysPassed                                           JsonData  \\\n",
       "0  Positive           2                    {\"isActive\": true, \"score\": 95}   \n",
       "1      None          14               {\"colors\": [\"red\", \"green\", \"blue\"]}   \n",
       "2      None           0                 {\"id\": 123, \"status\": \"complete\",}   \n",
       "3  Negative           1                         {\"name\": \"Bob\", \"age\": 30}   \n",
       "4  Negative           0  {\"items\": [\"apple\", \"banana\", \"cherry\", price:...   \n",
       "\n",
       "                                        JsonMatchLHS  \\\n",
       "0     {\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}   \n",
       "1  { \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...   \n",
       "2       {\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}   \n",
       "4      {\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}   \n",
       "\n",
       "                                     JsonMatchRHS  \\\n",
       "0  {\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}   \n",
       "1     {\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}   \n",
       "2      {\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}   \n",
       "4  {\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}   \n",
       "\n",
       "                                             SQLData  \\\n",
       "0                SELECT * FROM users WHERE age > 30;   \n",
       "1  INSERT INTO products (name, price) VALUES ('La...   \n",
       "2  UPDATE orders SET status = 'shipped' WHERE ord...   \n",
       "3                        SELECT name age FROM users;   \n",
       "4                         DELETE FROM WHERE id = 10;   \n",
       "\n",
       "                                          PythonData  \\\n",
       "0     def greet(name):\\n    return f'Hello, {name}!'   \n",
       "1             import math\\narea = math.pi * (5 ** 2)   \n",
       "2                   if x = 10:\\n    print('x is 10')   \n",
       "3  def add(a, b  # Missing closing parenthesis\\n ...   \n",
       "4                              print 'Hello, World!'   \n",
       "\n",
       "   PII for Answer (by gpt3.5)  \n",
       "0                         0.0  \n",
       "1                         0.0  \n",
       "2                         0.0  \n",
       "3                         0.0  \n",
       "4                         0.0  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "openai_prompting.as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LLM as a Judge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_criteria = BinaryClassificationPromptTemplate(      \n",
    "        criteria = \"\"\"Conciseness refers to the quality of being brief and to the point, while still providing all necessary information.\n",
    "            A concise response should:\n",
    "            - Provide the necessary information without unnecessary details or repetition.\n",
    "            - Be brief yet comprehensive enough to address the query.\n",
    "            - Use simple and direct language to convey the message effectively.\n",
    "        \"\"\",\n",
    "        target_category=\"concise\",\n",
    "        non_target_category=\"verbose\",\n",
    "        uncertainty=\"unknown\",\n",
    "        include_reasoning=True,\n",
    "        pre_messages=[(\"system\", \"You are a judge which evaluates text.\")],\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_evals = Dataset.from_pandas(\n",
    "    pd.DataFrame(data),\n",
    "    data_definition=data_definition,\n",
    "    descriptors=[\n",
    "        NegativityLLMEval(\"Answer\"),\n",
    "        PIILLMEval(\"Answer\"),\n",
    "        DeclineLLMEval(\"Answer\"),\n",
    "        BiasLLMEval(\"Answer\"),\n",
    "        ToxicityLLMEval(\"Answer\"),\n",
    "        ContextQualityLLMEval(\"Answer\", question=\"Question\"), #here answer substitutes a context, cause there is no context \n",
    "        LLMEval(\"Answer\", template=custom_criteria, provider = \"openai\", model = \"gpt-4o-mini\", alias=\"Answer conciseness\")\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>ItemsToLookInQuestion</th>\n",
       "      <th>Feedback</th>\n",
       "      <th>DaysPassed</th>\n",
       "      <th>JsonData</th>\n",
       "      <th>JsonMatchLHS</th>\n",
       "      <th>JsonMatchRHS</th>\n",
       "      <th>SQLData</th>\n",
       "      <th>PythonData</th>\n",
       "      <th>...</th>\n",
       "      <th>Decline category.category</th>\n",
       "      <th>Decline category.reasoning</th>\n",
       "      <th>Bias category.category</th>\n",
       "      <th>Bias category.reasoning</th>\n",
       "      <th>Toxicity category.category</th>\n",
       "      <th>Toxicity category.reasoning</th>\n",
       "      <th>ContextQuality category.category</th>\n",
       "      <th>ContextQuality category.reasoning</th>\n",
       "      <th>Answer conciseness.category</th>\n",
       "      <th>Answer conciseness.reasoning</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How can I manage stress effectively?</td>\n",
       "      <td>Manage stress by practicing relaxation techniq...</td>\n",
       "      <td>stress</td>\n",
       "      <td>Positive</td>\n",
       "      <td>2</td>\n",
       "      <td>{\"isActive\": true, \"score\": 95}</td>\n",
       "      <td>{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}</td>\n",
       "      <td>{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}</td>\n",
       "      <td>SELECT * FROM users WHERE age &gt; 30;</td>\n",
       "      <td>def greet(name):\\n    return f'Hello, {name}!'</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text provides constructive advice on manag...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text provides general advice on managing s...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text provides supportive advice on managin...</td>\n",
       "      <td>VALID</td>\n",
       "      <td>The text provides specific strategies for mana...</td>\n",
       "      <td>concise</td>\n",
       "      <td>The text provides a clear and direct suggestio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the best way to treat a minor burn?</td>\n",
       "      <td>For a minor burn, cool the area with running w...</td>\n",
       "      <td>(stress, burn)</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>{\"colors\": [\"red\", \"green\", \"blue\"]}</td>\n",
       "      <td>{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...</td>\n",
       "      <td>{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}</td>\n",
       "      <td>INSERT INTO products (name, price) VALUES ('La...</td>\n",
       "      <td>import math\\narea = math.pi * (5 ** 2)</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text provides instructions for treating a ...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text provides a straightforward instructio...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text provides instructions for treating a ...</td>\n",
       "      <td>VALID</td>\n",
       "      <td>The text provides clear instructions for treat...</td>\n",
       "      <td>concise</td>\n",
       "      <td>The text provides clear and necessary instruct...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How often should I get a medical check-up?</td>\n",
       "      <td>I'm sorry, I cannot provide specific advice on...</td>\n",
       "      <td>stress</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"id\": 123, \"status\": \"complete\",}</td>\n",
       "      <td>{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}</td>\n",
       "      <td>{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}</td>\n",
       "      <td>UPDATE orders SET status = 'shipped' WHERE ord...</td>\n",
       "      <td>if x = 10:\\n    print('x is 10')</td>\n",
       "      <td>...</td>\n",
       "      <td>DECLINE</td>\n",
       "      <td>The text expresses a refusal to provide specif...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text does not demonstrate any inclination ...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text does not contain any harmful, offensi...</td>\n",
       "      <td>INVALID</td>\n",
       "      <td>The text does not provide any information on h...</td>\n",
       "      <td>concise</td>\n",
       "      <td>The response clearly states that specific advi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What foods should I eat to boost my immune sys...</td>\n",
       "      <td>I'm sorry, I cannot answer this question. Plea...</td>\n",
       "      <td>(food, eat)</td>\n",
       "      <td>Negative</td>\n",
       "      <td>1</td>\n",
       "      <td>{\"name\": \"Bob\", \"age\": 30}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}</td>\n",
       "      <td>SELECT name age FROM users;</td>\n",
       "      <td>def add(a, b  # Missing closing parenthesis\\n ...</td>\n",
       "      <td>...</td>\n",
       "      <td>DECLINE</td>\n",
       "      <td>The text expresses an inability to answer a qu...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text does not exhibit any inclination or p...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text does not contain any harmful, offensi...</td>\n",
       "      <td>INVALID</td>\n",
       "      <td>The text does not provide any information abou...</td>\n",
       "      <td>concise</td>\n",
       "      <td>The response effectively communicates that the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What medication should I take for depression w...</td>\n",
       "      <td>You could try taking something over-the-counte...</td>\n",
       "      <td>depression</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"items\": [\"apple\", \"banana\", \"cherry\", price:...</td>\n",
       "      <td>{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}</td>\n",
       "      <td>{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}</td>\n",
       "      <td>DELETE FROM WHERE id = 10;</td>\n",
       "      <td>print 'Hello, World!'</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text suggests a potential solution (taking...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text provides a suggestion for managing de...</td>\n",
       "      <td>OK</td>\n",
       "      <td>The text suggests using over-the-counter medic...</td>\n",
       "      <td>INVALID</td>\n",
       "      <td>The text suggests taking over-the-counter medi...</td>\n",
       "      <td>concise</td>\n",
       "      <td>The suggestion to take over-the-counter medica...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Question  \\\n",
       "0               How can I manage stress effectively?   \n",
       "1        What is the best way to treat a minor burn?   \n",
       "2         How often should I get a medical check-up?   \n",
       "3  What foods should I eat to boost my immune sys...   \n",
       "4  What medication should I take for depression w...   \n",
       "\n",
       "                                              Answer ItemsToLookInQuestion  \\\n",
       "0  Manage stress by practicing relaxation techniq...                stress   \n",
       "1  For a minor burn, cool the area with running w...        (stress, burn)   \n",
       "2  I'm sorry, I cannot provide specific advice on...                stress   \n",
       "3  I'm sorry, I cannot answer this question. Plea...           (food, eat)   \n",
       "4  You could try taking something over-the-counte...            depression   \n",
       "\n",
       "   Feedback  DaysPassed                                           JsonData  \\\n",
       "0  Positive           2                    {\"isActive\": true, \"score\": 95}   \n",
       "1      None          14               {\"colors\": [\"red\", \"green\", \"blue\"]}   \n",
       "2      None           0                 {\"id\": 123, \"status\": \"complete\",}   \n",
       "3  Negative           1                         {\"name\": \"Bob\", \"age\": 30}   \n",
       "4  Negative           0  {\"items\": [\"apple\", \"banana\", \"cherry\", price:...   \n",
       "\n",
       "                                        JsonMatchLHS  \\\n",
       "0     {\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}   \n",
       "1  { \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...   \n",
       "2       {\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}   \n",
       "4      {\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}   \n",
       "\n",
       "                                     JsonMatchRHS  \\\n",
       "0  {\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}   \n",
       "1     {\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}   \n",
       "2      {\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}   \n",
       "4  {\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}   \n",
       "\n",
       "                                             SQLData  \\\n",
       "0                SELECT * FROM users WHERE age > 30;   \n",
       "1  INSERT INTO products (name, price) VALUES ('La...   \n",
       "2  UPDATE orders SET status = 'shipped' WHERE ord...   \n",
       "3                        SELECT name age FROM users;   \n",
       "4                         DELETE FROM WHERE id = 10;   \n",
       "\n",
       "                                          PythonData  ...  \\\n",
       "0     def greet(name):\\n    return f'Hello, {name}!'  ...   \n",
       "1             import math\\narea = math.pi * (5 ** 2)  ...   \n",
       "2                   if x = 10:\\n    print('x is 10')  ...   \n",
       "3  def add(a, b  # Missing closing parenthesis\\n ...  ...   \n",
       "4                              print 'Hello, World!'  ...   \n",
       "\n",
       "  Decline category.category  \\\n",
       "0                        OK   \n",
       "1                        OK   \n",
       "2                   DECLINE   \n",
       "3                   DECLINE   \n",
       "4                        OK   \n",
       "\n",
       "                          Decline category.reasoning Bias category.category  \\\n",
       "0  The text provides constructive advice on manag...                     OK   \n",
       "1  The text provides instructions for treating a ...                     OK   \n",
       "2  The text expresses a refusal to provide specif...                     OK   \n",
       "3  The text expresses an inability to answer a qu...                     OK   \n",
       "4  The text suggests a potential solution (taking...                     OK   \n",
       "\n",
       "                             Bias category.reasoning  \\\n",
       "0  The text provides general advice on managing s...   \n",
       "1  The text provides a straightforward instructio...   \n",
       "2  The text does not demonstrate any inclination ...   \n",
       "3  The text does not exhibit any inclination or p...   \n",
       "4  The text provides a suggestion for managing de...   \n",
       "\n",
       "  Toxicity category.category  \\\n",
       "0                         OK   \n",
       "1                         OK   \n",
       "2                         OK   \n",
       "3                         OK   \n",
       "4                         OK   \n",
       "\n",
       "                         Toxicity category.reasoning  \\\n",
       "0  The text provides supportive advice on managin...   \n",
       "1  The text provides instructions for treating a ...   \n",
       "2  The text does not contain any harmful, offensi...   \n",
       "3  The text does not contain any harmful, offensi...   \n",
       "4  The text suggests using over-the-counter medic...   \n",
       "\n",
       "  ContextQuality category.category  \\\n",
       "0                            VALID   \n",
       "1                            VALID   \n",
       "2                          INVALID   \n",
       "3                          INVALID   \n",
       "4                          INVALID   \n",
       "\n",
       "                   ContextQuality category.reasoning  \\\n",
       "0  The text provides specific strategies for mana...   \n",
       "1  The text provides clear instructions for treat...   \n",
       "2  The text does not provide any information on h...   \n",
       "3  The text does not provide any information abou...   \n",
       "4  The text suggests taking over-the-counter medi...   \n",
       "\n",
       "  Answer conciseness.category  \\\n",
       "0                     concise   \n",
       "1                     concise   \n",
       "2                     concise   \n",
       "3                     concise   \n",
       "4                     concise   \n",
       "\n",
       "                        Answer conciseness.reasoning  \n",
       "0  The text provides a clear and direct suggestio...  \n",
       "1  The text provides clear and necessary instruct...  \n",
       "2  The response clearly states that specific advi...  \n",
       "3  The response effectively communicates that the...  \n",
       "4  The suggestion to take over-the-counter medica...  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm_evals.as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom descriptors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-31T11:44:16.979569Z",
     "start_time": "2024-12-31T11:44:16.976971Z"
    }
   },
   "outputs": [],
   "source": [
    "#a custom funtion to apply over a single column and return a single column\n",
    "def is_empty_string_callable(data: DatasetColumn) -> DatasetColumn:\n",
    "    return DatasetColumn(type=\"cat\", \n",
    "                         data=pd.Series([\"EMPTY\" if val == \"\" else \"NON EMPTY\" for val in data.data])\n",
    "                        )\n",
    "\n",
    "#a custom funtion to apply over multiple columns and return a single column\n",
    "def exact_match_callable(dataset: Dataset) -> DatasetColumn:\n",
    "    return DatasetColumn(type=\"cat\",\n",
    "                         data=pd.Series([\"MATCH\" if val else \"MISMATCH\" for val in dataset.column(\"JsonMatchLHS\").data == dataset.column(\"JsonMatchRHS\").data])\n",
    "                        )\n",
    "\n",
    "#a custom funtion to apply over multiple columns and return multiple columns\n",
    "def concat_question_answer_callable(dataset: Dataset) -> Union[DatasetColumn, Dict[str, DatasetColumn]]:\n",
    "    return {\n",
    "        \"reversed_question\": DatasetColumn(type=\"cat\", data=pd.Series([value[::-1] for value in dataset.column(\"Question\").data])),\n",
    "        \"reversed_answer\": DatasetColumn(type=\"cat\", data=pd.Series([value[::-1] for value in dataset.column(\"Answer\").data])),\n",
    "           }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_descriptors = Dataset.from_pandas(\n",
    "    pd.DataFrame(data),\n",
    "    data_definition=data_definition,\n",
    "    descriptors=[\n",
    "        CustomColumnDescriptor(\"Question\", is_empty_string_callable, alias=\"is Question empty?\"),\n",
    "        CustomDescriptor(exact_match_callable, alias=\"Match between JsonMatchLHS and JsonMatchRHS\"),\n",
    "        CustomDescriptor(concat_question_answer_callable),\n",
    "    ],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>ItemsToLookInQuestion</th>\n",
       "      <th>Feedback</th>\n",
       "      <th>DaysPassed</th>\n",
       "      <th>JsonData</th>\n",
       "      <th>JsonMatchLHS</th>\n",
       "      <th>JsonMatchRHS</th>\n",
       "      <th>SQLData</th>\n",
       "      <th>PythonData</th>\n",
       "      <th>is Question empty?</th>\n",
       "      <th>Match between JsonMatchLHS and JsonMatchRHS</th>\n",
       "      <th>custom_descriptor:concat_question_answer_callable.reversed_question</th>\n",
       "      <th>custom_descriptor:concat_question_answer_callable.reversed_answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How can I manage stress effectively?</td>\n",
       "      <td>Manage stress by practicing relaxation techniq...</td>\n",
       "      <td>stress</td>\n",
       "      <td>Positive</td>\n",
       "      <td>2</td>\n",
       "      <td>{\"isActive\": true, \"score\": 95}</td>\n",
       "      <td>{\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}</td>\n",
       "      <td>{\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}</td>\n",
       "      <td>SELECT * FROM users WHERE age &gt; 30;</td>\n",
       "      <td>def greet(name):\\n    return f'Hello, {name}!'</td>\n",
       "      <td>NON EMPTY</td>\n",
       "      <td>MISMATCH</td>\n",
       "      <td>?ylevitceffe sserts eganam I nac woH</td>\n",
       "      <td>moc.tsohlacol//:ptth knil siht tuo kcehC .noit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the best way to treat a minor burn?</td>\n",
       "      <td>For a minor burn, cool the area with running w...</td>\n",
       "      <td>(stress, burn)</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>{\"colors\": [\"red\", \"green\", \"blue\"]}</td>\n",
       "      <td>{ \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...</td>\n",
       "      <td>{\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}</td>\n",
       "      <td>INSERT INTO products (name, price) VALUES ('La...</td>\n",
       "      <td>import math\\narea = math.pi * (5 ** 2)</td>\n",
       "      <td>NON EMPTY</td>\n",
       "      <td>MISMATCH</td>\n",
       "      <td>?nrub ronim a taert ot yaw tseb eht si tahW</td>\n",
       "      <td>.gnisserd elirets a htiw ti revoc dna setunim ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How often should I get a medical check-up?</td>\n",
       "      <td>I'm sorry, I cannot provide specific advice on...</td>\n",
       "      <td>stress</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"id\": 123, \"status\": \"complete\",}</td>\n",
       "      <td>{\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}</td>\n",
       "      <td>{\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}</td>\n",
       "      <td>UPDATE orders SET status = 'shipped' WHERE ord...</td>\n",
       "      <td>if x = 10:\\n    print('x is 10')</td>\n",
       "      <td>NON EMPTY</td>\n",
       "      <td>MISMATCH</td>\n",
       "      <td>?pu-kcehc lacidem a teg I dluohs netfo woH</td>\n",
       "      <td>.lanoisseforp erachtlaeh a tlusnoc esaelP .noi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What foods should I eat to boost my immune sys...</td>\n",
       "      <td>I'm sorry, I cannot answer this question. Plea...</td>\n",
       "      <td>(food, eat)</td>\n",
       "      <td>Negative</td>\n",
       "      <td>1</td>\n",
       "      <td>{\"name\": \"Bob\", \"age\": 30}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}</td>\n",
       "      <td>{\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}</td>\n",
       "      <td>SELECT name age FROM users;</td>\n",
       "      <td>def add(a, b  # Missing closing parenthesis\\n ...</td>\n",
       "      <td>NON EMPTY</td>\n",
       "      <td>MISMATCH</td>\n",
       "      <td>?metsys enummi ym tsoob ot tae I dluohs sdoof ...</td>\n",
       "      <td>.redivorp erachtlaeh desnecil a morf ecivda ke...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What medication should I take for depression w...</td>\n",
       "      <td>You could try taking something over-the-counte...</td>\n",
       "      <td>depression</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0</td>\n",
       "      <td>{\"items\": [\"apple\", \"banana\", \"cherry\", price:...</td>\n",
       "      <td>{\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}</td>\n",
       "      <td>{\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}</td>\n",
       "      <td>DELETE FROM WHERE id = 10;</td>\n",
       "      <td>print 'Hello, World!'</td>\n",
       "      <td>NON EMPTY</td>\n",
       "      <td>MISMATCH</td>\n",
       "      <td>?rotcod a gnitlusnoc tuohtiw noisserped rof ek...</td>\n",
       "      <td>.noisserped rof ,neforpubi ro niripsa ekil ,re...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Question  \\\n",
       "0               How can I manage stress effectively?   \n",
       "1        What is the best way to treat a minor burn?   \n",
       "2         How often should I get a medical check-up?   \n",
       "3  What foods should I eat to boost my immune sys...   \n",
       "4  What medication should I take for depression w...   \n",
       "\n",
       "                                              Answer ItemsToLookInQuestion  \\\n",
       "0  Manage stress by practicing relaxation techniq...                stress   \n",
       "1  For a minor burn, cool the area with running w...        (stress, burn)   \n",
       "2  I'm sorry, I cannot provide specific advice on...                stress   \n",
       "3  I'm sorry, I cannot answer this question. Plea...           (food, eat)   \n",
       "4  You could try taking something over-the-counte...            depression   \n",
       "\n",
       "   Feedback  DaysPassed                                           JsonData  \\\n",
       "0  Positive           2                    {\"isActive\": true, \"score\": 95}   \n",
       "1      None          14               {\"colors\": [\"red\", \"green\", \"blue\"]}   \n",
       "2      None           0                 {\"id\": 123, \"status\": \"complete\",}   \n",
       "3  Negative           1                         {\"name\": \"Bob\", \"age\": 30}   \n",
       "4  Negative           0  {\"items\": [\"apple\", \"banana\", \"cherry\", price:...   \n",
       "\n",
       "                                        JsonMatchLHS  \\\n",
       "0     {\"name\": \"Alice\", \"age\": 25, \"city\": \"London\"}   \n",
       "1  { \"name\" : \"Bob\" , \"age\" : 22 , \"city\" : \"Pari...   \n",
       "2       {\"name\": \"Eve\", \"age\": 28, \"city\": \"Berlin\"}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"country\": \"USA\"}   \n",
       "4      {\"name\": \"David\", \"age\": 35, \"city\": \"Tokyo\"}   \n",
       "\n",
       "                                     JsonMatchRHS  \\\n",
       "0  {\"city\": \"London\", \"age\": 25, \"name\": \"Alice\"}   \n",
       "1     {\"city\": \"Paris\", \"name\": \"Bob\", \"age\": 22}   \n",
       "2      {\"city\": \"Berlin\", \"age\": 28, \"name\": Eve}   \n",
       "3   {\"name\": \"Charlie\", \"age\": 30, \"city\": \"USA\"}   \n",
       "4  {\"city\": \"Tokyo\", \"age\": 35, \"name\": \"Daniel\"}   \n",
       "\n",
       "                                             SQLData  \\\n",
       "0                SELECT * FROM users WHERE age > 30;   \n",
       "1  INSERT INTO products (name, price) VALUES ('La...   \n",
       "2  UPDATE orders SET status = 'shipped' WHERE ord...   \n",
       "3                        SELECT name age FROM users;   \n",
       "4                         DELETE FROM WHERE id = 10;   \n",
       "\n",
       "                                          PythonData is Question empty?  \\\n",
       "0     def greet(name):\\n    return f'Hello, {name}!'          NON EMPTY   \n",
       "1             import math\\narea = math.pi * (5 ** 2)          NON EMPTY   \n",
       "2                   if x = 10:\\n    print('x is 10')          NON EMPTY   \n",
       "3  def add(a, b  # Missing closing parenthesis\\n ...          NON EMPTY   \n",
       "4                              print 'Hello, World!'          NON EMPTY   \n",
       "\n",
       "  Match between JsonMatchLHS and JsonMatchRHS  \\\n",
       "0                                    MISMATCH   \n",
       "1                                    MISMATCH   \n",
       "2                                    MISMATCH   \n",
       "3                                    MISMATCH   \n",
       "4                                    MISMATCH   \n",
       "\n",
       "  custom_descriptor:concat_question_answer_callable.reversed_question  \\\n",
       "0               ?ylevitceffe sserts eganam I nac woH                    \n",
       "1        ?nrub ronim a taert ot yaw tseb eht si tahW                    \n",
       "2         ?pu-kcehc lacidem a teg I dluohs netfo woH                    \n",
       "3  ?metsys enummi ym tsoob ot tae I dluohs sdoof ...                    \n",
       "4  ?rotcod a gnitlusnoc tuohtiw noisserped rof ek...                    \n",
       "\n",
       "  custom_descriptor:concat_question_answer_callable.reversed_answer  \n",
       "0  moc.tsohlacol//:ptth knil siht tuo kcehC .noit...                 \n",
       "1  .gnisserd elirets a htiw ti revoc dna setunim ...                 \n",
       "2  .lanoisseforp erachtlaeh a tlusnoc esaelP .noi...                 \n",
       "3  .redivorp erachtlaeh desnecil a morf ecivda ke...                 \n",
       "4  .noisserped rof ,neforpubi ro niripsa ekil ,re...                 "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "custom_descriptors.as_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}