edits

truera · sfc-gh-dhuang · Dec 10, 2024 · Oct 29, 2024 · Oct 31, 2024 · Oct 31, 2024
commit 9514bce4bdea052cff59da48b07594aeabb94957
diff --git a/.gitignore b/.gitignore
@@ -54,6 +54,7 @@ credentials.toml
 **/results/*.csv
 **/experiments/mlruns/
 **/experiments/*.zip
+**/experiments/*/*.csv
 
 # test files generated by pytest
 test-output.xml

diff --git a/...benchmark_frameworks/experiments/context_relevance/trec_dl_passage_relevance_judges.ipynb b/...benchmark_frameworks/experiments/context_relevance/trec_dl_passage_relevance_judges.ipynb
@@ -213,13 +213,7 @@
     "\n",
     "\n",
     "trec_combined_df = pd.DataFrame(trec_combined)\n",
-    "trec_combined_df.to_csv(\n",
-    "    \"trec_dl_2021_2022_combined_scoreddocs_intervals.csv\", index=False\n",
-    ")\n",
     "\n",
-    "trec_combined_df = pd.read_csv(\n",
-    "    \"trec_dl_2021_2022_combined_scoreddocs_intervals.csv\"\n",
-    ")\n",
     "print(f\"Totoal number of samples: {len(trec_combined_df)}\")"
    ]
   },
@@ -517,7 +511,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "gaurav_prompt = \"\"\"\n",
+    "internal_prompt = \"\"\"\n",
     "You are an expert search result rater. You are given a user query and a search result. Your task is to rate the search result based on its relevance to the user query. You should rate the search result on a scale of 0 to 3, where:\n",
     "    0: The search result has no relevance to the user query.\n",
     "    1: The search result has low relevance to the user query. In this case the search result may contain some information which seems very slightly related to the user query but not enough information to answer the user query. The search result contains some references or very limited information about some entities present in the user query. In case the query is a statement on a topic, the search result should be tangentially related to it.\n",
@@ -676,7 +670,7 @@
     "\n",
     "\n",
     "# Function to rate context relevance\n",
-    "def gaurav_prompt_relevance(\n",
+    "def internal_prompt_relevance(\n",
     "    query: str, passage: str, model_engine=\"gpt-4o\"\n",
     ") -> dict:\n",
     "    # Prepare the prompt\n",
@@ -685,7 +679,7 @@
     "        messages=[\n",
     "            {\n",
     "                \"role\": \"system\",\n",
-    "                \"content\": gaurav_prompt,\n",
+    "                \"content\": internal_prompt,\n",
     "            },\n",
     "            {\n",
     "                \"role\": \"user\",\n",
@@ -824,11 +818,11 @@
     "        # print(f\"Query: {query}\")\n",
     "        # print(f\"Passage: {passage}\")\n",
     "\n",
-    "        # Gaurav Prompt\n",
-    "        gaurav_result = gaurav_prompt_relevance(\n",
+    "        # Snowflake internal prompt\n",
+    "        internal_result = internal_prompt_relevance(\n",
     "            query, passage, model_engine=model\n",
     "        )\n",
-    "        gaurav_rating = gaurav_result[\"rating\"]\n",
+    "        internal_rating = internal_result[\"rating\"]\n",
     "\n",
     "        # Umbrela Prompt\n",
     "        umbrela_result = umbrela_prompt_relevance(\n",
@@ -850,7 +844,7 @@
     "            \"query\": query,\n",
     "            \"passage\": passage,\n",
     "            \"ground_truth\": ground_truth,\n",
-    "            \"gaurav_rating\": gaurav_rating,\n",
+    "            \"internal_rating\": internal_rating,\n",
     "            \"umbrela_rating\": umbrela_rating,\n",
     "            \"categorical_rating\": categorical_rating,\n",
     "        })\n",
@@ -930,7 +924,7 @@
     "# Example usage\n",
     "csv_file = \"gpt-4o_3_prompts_results.csv\"\n",
     "plot_confusion_matrix_with_metrics(\n",
-    "    csv_file, \"gaurav_rating\", \"Gaurav Ratings: \"\n",
+    "    csv_file, \"internal_rating\", \"Internal Ratings: \"\n",
     ")\n",
     "plot_confusion_matrix_with_metrics(\n",
     "    csv_file, \"umbrela_rating\", \"Umbrela Ratings: \"\n",

diff --git a/tests/e2e/data/simple.ipynb b/tests/e2e/data/simple.ipynb