Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TREC DL and LLM AggreFact experiments for relevance benchmark + prompts comparisons and groundedness vs Bespoke Minicheck 7B #1660

Merged
merged 23 commits into from
Dec 10, 2024
Merged

TREC DL and LLM AggreFact experiments for relevance benchmark + prompts comparisons and groundedness vs Bespoke Minicheck 7B #1660

Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
5aea893
save
sfc-gh-dhuang Oct 29, 2024
6a8d32e
improve trubasicapp setup for competitive experiments, add mlflow ins…
sfc-gh-dhuang Oct 31, 2024
7968d52
save
sfc-gh-dhuang Oct 31, 2024
6b91ffe
save progress
sfc-gh-dhuang Nov 12, 2024
65d92ce
save
sfc-gh-dhuang Nov 12, 2024
b8325a4
save
sfc-gh-dhuang Nov 25, 2024
cf5979a
Merge branch 'daniel/prompt-opt' into trec-exp-adding-rubric
sfc-gh-dhuang Nov 25, 2024
17941b0
add agreement analysis with scoreddocs
sfc-gh-dhuang Nov 25, 2024
fbe2ae4
notebook updates
sfc-gh-dhuang Dec 3, 2024
dd3c1f7
Merge branch 'main' into trec-exp-adding-rubric
sfc-gh-dhuang Dec 3, 2024
99d2a2b
cleanup
sfc-gh-dhuang Dec 3, 2024
78e3fb5
dataset preprocessing script update
sfc-gh-dhuang Dec 3, 2024
4206707
cleanup competitive analysis
sfc-gh-dhuang Dec 3, 2024
8554d04
Merge branch 'main' into trec-exp-adding-rubric
sfc-gh-dhuang Dec 3, 2024
fb9c246
add llm aggrefact nb
sfc-gh-dhuang Dec 4, 2024
35432e7
add llm-aggrefact experiment notebook
sfc-gh-dhuang Dec 5, 2024
360fbb8
Merge branch 'main' into trec-exp-adding-rubric
sfc-gh-dhuang Dec 5, 2024
cb743c5
move notebooks
sfc-gh-dhuang Dec 5, 2024
680eab4
Merge branch 'main' into trec-exp-adding-rubric
sfc-gh-dhuang Dec 5, 2024
9514bce
edits
sfc-gh-dhuang Dec 9, 2024
4b49b7b
pr comments
sfc-gh-dhuang Dec 9, 2024
d0909a7
Merge branch 'main' into trec-exp-adding-rubric
sfc-gh-dhuang Dec 10, 2024
0975049
add back e2e/data nb
sfc-gh-dhuang Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
save
  • Loading branch information
@sfc-gh-dhuang
sfc-gh-dhuang committed Nov 25, 2024
commit 7968d5218afdf68605986367bdfc137c92855faf
64 changes: 44 additions & 20 deletions src/benchmark/trulens/benchmark/benchmark_frameworks/experiments/competitive_analysis.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@
" 1 if label >= 0.5 else 0 for label in qags_xsum_true_labels\n",
"]\n",
"combined_dataset = pd.concat(\n",
" [qags_cnn_dm, qags_xsum, summeval], ignore_index=True\n",
" [qags_cnn_dm, qags_xsum, summeval], ignore_index=False\n",
")\n",
"combined_true_labels = (\n",
" qags_cnn_dm_true_labels + qags_xsum_true_labels + summeval_true_labels\n",
Expand All @@ -123,6 +123,15 @@
"print(f\"Total number of samples: {len(combined_dataset)}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"combined_dataset"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -158,6 +167,9 @@
"from ragas.llms import llm_factory\n",
"from ragas.metrics import faithfulness as faithfulness_ragas\n",
"from trulens.apps.basic import TruBasicApp\n",
"from trulens.core import Feedback\n",
"from trulens.core import Select\n",
"from trulens.feedback import GroundTruthAgreement\n",
"from trulens.providers.openai import OpenAI\n",
"\n",
"OPENAI_LLM_NAME = \"gpt-4o-mini\"\n",
Expand Down Expand Up @@ -233,8 +245,20 @@
" if len(dataset_df) != len(true_labels):\n",
" raise ValueError(\"dataset df must have the same length as labels\")\n",
"\n",
" ground_truth = GroundTruthAgreement(dataset_df, provider=gpt_4o_mini)\n",
"\n",
" f_mae = (\n",
" Feedback(ground_truth.absolute_error, name=\"Mean Absolute Error\")\n",
" .on(Select.Record.calls[0].args.args[0])\n",
" .on(Select.Record.calls[0].args.args[1])\n",
" .on_output()\n",
" )\n",
"\n",
" tru_wrapped_basic_app = TruBasicApp(\n",
" evaluate_func_wrapper, app_name=app_name, app_version=app_version\n",
" evaluate_func_wrapper,\n",
" app_name=app_name,\n",
" app_version=app_version,\n",
" feedbacks=[f_mae],\n",
" )\n",
"\n",
" for i in range(len(dataset_df)):\n",
Expand All @@ -261,26 +285,26 @@
"run_experiment_and_record(\n",
" evaluate_func_wrapper=trulens_groundedness,\n",
" app_name=\"trulens-groundedness\",\n",
" app_version=\"10302024\",\n",
" dataset_df=combined_dataset,\n",
" true_labels=combined_true_labels,\n",
")\n",
"\n",
"run_experiment_and_record(\n",
" evaluate_func_wrapper=ragas_faithfulness,\n",
" app_name=\"ragas-faithfulness\",\n",
" app_version=\"10302024\",\n",
" dataset_df=combined_dataset,\n",
" true_labels=combined_true_labels,\n",
" app_version=\"10312024\",\n",
" dataset_df=combined_dataset[:2],\n",
" true_labels=combined_true_labels[:2],\n",
")\n",
"\n",
"run_experiment_and_record(\n",
" evaluate_func_wrapper=mlflow_faithfulness,\n",
" app_name=\"mlflow-faithfulness\",\n",
" app_version=\"10302024\",\n",
" dataset_df=combined_dataset,\n",
" true_labels=combined_true_labels,\n",
")"
"# run_experiment_and_record(\n",
"# evaluate_func_wrapper=ragas_faithfulness,\n",
"# app_name=\"ragas-faithfulness\",\n",
"# app_version=\"10312024\",\n",
"# dataset_df=combined_dataset,\n",
"# true_labels=combined_true_labels,\n",
"# )\n",
"\n",
"# run_experiment_and_record(\n",
"# evaluate_func_wrapper=mlflow_faithfulness,\n",
"# app_name=\"mlflow-faithfulness\",\n",
"# app_version=\"1032024\",\n",
"# dataset_df=combined_dataset,\n",
"# true_labels=combined_true_labels,\n",
"# )"
]
},
{
Expand Down