Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TREC DL and LLM AggreFact experiments for relevance benchmark + prompts comparisons and groundedness vs Bespoke Minicheck 7B #1660

Merged
merged 23 commits into from
Dec 10, 2024
Merged

TREC DL and LLM AggreFact experiments for relevance benchmark + prompts comparisons and groundedness vs Bespoke Minicheck 7B #1660

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
5aea893
save
sfc-gh-dhuang Oct 29, 2024
6a8d32e
improve trubasicapp setup for competitive experiments, add mlflow ins…
sfc-gh-dhuang Oct 31, 2024
7968d52
save
sfc-gh-dhuang Oct 31, 2024
6b91ffe
save progress
sfc-gh-dhuang Nov 12, 2024
65d92ce
save
sfc-gh-dhuang Nov 12, 2024
b8325a4
save
sfc-gh-dhuang Nov 25, 2024
cf5979a
Merge branch 'daniel/prompt-opt' into trec-exp-adding-rubric
sfc-gh-dhuang Nov 25, 2024
17941b0
add agreement analysis with scoreddocs
sfc-gh-dhuang Nov 25, 2024
fbe2ae4
notebook updates
sfc-gh-dhuang Dec 3, 2024
dd3c1f7
Merge branch 'main' into trec-exp-adding-rubric
sfc-gh-dhuang Dec 3, 2024
99d2a2b
cleanup
sfc-gh-dhuang Dec 3, 2024
78e3fb5
dataset preprocessing script update
sfc-gh-dhuang Dec 3, 2024
4206707
cleanup competitive analysis
sfc-gh-dhuang Dec 3, 2024
8554d04
Merge branch 'main' into trec-exp-adding-rubric
sfc-gh-dhuang Dec 3, 2024
fb9c246
add llm aggrefact nb
sfc-gh-dhuang Dec 4, 2024
35432e7
add llm-aggrefact experiment notebook
sfc-gh-dhuang Dec 5, 2024
360fbb8
Merge branch 'main' into trec-exp-adding-rubric
sfc-gh-dhuang Dec 5, 2024
cb743c5
move notebooks
sfc-gh-dhuang Dec 5, 2024
680eab4
Merge branch 'main' into trec-exp-adding-rubric
sfc-gh-dhuang Dec 5, 2024
9514bce
edits
sfc-gh-dhuang Dec 9, 2024
4b49b7b
pr comments
sfc-gh-dhuang Dec 9, 2024
d0909a7
Merge branch 'main' into trec-exp-adding-rubric
sfc-gh-dhuang Dec 10, 2024
0975049
add back e2e/data nb
sfc-gh-dhuang Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
save
  • Loading branch information
@sfc-gh-dhuang
sfc-gh-dhuang committed Nov 25, 2024
commit 65d92cea4687a27d7bffcaa13003f25f1c65cfa8
97 changes: 49 additions & 48 deletions src/benchmark/trulens/benchmark/benchmark_frameworks/experiments/competitive_analysis.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,45 +196,6 @@
"session.reset_database()"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"from trulens.core import Provider\n",
"\n",
"\n",
"THRESHOLD = 0.33\n",
"class CustomTermFeedback(Provider):\n",
" def true_positive(self, output: str) -> float:\n",
" feedback_score, gt_score = float(output.split(\";\")[0]), float(output.split(\";\")[1])\n",
" binary_score = 1 if feedback_score >= 0.5 else 0\n",
" binary_gt_score = 1 if gt_score >= THRESHOLD else 0\n",
" return 1.0 if binary_score == 1 and binary_gt_score == 1 else 0.0\n",
" \n",
" def true_negative(self, output: str) -> float:\n",
" feedback_score, gt_score = float(output.split(\";\")[0]), float(output.split(\";\")[1])\n",
" binary_score = 1 if feedback_score >= 0.5 else 0\n",
" binary_gt_score = 1 if gt_score >= THRESHOLD else 0\n",
" return 1.0 if binary_score == 0 and binary_gt_score == 0 else 0.0\n",
"\n",
" def false_positive(self, output: str) -> float:\n",
" feedback_score, gt_score = float(output.split(\";\")[0]), float(output.split(\";\")[1])\n",
" binary_score = 1 if feedback_score >= 0.5 else 0\n",
" binary_gt_score = 1 if gt_score >= THRESHOLD else 0\n",
" return 1.0 if binary_score == 1 and binary_gt_score == 0 else 0.0\n",
" def false_negative(self, output: str) -> float:\n",
" feedback_score, gt_score = float(output.split(\";\")[0]), float(output.split(\";\")[1])\n",
" binary_score = 1 if feedback_score >= 0.5 else 0\n",
" binary_gt_score = 1 if gt_score >= THRESHOLD else 0\n",
" return 1.0 if binary_score == 0 and binary_gt_score == 1 else 0.0\n",
" \n",
" def term_absolute_error(self, output: str) -> float:\n",
" feedback_score, gt_score = float(output.split(\";\")[0]), float(output.split(\";\")[1])\n",
" return abs(feedback_score - gt_score) "
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -790,15 +751,17 @@
},
{
"cell_type": "code",
"execution_count": 31,
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"from trulens.benchmark.benchmark_frameworks.experiments.dataset_preprocessing import (\n",
" generate_ms_marco_trec_dl_annotation_benchmark,\n",
")\n",
"\n",
"trec_doc_2022 = list(generate_ms_marco_trec_dl_annotation_benchmark(dataset_path=\"msmarco-document-v2/trec-dl-2022\", max_samples_per_bucket=150))"
"# trec_doc_2022 = list(generate_ms_marco_trec_dl_annotation_benchmark(dataset_path=\"msmarco-document-v2/trec-dl-2022\", max_samples_per_bucket=150))\n",
"\n",
"trec_passage_2022 = list(generate_ms_marco_trec_dl_annotation_benchmark(dataset_path=\"msmarco-passage-v2/trec-dl-2022\", max_samples_per_bucket=150))"
]
},
{
Expand All @@ -818,7 +781,7 @@
}
],
"source": [
"len(trec_doc_2022)"
"len(trec_passage_2022)"
]
},
{
Expand All @@ -827,7 +790,7 @@
"metadata": {},
"outputs": [],
"source": [
"trec_doc_2022_true_labels = [entry[\"expected_score\"] for entry in trec_doc_2022]"
"trec_passage_2022_true_labels = [entry[\"expected_score\"] for entry in trec_passage_2022]"
]
},
{
Expand All @@ -851,7 +814,7 @@
" visualize_expected_score_distribution,\n",
")\n",
"\n",
"visualize_expected_score_distribution(trec_doc_2022_true_labels)"
"visualize_expected_score_distribution(trec_passage_2022_true_labels)"
]
},
{
Expand Down Expand Up @@ -937,6 +900,43 @@
"from trulens.core import Feedback\n",
"\n",
"\n",
"from trulens.core import Provider\n",
"\n",
"\n",
"THRESHOLD = 0.5 # for passage retrieval annotation, we consider a score of 0.5 or above as relevant\n",
"class CustomTermFeedback(Provider):\n",
" def true_positive(self, output: str) -> float:\n",
" feedback_score, gt_score = float(output.split(\";\")[0]), float(output.split(\";\")[1])\n",
" binary_score = 1 if feedback_score >= 0.5 else 0\n",
" binary_gt_score = 1 if gt_score >= THRESHOLD else 0\n",
" return 1.0 if binary_score == 1 and binary_gt_score == 1 else 0.0\n",
" \n",
" def true_negative(self, output: str) -> float:\n",
" feedback_score, gt_score = float(output.split(\";\")[0]), float(output.split(\";\")[1])\n",
" binary_score = 1 if feedback_score >= 0.5 else 0\n",
" binary_gt_score = 1 if gt_score >= THRESHOLD else 0\n",
" return 1.0 if binary_score == 0 and binary_gt_score == 0 else 0.0\n",
"\n",
" def false_positive(self, output: str) -> float:\n",
" feedback_score, gt_score = float(output.split(\";\")[0]), float(output.split(\";\")[1])\n",
" binary_score = 1 if feedback_score >= 0.5 else 0\n",
" binary_gt_score = 1 if gt_score >= THRESHOLD else 0\n",
" return 1.0 if binary_score == 1 and binary_gt_score == 0 else 0.0\n",
" def false_negative(self, output: str) -> float:\n",
" feedback_score, gt_score = float(output.split(\";\")[0]), float(output.split(\";\")[1])\n",
" binary_score = 1 if feedback_score >= 0.5 else 0\n",
" binary_gt_score = 1 if gt_score >= THRESHOLD else 0\n",
" return 1.0 if binary_score == 0 and binary_gt_score == 1 else 0.0\n",
" \n",
" def term_absolute_error(self, output: str) -> float:\n",
" feedback_score, gt_score = float(output.split(\";\")[0]), float(output.split(\";\")[1])\n",
" return abs(feedback_score - gt_score) \n",
" \n",
" def raw_gt_score(self, output: str) -> float:\n",
" return float(output.split(\";\")[1])\n",
" \n",
" def raw_feedback_score(self, output: str) -> float:\n",
" return float(output.split(\";\")[0])\n",
"\n",
"custom_term_feedback = CustomTermFeedback()\n",
"\n",
Expand All @@ -945,14 +945,15 @@
"f_fp = Feedback(custom_term_feedback.false_positive, name=\"False Positive\", higher_is_better=False).on_output()\n",
"f_fn = Feedback(custom_term_feedback.false_negative, name=\"False Negative\", higher_is_better=False).on_output()\n",
"f_abs_err = Feedback(custom_term_feedback.term_absolute_error, name=\"Absolute Error\", higher_is_better=False).on_output()\n",
"f_raw_gt_score = Feedback(custom_term_feedback.raw_gt_score, name=\"Raw GT Score\", higher_is_better=True).on_output()\n",
"f_raw_feedback_score = Feedback(custom_term_feedback.raw_feedback_score, name=\"Raw Feedback Score\", higher_is_better=True).on_output()\n",
"\n",
"\n",
"CUSTOM_FEEDBACK_FUNCS = [f_tp, f_tn, f_fp, f_fn, f_abs_err]\n",
"CUSTOM_FEEDBACK_FUNCS = [f_tp, f_tn, f_fp, f_fn, f_abs_err, f_raw_gt_score, f_raw_feedback_score]\n",
"\n",
"def run_experiment_for_provider(provider, dataset_df):\n",
" tru_wrapped_context_relevance_app = TruBasicApp(\n",
" trulens_context_relevance,\n",
" app_name=\"trec-dl-doc-2022-11102024\",\n",
" app_name=\"trec-dl-passage-2022\",\n",
" app_version=f\"{provider.model_engine}-context-relevance\",\n",
" feedbacks=CUSTOM_FEEDBACK_FUNCS,\n",
" )\n",
Expand All @@ -973,7 +974,7 @@
"\n",
"for provider in PROVIDERS:\n",
" print(f\"Running provider: {provider.model_engine}\")\n",
" run_experiment_for_provider(provider, trec_doc_2022)\n",
" run_experiment_for_provider(provider, trec_passage_2022)\n",
"\n",
"# with concurrent.futures.ThreadPoolExecutor() as executor:\n",
"# futures = [executor.submit(run_experiment_for_provider, provider, trec_doc_2022) for provider in PROVIDERS]\n",
Expand Down
2 changes: 1 addition & 1 deletion src/benchmark/trulens/benchmark/benchmark_frameworks/experiments/dataset_preprocessing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ def generate_balanced_ms_marco_hard_negatives_dataset(


def generate_ms_marco_trec_dl_annotation_benchmark(
dataset_path: str = "msmarco-document-v2/trec-dl-2022",
dataset_path: str = "msmarco-passage-v2/trec-dl-2022",
max_samples_per_bucket: int = 100,
):
dataset = ir_datasets.load(dataset_path)
Expand Down