tweet pretrained results

castorini · Mar 3, 2019 · e1e71fe · e1e71fe
1 parent 266923c
commit e1e71fe
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -34,4 +34,31 @@ score = Lambda * bm25 + (1.0-Lambda) * (bert_high_sent_1 + bert_high_sent_2/2)
 | 0.2511 | 0.85 |
 | 0.2495 |  0.9 |
 | 0.2484 | 0.95 |
-| 0.2451 | 1 |
+| 0.2451 | 1 |
+
+
+## BERT pretrained with Tweets
+
+|   MAP  |   Lambda   |
+|:------:|:----------:|
+| 0.2378 | 0    |
+| 0.2413 | 0.05 |
+| 0.2445 | 0.1  |
+| 0.2478 | 0.15 |
+| 0.2505 | 0.2  |
+| 0.2525 | 0.25 |
+| 0.2543 | 0.3  |
+| 0.2562 | 0.35 |
+| 0.2579 | 0.4  |
+| 0.2588 | 0.45 |
+| 0.2596 | 0.5  |
+| 0.2595 | 0.55 |
+| 0.2594 | 0.6  |
+| 0.2594 | 0.65 |
+| 0.2585 | 0.7  |
+| 0.2576 | 0.75 |
+| 0.2557 | 0.8  |
+| 0.2537 | 0.85 |
+| 0.2515 | 0.9  |
+| 0.2489 | 0.95 |
+| 0.2451 | 1    |
diff --git a/eval_bert.py b/eval_bert.py
@@ -49,7 +49,7 @@ def load_q_doc_bert(bertF, doc_dict, q_dict, bm25_dict, topK, w):
     score_dict = defaultdict(dict)
     with open(bertF) as bF:
         for line in bF:
-            q, _, d, _, score, _, _ = line.strip().split()
+            q, _, d, _, score, _ = line.strip().split()
             sent = doc_dict[d]
             doc = sent.split('_')[0]
             score = float(score)
@@ -87,7 +87,7 @@ def main():
     rel_dict, nonrel_dict, all_dict = load_nist_qrels()
     doc_dict, q_dict, score_dict = load_q_doc_bm25('robust04_bm25_test.csv')
     # eval_bm25('robust04_bm25_test.csv')
-    load_q_doc_bert('predict.trec_robust04_bm25_qa', doc_dict, q_dict,
+    load_q_doc_bert('prediction.trec_tweet_title', doc_dict, q_dict,
     	score_dict, 2, w)
 
 if __name__ == "__main__":