Refactor regression framework (castorini#1690)

First pass at addressing castorini#1680
dannychn11 · Dec 11, 2021 · fb119b0 · fb119b0
1 parent 12149f8
commit fb119b0
Show file tree

Hide file tree

Showing 182 changed files with 2,233 additions and 2,890 deletions.
diff --git a/docs/regressions-backgroundlinking18.md b/docs/regressions-backgroundlinking18.md
@@ -10,11 +10,11 @@ Typical indexing command:
 
 ```
 nohup sh target/appassembler/bin/IndexCollection -collection WashingtonPostCollection \
- -input /path/to/backgroundlinking18 \
- -index indexes/lucene-index.core18.pos+docvectors+raw \
+ -input /path/to/wapo.v2 \
+ -index indexes/lucene-index.wapo.v2.pos+docvectors+raw \
  -generator WashingtonPostGenerator \
  -threads 1 -storePositions -storeDocvectors -storeRaw \
-  >& logs/log.backgroundlinking18 &
+  >& logs/log.wapo.v2 &
 ```
 
 The directory `/path/to/core18/` should be the root directory of the [TREC Washington Post Corpus](https://trec.nist.gov/data/wapost/), i.e., `ls /path/to/core18/`
@@ -32,30 +32,30 @@ Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/m
 After indexing has completed, you should be able to perform retrieval as follows:
 
 ```
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.core18.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.wapo.v2.pos+docvectors+raw \
  -topicreader BackgroundLinking -topics src/main/resources/topics-and-qrels/topics.backgroundlinking18.txt \
- -output runs/run.backgroundlinking18.bm25.topics.backgroundlinking18.txt \
+ -output runs/run.wapo.v2.bm25.topics.backgroundlinking18.txt \
  -backgroundlinking -backgroundlinking.k 100 -bm25 -hits 100 &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.core18.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.wapo.v2.pos+docvectors+raw \
  -topicreader BackgroundLinking -topics src/main/resources/topics-and-qrels/topics.backgroundlinking18.txt \
- -output runs/run.backgroundlinking18.bm25+rm3.topics.backgroundlinking18.txt \
+ -output runs/run.wapo.v2.bm25+rm3.topics.backgroundlinking18.txt \
  -backgroundlinking -backgroundlinking.k 100 -bm25 -rm3 -hits 100 &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.core18.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.wapo.v2.pos+docvectors+raw \
  -topicreader BackgroundLinking -topics src/main/resources/topics-and-qrels/topics.backgroundlinking18.txt \
- -output runs/run.backgroundlinking18.bm25+rm3+df.topics.backgroundlinking18.txt \
+ -output runs/run.wapo.v2.bm25+rm3+df.topics.backgroundlinking18.txt \
  -backgroundlinking -backgroundlinking.datefilter -backgroundlinking.k 100 -bm25 -rm3 -hits 100 &
 ```
 
 Evaluation can be performed using `trec_eval`:
 
 ```
-tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking18.txt runs/run.backgroundlinking18.bm25.topics.backgroundlinking18.txt
+tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking18.txt runs/run.wapo.v2.bm25.topics.backgroundlinking18.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking18.txt runs/run.backgroundlinking18.bm25+rm3.topics.backgroundlinking18.txt
+tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking18.txt runs/run.wapo.v2.bm25+rm3.topics.backgroundlinking18.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking18.txt runs/run.backgroundlinking18.bm25+rm3+df.topics.backgroundlinking18.txt
+tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking18.txt runs/run.wapo.v2.bm25+rm3+df.topics.backgroundlinking18.txt
 ```
 
 ## Effectiveness

diff --git a/docs/regressions-backgroundlinking19.md b/docs/regressions-backgroundlinking19.md
@@ -10,11 +10,11 @@ Typical indexing command:
 
 ```
 nohup sh target/appassembler/bin/IndexCollection -collection WashingtonPostCollection \
- -input /path/to/backgroundlinking19 \
- -index indexes/lucene-index.core18.pos+docvectors+raw \
+ -input /path/to/wapo.v2 \
+ -index indexes/lucene-index.wapo.v2.pos+docvectors+raw \
  -generator WashingtonPostGenerator \
  -threads 1 -storePositions -storeDocvectors -storeRaw \
-  >& logs/log.backgroundlinking19 &
+  >& logs/log.wapo.v2 &
 ```
 
 The directory `/path/to/core18/` should be the root directory of the [TREC Washington Post Corpus](https://trec.nist.gov/data/wapost/), i.e., `ls /path/to/core18/`
@@ -32,30 +32,30 @@ Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/m
 After indexing has completed, you should be able to perform retrieval as follows:
 
 ```
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.core18.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.wapo.v2.pos+docvectors+raw \
  -topicreader BackgroundLinking -topics src/main/resources/topics-and-qrels/topics.backgroundlinking19.txt \
- -output runs/run.backgroundlinking19.bm25.topics.backgroundlinking19.txt \
+ -output runs/run.wapo.v2.bm25.topics.backgroundlinking19.txt \
  -backgroundlinking -backgroundlinking.k 100 -bm25 -hits 100 &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.core18.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.wapo.v2.pos+docvectors+raw \
  -topicreader BackgroundLinking -topics src/main/resources/topics-and-qrels/topics.backgroundlinking19.txt \
- -output runs/run.backgroundlinking19.bm25+rm3.topics.backgroundlinking19.txt \
+ -output runs/run.wapo.v2.bm25+rm3.topics.backgroundlinking19.txt \
  -backgroundlinking -backgroundlinking.k 100 -bm25 -rm3 -hits 100 &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.core18.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.wapo.v2.pos+docvectors+raw \
  -topicreader BackgroundLinking -topics src/main/resources/topics-and-qrels/topics.backgroundlinking19.txt \
- -output runs/run.backgroundlinking19.bm25+rm3+df.topics.backgroundlinking19.txt \
+ -output runs/run.wapo.v2.bm25+rm3+df.topics.backgroundlinking19.txt \
  -backgroundlinking -backgroundlinking.datefilter -backgroundlinking.k 100 -bm25 -rm3 -hits 100 &
 ```
 
 Evaluation can be performed using `trec_eval`:
 
 ```
-tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking19.txt runs/run.backgroundlinking19.bm25.topics.backgroundlinking19.txt
+tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking19.txt runs/run.wapo.v2.bm25.topics.backgroundlinking19.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking19.txt runs/run.backgroundlinking19.bm25+rm3.topics.backgroundlinking19.txt
+tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking19.txt runs/run.wapo.v2.bm25+rm3.topics.backgroundlinking19.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking19.txt runs/run.backgroundlinking19.bm25+rm3+df.topics.backgroundlinking19.txt
+tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking19.txt runs/run.wapo.v2.bm25+rm3+df.topics.backgroundlinking19.txt
 ```
 
 ## Effectiveness

diff --git a/docs/regressions-backgroundlinking20.md b/docs/regressions-backgroundlinking20.md
@@ -10,11 +10,11 @@ Typical indexing command:
 
 ```
 nohup sh target/appassembler/bin/IndexCollection -collection WashingtonPostCollection \
- -input /path/to/backgroundlinking20 \
- -index indexes/lucene-index.core18-v3.pos+docvectors+raw \
+ -input /path/to/wapo.v3 \
+ -index indexes/lucene-index.wapo.v3.pos+docvectors+raw \
  -generator WashingtonPostGenerator \
  -threads 1 -storePositions -storeDocvectors -storeRaw \
-  >& logs/log.backgroundlinking20 &
+  >& logs/log.wapo.v3 &
 ```
 
 The directory `/path/to/core18/` should be the root directory of the [TREC Washington Post Corpus *v3*](https://trec.nist.gov/data/wapost/), i.e., `ls /path/to/core18/`
@@ -32,30 +32,30 @@ Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/m
 After indexing has completed, you should be able to perform retrieval as follows:
 
 ```
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.core18-v3.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.wapo.v3.pos+docvectors+raw \
  -topicreader BackgroundLinking -topics src/main/resources/topics-and-qrels/topics.backgroundlinking20.txt \
- -output runs/run.backgroundlinking20.bm25.topics.backgroundlinking20.txt \
+ -output runs/run.wapo.v3.bm25.topics.backgroundlinking20.txt \
  -backgroundlinking -backgroundlinking.k 100 -bm25 -hits 100 &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.core18-v3.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.wapo.v3.pos+docvectors+raw \
  -topicreader BackgroundLinking -topics src/main/resources/topics-and-qrels/topics.backgroundlinking20.txt \
- -output runs/run.backgroundlinking20.bm25+rm3.topics.backgroundlinking20.txt \
+ -output runs/run.wapo.v3.bm25+rm3.topics.backgroundlinking20.txt \
  -backgroundlinking -backgroundlinking.k 100 -bm25 -rm3 -hits 100 &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.core18-v3.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.wapo.v3.pos+docvectors+raw \
  -topicreader BackgroundLinking -topics src/main/resources/topics-and-qrels/topics.backgroundlinking20.txt \
- -output runs/run.backgroundlinking20.bm25+rm3+df.topics.backgroundlinking20.txt \
+ -output runs/run.wapo.v3.bm25+rm3+df.topics.backgroundlinking20.txt \
  -backgroundlinking -backgroundlinking.datefilter -backgroundlinking.k 100 -bm25 -rm3 -hits 100 &
 ```
 
 Evaluation can be performed using `trec_eval`:
 
 ```
-tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking20.txt runs/run.backgroundlinking20.bm25.topics.backgroundlinking20.txt
+tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking20.txt runs/run.wapo.v3.bm25.topics.backgroundlinking20.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking20.txt runs/run.backgroundlinking20.bm25+rm3.topics.backgroundlinking20.txt
+tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking20.txt runs/run.wapo.v3.bm25+rm3.topics.backgroundlinking20.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking20.txt runs/run.backgroundlinking20.bm25+rm3+df.topics.backgroundlinking20.txt
+tools/eval/trec_eval.9.0.4/trec_eval -c -M1000 -m ndcg_cut.5 -c -M1000 -m map src/main/resources/topics-and-qrels/qrels.backgroundlinking20.txt runs/run.wapo.v3.bm25+rm3+df.topics.backgroundlinking20.txt
 ```
 
 ## Effectiveness

diff --git a/docs/regressions-car17v1.5.md b/docs/regressions-car17v1.5.md
@@ -10,11 +10,11 @@ Typical indexing command:
 
 ```
 nohup sh target/appassembler/bin/IndexCollection -collection CarCollection \
- -input /path/to/car17v1.5 \
- -index indexes/lucene-index.car17v1.5.pos+docvectors+raw \
+ -input /path/to/car-paragraphCorpus.v1.5 \
+ -index indexes/lucene-index.car-paragraphCorpus.v1.5.pos+docvectors+raw \
  -generator DefaultLuceneDocumentGenerator \
  -threads 1 -storePositions -storeDocvectors -storeRaw \
-  >& logs/log.car17v1.5 &
+  >& logs/log.car-paragraphCorpus.v1.5 &
 ```
 
 The directory `/path/to/car17v1.5` should be the root directory of Complex Answer Retrieval (CAR) paragraph corpus (v1.5), which can be downloaded [here](http://trec-car.cs.unh.edu/datareleases/).
@@ -33,51 +33,51 @@ Specifically, this is the section-level passage retrieval task with automatic gr
 After indexing has completed, you should be able to perform retrieval as follows:
 
 ```
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car17v1.5.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car-paragraphCorpus.v1.5.pos+docvectors+raw \
  -topicreader Car -topics src/main/resources/topics-and-qrels/topics.car17v1.5.benchmarkY1test.txt \
- -output runs/run.car17v1.5.bm25.topics.car17v1.5.benchmarkY1test.txt \
+ -output runs/run.car-paragraphCorpus.v1.5.bm25.topics.car17v1.5.benchmarkY1test.txt \
  -bm25 &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car17v1.5.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car-paragraphCorpus.v1.5.pos+docvectors+raw \
  -topicreader Car -topics src/main/resources/topics-and-qrels/topics.car17v1.5.benchmarkY1test.txt \
- -output runs/run.car17v1.5.bm25+rm3.topics.car17v1.5.benchmarkY1test.txt \
+ -output runs/run.car-paragraphCorpus.v1.5.bm25+rm3.topics.car17v1.5.benchmarkY1test.txt \
  -bm25 -rm3 &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car17v1.5.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car-paragraphCorpus.v1.5.pos+docvectors+raw \
  -topicreader Car -topics src/main/resources/topics-and-qrels/topics.car17v1.5.benchmarkY1test.txt \
- -output runs/run.car17v1.5.bm25+ax.topics.car17v1.5.benchmarkY1test.txt \
+ -output runs/run.car-paragraphCorpus.v1.5.bm25+ax.topics.car17v1.5.benchmarkY1test.txt \
  -bm25 -axiom -axiom.deterministic -rerankCutoff 20 &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car17v1.5.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car-paragraphCorpus.v1.5.pos+docvectors+raw \
  -topicreader Car -topics src/main/resources/topics-and-qrels/topics.car17v1.5.benchmarkY1test.txt \
- -output runs/run.car17v1.5.ql.topics.car17v1.5.benchmarkY1test.txt \
+ -output runs/run.car-paragraphCorpus.v1.5.ql.topics.car17v1.5.benchmarkY1test.txt \
  -qld &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car17v1.5.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car-paragraphCorpus.v1.5.pos+docvectors+raw \
  -topicreader Car -topics src/main/resources/topics-and-qrels/topics.car17v1.5.benchmarkY1test.txt \
- -output runs/run.car17v1.5.ql+rm3.topics.car17v1.5.benchmarkY1test.txt \
+ -output runs/run.car-paragraphCorpus.v1.5.ql+rm3.topics.car17v1.5.benchmarkY1test.txt \
  -qld -rm3 &
 
-nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car17v1.5.pos+docvectors+raw \
+nohup target/appassembler/bin/SearchCollection -index indexes/lucene-index.car-paragraphCorpus.v1.5.pos+docvectors+raw \
  -topicreader Car -topics src/main/resources/topics-and-qrels/topics.car17v1.5.benchmarkY1test.txt \
- -output runs/run.car17v1.5.ql+ax.topics.car17v1.5.benchmarkY1test.txt \
+ -output runs/run.car-paragraphCorpus.v1.5.ql+ax.topics.car17v1.5.benchmarkY1test.txt \
  -qld -axiom -axiom.deterministic -rerankCutoff 20 &
 ```
 
 Evaluation can be performed using `trec_eval`:
 
 ```
-tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car17v1.5.bm25.topics.car17v1.5.benchmarkY1test.txt
+tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car-paragraphCorpus.v1.5.bm25.topics.car17v1.5.benchmarkY1test.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car17v1.5.bm25+rm3.topics.car17v1.5.benchmarkY1test.txt
+tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car-paragraphCorpus.v1.5.bm25+rm3.topics.car17v1.5.benchmarkY1test.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car17v1.5.bm25+ax.topics.car17v1.5.benchmarkY1test.txt
+tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car-paragraphCorpus.v1.5.bm25+ax.topics.car17v1.5.benchmarkY1test.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car17v1.5.ql.topics.car17v1.5.benchmarkY1test.txt
+tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car-paragraphCorpus.v1.5.ql.topics.car17v1.5.benchmarkY1test.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car17v1.5.ql+rm3.topics.car17v1.5.benchmarkY1test.txt
+tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car-paragraphCorpus.v1.5.ql+rm3.topics.car17v1.5.benchmarkY1test.txt
 
-tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car17v1.5.ql+ax.topics.car17v1.5.benchmarkY1test.txt
+tools/eval/trec_eval.9.0.4/trec_eval -m map -m recip_rank src/main/resources/topics-and-qrels/qrels.car17v1.5.benchmarkY1test.txt runs/run.car-paragraphCorpus.v1.5.ql+ax.topics.car17v1.5.benchmarkY1test.txt
 ```
 
 ## Effectiveness