Skip to content

Commit

Permalink
Fixed metrics for ClueWeb plots in ECIR 2019 axiomatic runbook (casto…
Browse files Browse the repository at this point in the history 
…rini#554)
  • Loading branch information
@Peilin-Yang @lintool
Peilin-Yang authored and lintool committed Jan 15, 2019
1 parent 7a2983b commit e47faca
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 13 deletions.
8 changes: 4 additions & 4 deletions docs/runbook-ecir2019-axiomatic.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ python src/main/python/ecir2019_axiomatic/run_batch.py --collection robust05 --m
python src/main/python/ecir2019_axiomatic/run_batch.py --collection core17 --models bm25 ql f2exp --n 32 --run --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection wt10g --models bm25 ql f2exp --n 32 --run --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection gov2 --models bm25 ql f2exp --n 32 --run --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection cw09b --models bm25 ql f2exp --n 32 --run --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection cw12b13 --models bm25 ql f2exp --n 32 --run --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection cw09b --models bm25 ql f2exp --n 32 --metrics ndcg20 --run --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection cw12b13 --models bm25 ql f2exp --n 32 --metrics ndcg20 --run --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection mb11 --models bm25 ql f2exp --n 32 --run --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection mb13 --models bm25 ql f2exp --n 32 --run --plot
```
Expand Down Expand Up @@ -63,8 +63,8 @@ python src/main/python/ecir2019_axiomatic/run_batch.py --collection robust05 --m
python src/main/python/ecir2019_axiomatic/run_batch.py --collection core17 --models bm25 --n 32 --run --random --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection wt10g --models bm25 --n 32 --run --random --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection gov2 --models bm25 --n 32 --run --random --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection cw09b --models bm25 --n 32 --run --random --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection cw12b13 --models bm25 --n 32 --run --random --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection cw09b --models bm25 --n 32 --metrics ndcg20 --run --random --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection cw12b13 --models bm25 --n 32 --metrics ndcg20 --run --random --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection mb11 --models bm25 --n 32 --run --random --plot
python src/main/python/ecir2019_axiomatic/run_batch.py --collection mb13 --models bm25 --n 32 --run --random --plot
```
Expand Down
14 changes: 10 additions & 4 deletions src/main/python/ecir2019_axiomatic/plots.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,13 +70,16 @@ def read_data(self, fn):
all_results[model].append((float(para), float(score)))
return all_results

def plot_params_sensitivity(self, collection, output_root):
def plot_params_sensitivity(self, collection, output_root, metrics):
if not os.path.exists(os.path.join(output_root, self.plots_root)):
os.makedirs(os.path.join(output_root, self.plots_root))

for fn in os.listdir(os.path.join(output_root, self.effectiveness_root)):
if not fn.startswith('axiom_paras_sensitivity_'):
continue
metric = os.path.splitext(fn)[0].split('_')[-1]
if metric not in metrics:
continue
all_results = self.read_data(os.path.join(output_root, self.effectiveness_root, fn))
ls = ['-', '--', ':']
colors = ['r', 'g', 'b']
Expand All @@ -94,7 +97,7 @@ def plot_params_sensitivity(self, collection, output_root):
ax.set_xlabel(r'$\beta$')
ax.set_ylabel('AP' if not 'cw' in collection else 'NDCG@20')
ax.legend(loc=4)
output_fn = os.path.join(output_root, self.plots_root, 'params_sensitivity_{}.eps'.format(collection))
output_fn = os.path.join(output_root, self.plots_root, 'params_sensitivity_{}_{}.eps'.format(collection, metric))
plt.savefig(output_fn, bbox_inches='tight', format='eps')

def read_coverage_data(self, fn):
Expand Down Expand Up @@ -158,13 +161,16 @@ def plot_per_topic_analysis(self, collection, output_root):
output_fn = os.path.join(output_root, self.plots_root, 'per_query_{}_{}.eps'.format(collection, beta))
plt.savefig(output_fn, bbox_inches='tight', format='eps')

def plot_random_seeds(self, collection, output_root, beta):
def plot_random_seeds(self, collection, output_root, beta, metrics):
if not os.path.exists(os.path.join(output_root, self.plots_root)):
os.makedirs(os.path.join(output_root, self.plots_root))

for fn in os.listdir(os.path.join(output_root, self.effectiveness_root)):
if not fn.startswith('axiom_random_seeds_'):
continue
metric = os.path.splitext(fn)[0].split('_')[-1]
if metric not in metrics:
continue
fig, ax = plt.subplots(1, 1, figsize=(3, 3))
random_seeds_results = self.read_data(os.path.join(output_root, self.effectiveness_root, fn))
paras_results = self.read_data(os.path.join(output_root, self.effectiveness_root, fn.replace('axiom_random_seeds_', 'axiom_paras_sensitivity_')))
Expand All @@ -181,5 +187,5 @@ def plot_random_seeds(self, collection, output_root, beta):
ax.boxplot(y, showfliers=True)
ax.set_xticks([])
#ax.legend(loc=0) # Legend is muted -- the explanations are in the text (caption)
output_fn = os.path.join(output_root, self.plots_root, 'random_seeds_{}.eps'.format(collection))
output_fn = os.path.join(output_root, self.plots_root, 'random_seeds_{}_{}.eps'.format(collection, metric))
plt.savefig(output_fn, bbox_inches='tight', format='eps')
10 changes: 5 additions & 5 deletions src/main/python/ecir2019_axiomatic/run_batch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ def batch_output_effectiveness(collection_yaml, models_yaml, output_root, random
logger.info('='*10+'Starting Output Effectiveness'+'='*10)
Effectiveness(index_path).output_effectiveness(this_output_root, random)

def plot(collection_yaml, output_root, random = False):
def plot(collection_yaml, output_root, metrics, random = False):
this_output_root = os.path.join(output_root, collection_yaml['name'])
if random:
if collection_yaml['name'] == 'disk12' or collection_yaml['name'] == 'robust04' or collection_yaml['name'] == 'robust05' or collection_yaml['name'] == 'core17':
Expand All @@ -134,9 +134,9 @@ def plot(collection_yaml, output_root, random = False):
beta = 1.0
else: # Web collections
beta = 0.1
Plots().plot_random_seeds(collection_yaml['name'], this_output_root, beta)
Plots().plot_random_seeds(collection_yaml['name'], this_output_root, beta, metrics)
else:
Plots().plot_params_sensitivity(collection_yaml['name'], this_output_root)
Plots().plot_params_sensitivity(collection_yaml['name'], this_output_root, metrics)

def cal_coverage(collection_yaml, model_yaml, output_root):
index_path = get_index_path(collection_yaml)
Expand Down Expand Up @@ -258,9 +258,9 @@ def concatenate_qrels():
if args.plot:
from plots import Plots
if args.cal_coverage:
plot_coverage(collection_yaml, args.output_root)
plot_coverage(collection_yaml, args.output_root, args.metrics)
else:
plot(collection_yaml, args.output_root, args.random)
plot(collection_yaml, args.output_root, args.metrics, args.random)

if args.per_topic_analysis:
from plots import Plots
Expand Down

0 comments on commit e47faca

Please sign in to comment.