Fix flaky mrtydi test on M1 (#1495)

castorini · Apr 11, 2023 · 1aea2b0 · 1aea2b0
1 parent cfd983b
commit 1aea2b0
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/scripts/repro_matrix/run_all_mrtydi.py b/scripts/repro_matrix/run_all_mrtydi.py
@@ -87,6 +87,10 @@ def print_results(metric, split):
                             elif name == 'mdpr-tied-pft-msmarco-ft-all.ko' and split == 'train' \
                                     and math.isclose(score, float(expected[metric]), abs_tol=4e-4):
                                 result_str = okish_str
+                            # Flaky test: small difference on Mac Studio (M1)
+                            elif name == 'mdpr-tied-pft-msmarco.th' and split == 'train' \
+                                    and math.isclose(score, float(expected[metric]), abs_tol=3e-4):
+                                result_str = okish_str
                             else:
                                 result_str = fail_str + f' expected {expected[metric]:.4f}'
                             print(f'      {metric:7}: {score:.4f} {result_str}')