From 83537a32814b20fe7fe6e41e68d61ffea4b1fc5f Mon Sep 17 00:00:00 2001
From: Jimmy Lin <jimmylin@uwaterloo.ca>
Date: Mon, 16 Sep 2024 20:35:41 -0400
Subject: [PATCH] Switch from using jtrec_eval to trec_eval (#1986)

+ trec_eval is already bundled with Anserini.
+ clean up trec_eval output to not be so noisy.
---
 integrations/clprf/test_clprf.py              | 132 +++++-------------
 .../sparse/test_lucenesearcher_check_irst.py  |  48 ++-----
 integrations/utils.py                         |   3 -
 pyserini/eval/trec_eval.py                    |  51 ++++---
 pyserini/pyclass.py                           |  10 +-
 .../simple_trec_run_unjudged_keep.txt         |   1 -
 .../simple_trec_run_unjudged_remove.txt       |   1 -
 tests/test_trectools.py                       |   2 +-
 8 files changed, 87 insertions(+), 161 deletions(-)

diff --git a/integrations/clprf/test_clprf.py b/integrations/clprf/test_clprf.py
index 83c907b70..8c4595f59 100644
--- a/integrations/clprf/test_clprf.py
+++ b/integrations/clprf/test_clprf.py
@@ -97,9 +97,7 @@ def test_cross_validation(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2462, delta=0.0001)
 
     def test_core17(self):
@@ -127,9 +125,7 @@ def test_core17_lr(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2473, delta=0.0001)
 
     def test_core17_lr_rm3(self):
@@ -151,9 +147,7 @@ def test_core17_lr_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2926, delta=0.0001)
 
     def test_core17_svm(self):
@@ -175,9 +169,7 @@ def test_core17_svm(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2385, delta=0.0001)
 
     def test_core17_svm_rm3(self):
@@ -199,9 +191,7 @@ def test_core17_svm_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2956, delta=0.0001)
 
     def test_core17_avg(self):
@@ -223,9 +213,7 @@ def test_core17_avg(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2442, delta=0.0001)
 
     def test_core17_avg_rm3(self):
@@ -247,9 +235,7 @@ def test_core17_avg_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2950, delta=0.0001)
 
     def test_core17_rrf(self):
@@ -284,9 +270,7 @@ def test_core17_rrf(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2445, delta=0.0001)
 
     def test_core17_rrf_rm3(self):
@@ -321,9 +305,7 @@ def test_core17_rrf_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2957, delta=0.0001)
 
     def test_core18(self):
@@ -351,9 +333,7 @@ def test_core18_lr(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2839, delta=0.0001)
 
     def test_core18_lr_rm3(self):
@@ -375,9 +355,7 @@ def test_core18_lr_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.3216, delta=0.0001)
 
     def test_core18_svm(self):
@@ -399,9 +377,7 @@ def test_core18_svm(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2841, delta=0.0001)
 
     def test_core18_svm_rm3(self):
@@ -423,9 +399,7 @@ def test_core18_svm_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.3200, delta=0.0001)
 
     def test_core18_avg(self):
@@ -447,9 +421,7 @@ def test_core18_avg(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2861, delta=0.0001)
 
     def test_core18_avg_rm3(self):
@@ -471,9 +443,7 @@ def test_core18_avg_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.3213, delta=0.0001)
 
     def test_core18_rrf(self):
@@ -508,9 +478,7 @@ def test_core18_rrf(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2880, delta=0.0001)
 
     def test_core18_rrf_rm3(self):
@@ -545,9 +513,7 @@ def test_core18_rrf_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.3205, delta=0.0001)
 
     def test_robust04(self):
@@ -575,9 +541,7 @@ def test_robust04_lr(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2747, delta=0.0001)
 
     def test_robust04_lr_rm3(self):
@@ -599,9 +563,7 @@ def test_robust04_lr_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2969, delta=0.0001)
 
     def test_robust04_svm(self):
@@ -623,9 +585,7 @@ def test_robust04_svm(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2726, delta=0.0001)
 
     def test_robust04_svm_rm3(self):
@@ -647,9 +607,7 @@ def test_robust04_svm_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2972, delta=0.0001)
 
     def test_robust04_avg(self):
@@ -671,9 +629,7 @@ def test_robust04_avg(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.276, delta=0.0001)
 
     def test_robust04_avg_rm3(self):
@@ -695,9 +651,7 @@ def test_robust04_avg_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2980, delta=0.0001)
 
     def test_robust04_rrf(self):
@@ -732,9 +686,7 @@ def test_robust04_rrf(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.275, delta=0.0001)
 
     def test_robust04_rrf_rm3(self):
@@ -769,9 +721,7 @@ def test_robust04_rrf_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2977, delta=0.0001)
 
     def test_robust05(self):
@@ -799,9 +749,7 @@ def test_robust05_lr(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2477, delta=0.0001)
 
     def test_robust05_lr_rm3(self):
@@ -823,9 +771,7 @@ def test_robust05_lr_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2872, delta=0.0001)
 
     def test_robust05_svm(self):
@@ -847,9 +793,7 @@ def test_robust05_svm(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2486, delta=0.0001)
 
     def test_robust05_svm_rm3(self):
@@ -871,9 +815,7 @@ def test_robust05_svm_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2871, delta=0.0001)
 
     def test_robust05_avg(self):
@@ -895,9 +837,7 @@ def test_robust05_avg(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2485, delta=0.0001)
 
     def test_robust05_avg_rm3(self):
@@ -919,9 +859,7 @@ def test_robust05_avg_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2880, delta=0.0001)
 
     def test_robust05_rrf(self):
@@ -956,9 +894,7 @@ def test_robust05_rrf(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2401, delta=0.0001)
 
     def test_robust05_rrf_rm3(self):
@@ -993,9 +929,7 @@ def test_robust05_rrf_rm3(self):
         score = parse_score(stdout, 'map')
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertAlmostEqual(score, 0.2808, delta=0.0001)
 
     def tearDown(self):
diff --git a/integrations/sparse/test_lucenesearcher_check_irst.py b/integrations/sparse/test_lucenesearcher_check_irst.py
index e51eadb48..cd2d9bdd0 100644
--- a/integrations/sparse/test_lucenesearcher_check_irst.py
+++ b/integrations/sparse/test_lucenesearcher_check_irst.py
@@ -54,9 +54,7 @@ def test_sum_aggregation_dl19_passage(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.3281)
         self.assertEqual(ndcg_score, 0.5260)
 
@@ -78,9 +76,7 @@ def test_sum_aggregation_dl20_passage(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.3520)
         self.assertEqual(ndcg_score, 0.5578)
     
@@ -103,9 +99,7 @@ def test_max_aggregation_dl19(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.3286)
         self.assertEqual(ndcg_score, 0.5371)
 
@@ -128,9 +122,7 @@ def test_max_aggregation_dl20_passage(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.3357)
         self.assertEqual(ndcg_score, 0.5469)
 
@@ -170,9 +162,7 @@ def test_sum_aggregation_dl19_doc(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.2524)
         self.assertEqual(ndcg_score, 0.5494)
 
@@ -194,9 +184,7 @@ def test_sum_aggregation_dl20_doc(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.3825)
         self.assertEqual(ndcg_score, 0.5559)
 
@@ -219,9 +207,7 @@ def test_max_aggregation_dl19_doc(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.2205)
         self.assertEqual(ndcg_score, 0.4917)
 
@@ -244,9 +230,7 @@ def test_max_aggregation_dl20_doc(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.3373)
         self.assertEqual(ndcg_score, 0.5015)
 
@@ -287,9 +271,7 @@ def test_sum_aggregation_dl19_doc_seg(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.2711)
         self.assertEqual(ndcg_score, 0.5596)
 
@@ -312,9 +294,7 @@ def test_sum_aggregation_dl20_doc_seg(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.3759)
         self.assertEqual(ndcg_score, 0.5343)
 
@@ -338,9 +318,7 @@ def test_max_aggregation_dl19_doc_seg(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.2425)
         self.assertEqual(ndcg_score, 0.5193)
 
@@ -364,9 +342,7 @@ def test_max_aggregation_dl20_doc_seg(self):
         ndcg_score = parse_score(stdout, "ndcg")
 
         self.assertEqual(status, 0)
-        # Currently, we get 'WARNING: Using incubator modules: jdk.incubator.vector\n' from stderr,
-        # so turn off check until this issue is resolved in a later JDK version.
-        # self.assertEqual(stderr, '')
+        self.assertEqual(stderr, '')
         self.assertEqual(map_score, 0.3496)
         self.assertEqual(ndcg_score, 0.5089)
 
diff --git a/integrations/utils.py b/integrations/utils.py
index e7eb6a3e8..de88ee7c0 100644
--- a/integrations/utils.py
+++ b/integrations/utils.py
@@ -45,9 +45,6 @@ def run_command(cmd, echo=False):
 def parse_score(output, metric, digits=4):
     """Function for parsing the output from `pyserini.eval.trec_eval`."""
     lines = output.split('\n')
-    # The output begins with a bunch of debug information, get rid of lines until we get to 'Results'
-    while 'Results' not in lines[0]:
-        lines.pop(0)
 
     for line in lines:
         if metric in line:
diff --git a/pyserini/eval/trec_eval.py b/pyserini/eval/trec_eval.py
index 97c265b58..4e11ab904 100644
--- a/pyserini/eval/trec_eval.py
+++ b/pyserini/eval/trec_eval.py
@@ -13,29 +13,44 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Example usage
-# python -m pyserini.eval.trec_eval -m ndcg_cut.10,20 -m all_trec qrels.dev.small.tsv runs/run.Colbert.txt -remove-unjudged -cutoffs.20,50
 
+# Example usage:
+# python -m pyserini.eval.trec_eval -c \
+#   -m ndcg_cut.10 \
+#   -m judged.5,10 beir-v1.0.0-arguana-test run.beir.contriever-msmarco.arguana.txt -remove-unjudged
 
+# From Jimmy, Sept 2024 -
+#
+# This file has a load sequence that is very different from all the other files.
+# The JVM by default in Pyserini is loaded with the option '--add-modules=jdk.incubator.vector', which triggers the
+# following warning: 'WARNING: Using incubator modules: jdk.incubator.vector'
+#
+# I have looked extensively online and was not able to find a way to suppress that warning.
+# The solution here is to start the JVM without the vector module, which isn't needed here.
+# This explains the code sequence below.
+
+import glob
+import importlib.resources
+import jnius_config
 import os
-import re
-import subprocess
-import sys
-import platform
 import pandas as pd
+import platform
 import tempfile
+import subprocess
+import sys
 
-from pyserini.search import get_qrels_file
-from pyserini.util import download_evaluation_script
+# Don't use the jdk.incubator.vector module.
+jar_directory = str(importlib.resources.files("pyserini.resources.jars").joinpath(''))
+jar_path = glob.glob(os.path.join(jar_directory, '*.jar'))[0]
+jnius_config.add_classpath(jar_path)
 
-script_path = download_evaluation_script('trec_eval')
+# This triggers loading of the JVM.
+from jnius import autoclass
 
-if platform.platform().startswith('macOS'):
-    # Hack around the fact that jtrec_eval hasn't been compiled for Mac M processors.
-    # Explicitly set os to x86, and then force the use of Rosetta.
-    cmd_prefix = ['java', '-Dos.arch=x86_64', '-jar', script_path]
-else:
-    cmd_prefix = ['java', '-jar', script_path]
+# Now we can load qrels
+from pyserini.search import get_qrels_file
+
+cmd_prefix = ['java', '-cp', jar_path, 'trec_eval']
 
 args = sys.argv
 
@@ -99,7 +114,10 @@
 else:
     cmd = cmd_prefix
 
-print(f'Running command: {cmd}')
+# We're going to shell out to call trec_eval.
+# Obvious question here: why we *not* just call the trec_eval main (Java) class, which already wraps the executable?
+# in Java (which wraps the binaries). The answer is that the Java class explicitly calls System.exit, so we wouldn't
+# be able to do cleanup here in Python.
 shell = platform.system() == "Windows"
 process = subprocess.Popen(cmd,
                            stdout=subprocess.PIPE,
@@ -109,7 +127,6 @@
 if stderr:
     print(stderr.decode("utf-8"))
 
-print('Results:')
 print(stdout.decode("utf-8").rstrip())
 
 for judged in judged_result:
diff --git a/pyserini/pyclass.py b/pyserini/pyclass.py
index 4b1c30c03..5f80aff4a 100644
--- a/pyserini/pyclass.py
+++ b/pyserini/pyclass.py
@@ -20,9 +20,13 @@
 
 from .setup import configure_classpath, os
 
-# If the environment variable isn't defined, look in the current directory.
-configure_classpath(os.environ['ANSERINI_CLASSPATH'] if 'ANSERINI_CLASSPATH' in os.environ else
-                    os.path.join(os.path.split(__file__)[0], 'resources/jars/'))
+try:
+    # If the environment variable isn't defined, look in the current directory.
+    configure_classpath(os.environ['ANSERINI_CLASSPATH'] if 'ANSERINI_CLASSPATH' in os.environ else
+                        os.path.join(os.path.split(__file__)[0], 'resources/jars/'))
+except:
+    # This might happen if the JVM's already been initialized. Just eat the error.
+    pass
 
 from jnius import autoclass, cast
 
diff --git a/tests/resources/simple_trec_run_unjudged_keep.txt b/tests/resources/simple_trec_run_unjudged_keep.txt
index edaf0a6d3..2c9038db1 100644
--- a/tests/resources/simple_trec_run_unjudged_keep.txt
+++ b/tests/resources/simple_trec_run_unjudged_keep.txt
@@ -1,4 +1,3 @@
-Results:
 ndcg_cut_5            	all	0.0848
 ndcg_cut_10           	all	0.0550
 judged_5              	all	0.5000
diff --git a/tests/resources/simple_trec_run_unjudged_remove.txt b/tests/resources/simple_trec_run_unjudged_remove.txt
index 48a44c190..747fc981c 100644
--- a/tests/resources/simple_trec_run_unjudged_remove.txt
+++ b/tests/resources/simple_trec_run_unjudged_remove.txt
@@ -1,4 +1,3 @@
-Results:
 ndcg_cut_5            	all	0.1131
 ndcg_cut_10           	all	0.0734
 judged_5              	all	1.0000
diff --git a/tests/test_trectools.py b/tests/test_trectools.py
index 019495508..4dbe5e9ea 100644
--- a/tests/test_trectools.py
+++ b/tests/test_trectools.py
@@ -79,7 +79,7 @@ def test_normalize_scores(self):
                                     self.output_path))
 
     # This and the next test case go together - to keep and to remove unjudged docs.
-    def test_undjudged_keep(self):
+    def test_unjudged_keep(self):
         qrels_path = os.path.join(self.root, 'tools/topics-and-qrels/qrels.covid-round1.txt')
         run_path = os.path.join(self.root, 'tests/resources/simple_trec_run_filter.txt')
         results = subprocess.check_output(