Refactor and prune dependencies (#1997)

Starts to untangles "core" dependencies (tests/, integrations/) from "optional" dependencies (tests-optional/, integrations-optional/). faiss is considered optional.
castorini · Oct 8, 2024 · 7ed8369 · 7ed8369
1 parent 67d07a0
commit 7ed8369
Show file tree

Hide file tree

Showing 116 changed files with 907 additions and 624 deletions.
diff --git a/integrations/dense/test_ance.py → integrations-optional/dense/test_ance.py b/integrations/dense/test_ance.py → integrations-optional/dense/test_ance.py
@@ -20,9 +20,9 @@
 import os
 import unittest
 
-from integrations.utils import clean_files, run_command, parse_score, parse_score_qa, parse_score_msmarco
-from pyserini.search import QueryEncoder
+from integrations.utils import clean_files, run_command, parse_score_qa, parse_score_msmarco
 from pyserini.search import get_topics
+from pyserini.search.faiss._searcher import QueryEncoder
 
 
 class TestAnce(unittest.TestCase):

diff --git a/integrations/dense/test_dpr.py → integrations-optional/dense/test_dpr.py b/integrations/dense/test_dpr.py → integrations-optional/dense/test_dpr.py
@@ -22,8 +22,8 @@
 import unittest
 
 from integrations.utils import clean_files, run_command, parse_score_qa
-from pyserini.search import QueryEncoder
 from pyserini.search import get_topics
+from pyserini.search.faiss._searcher import QueryEncoder
 
 
 class TestDpr(unittest.TestCase):

diff --git a/integrations/dense/test_encode.py → integrations-optional/dense/test_encode.py b/integrations/dense/test_encode.py → integrations-optional/dense/test_encode.py
diff --git a/integrations/dense/test_tct_colbert-v2.py → ...ons-optional/dense/test_tct_colbert-v2.py b/integrations/dense/test_tct_colbert-v2.py → ...ons-optional/dense/test_tct_colbert-v2.py
diff --git a/integrations/dense/test_tct_colbert.py → ...ations-optional/dense/test_tct_colbert.py b/integrations/dense/test_tct_colbert.py → ...ations-optional/dense/test_tct_colbert.py
@@ -21,8 +21,8 @@
 import unittest
 
 from integrations.utils import clean_files, run_command, parse_score
-from pyserini.search import QueryEncoder
 from pyserini.search import get_topics
+from pyserini.search.faiss._searcher import QueryEncoder
 
 
 class TestTctColBert(unittest.TestCase):

diff --git a/integrations/papers/test_ecir2023.py → ...grations-optional/papers/test_ecir2023.py b/integrations/papers/test_ecir2023.py → ...grations-optional/papers/test_ecir2023.py
diff --git a/integrations/papers/test_sigir2021.py → ...rations-optional/papers/test_sigir2021.py b/integrations/papers/test_sigir2021.py → ...rations-optional/papers/test_sigir2021.py
@@ -22,9 +22,9 @@
 from integrations.utils import clean_files, run_command, parse_score_msmarco
 from pyserini.dsearch import SimpleDenseSearcher, TctColBertQueryEncoder
 from pyserini.hsearch import HybridSearcher
-from pyserini.index import IndexReader
-from pyserini.search import SimpleSearcher
+from pyserini.index.lucene import LuceneIndexReader
 from pyserini.search import get_topics, get_qrels
+from pyserini.search._deprecated import SimpleSearcher
 
 
 class TestSIGIR2021(unittest.TestCase):
@@ -100,7 +100,7 @@ def test_figure5(self):
         """Sample code in Figure 5."""
 
         # Initialize from a pre-built index:
-        reader = IndexReader.from_prebuilt_index('robust04')
+        reader = LuceneIndexReader.from_prebuilt_index('robust04')
 
         terms = reader.terms()
         term = next(terms)

diff --git a/...enesearcher_check_ltr_msmarco_document.py → ...enesearcher_check_ltr_msmarco_document.py b/...enesearcher_check_ltr_msmarco_document.py → ...enesearcher_check_ltr_msmarco_document.py
diff --git a/...cenesearcher_check_ltr_msmarco_passage.py → ...cenesearcher_check_ltr_msmarco_passage.py b/...cenesearcher_check_ltr_msmarco_passage.py → ...cenesearcher_check_ltr_msmarco_passage.py
diff --git a/integrations/sparse/test_nmslib.py → integrations-optional/sparse/test_nmslib.py b/integrations/sparse/test_nmslib.py → integrations-optional/sparse/test_nmslib.py
diff --git a/integrations/clprf/test_clprf.py b/integrations/clprf/test_clprf.py
@@ -21,7 +21,7 @@
 
 from integrations.lucenesearcher_score_checker import LuceneSearcherScoreChecker
 from integrations.utils import run_command, parse_score
-from pyserini.search import LuceneSearcher
+from pyserini.search.lucene import LuceneSearcher
 
 
 class TestSearchIntegration(unittest.TestCase):

diff --git a/integrations/sparse/test_lucenesearcher_check_core17.py b/integrations/sparse/test_lucenesearcher_check_core17.py
@@ -17,7 +17,7 @@
 import unittest
 
 from integrations.lucenesearcher_anserini_checker import LuceneSearcherAnseriniMatchChecker
-from pyserini.search import LuceneSearcher
+from pyserini.search.lucene import LuceneSearcher
 
 
 class CheckSearchResultsAgainstAnseriniForCore17(unittest.TestCase):

diff --git a/integrations/sparse/test_lucenesearcher_check_core18.py b/integrations/sparse/test_lucenesearcher_check_core18.py
@@ -17,7 +17,7 @@
 import unittest
 
 from integrations.lucenesearcher_anserini_checker import LuceneSearcherAnseriniMatchChecker
-from pyserini.search import LuceneSearcher
+from pyserini.search.lucene import LuceneSearcher
 
 
 class CheckSearchResultsAgainstAnseriniForCore18(unittest.TestCase):

diff --git a/integrations/sparse/test_lucenesearcher_check_robust04.py b/integrations/sparse/test_lucenesearcher_check_robust04.py
@@ -17,7 +17,7 @@
 import unittest
 
 from integrations.lucenesearcher_anserini_checker import LuceneSearcherAnseriniMatchChecker
-from pyserini.search import LuceneSearcher
+from pyserini.search.lucene import LuceneSearcher
 
 
 class CheckSearchResultsAgainstAnseriniForRobust04(unittest.TestCase):

diff --git a/integrations/sparse/test_lucenesearcher_check_robust05.py b/integrations/sparse/test_lucenesearcher_check_robust05.py
@@ -17,7 +17,7 @@
 import unittest
 
 from integrations.lucenesearcher_anserini_checker import LuceneSearcherAnseriniMatchChecker
-from pyserini.search import LuceneSearcher
+from pyserini.search.lucene import LuceneSearcher
 
 
 class CheckSearchResultsAgainstAnseriniForRobust05(unittest.TestCase):

diff --git a/integrations/sparse/test_simple_fusion_search_integration.py b/integrations/sparse/test_simple_fusion_search_integration.py
@@ -24,7 +24,7 @@
 
 from pyserini.fusion import FusionMethod
 from pyserini.search import get_topics
-from pyserini.search import LuceneFusionSearcher
+from pyserini.search.lucene import LuceneFusionSearcher
 from pyserini.trectools import TrecRun
 from pyserini.util import download_url, download_and_unpack_index
 

diff --git a/pyserini/2cr/atomic.py b/pyserini/2cr/atomic.py
@@ -1,14 +1,31 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import argparse
+import importlib.resources
+import math
 import os
 import sys
+import time
 from collections import defaultdict
 from string import Template
-import importlib.resources
-import time
+
 import yaml
-import math
-from ._base import run_eval_and_return_metric, ok_str, fail_str
 
+from ._base import run_eval_and_return_metric, ok_str, fail_str
 
 atomic_models = [
     'ViT-L-14.laion2b_s32b_b82k',

diff --git a/pyserini/2cr/beir.py b/pyserini/2cr/beir.py
@@ -15,6 +15,7 @@
 #
 
 import argparse
+import importlib.resources
 import math
 import os
 import sys
@@ -23,7 +24,6 @@
 from datetime import datetime
 from string import Template
 
-import importlib.resources
 import yaml
 
 from ._base import run_eval_and_return_metric, ok_str, okish_str, fail_str

diff --git a/pyserini/2cr/ciral.py b/pyserini/2cr/ciral.py
@@ -15,18 +15,18 @@
 #
 
 import argparse
+import importlib.resources
 import math
 import os
 import sys
 import time
-import importlib.resources
 from collections import defaultdict, OrderedDict
 from datetime import datetime
 from string import Template
 
 import yaml
 
-from ._base import run_eval_and_return_metric, ok_str, okish_str, fail_str
+from ._base import run_eval_and_return_metric, ok_str, fail_str
 
 dense_threads = 16
 dense_batch_size = 512

diff --git a/pyserini/2cr/miracl.py b/pyserini/2cr/miracl.py
@@ -15,6 +15,7 @@
 #
 
 import argparse
+import importlib.resources
 import math
 import os
 import subprocess
@@ -24,7 +25,6 @@
 from datetime import datetime
 from string import Template
 
-import importlib.resources
 import yaml
 
 from ._base import run_eval_and_return_metric, ok_str, okish_str, fail_str

diff --git a/pyserini/2cr/mrtydi.py b/pyserini/2cr/mrtydi.py
@@ -15,6 +15,7 @@
 #
 
 import argparse
+import importlib.resources
 import math
 import os
 import sys
@@ -23,7 +24,6 @@
 from datetime import datetime
 from string import Template
 
-import importlib.resources
 import yaml
 
 from ._base import run_eval_and_return_metric, ok_str, okish_str, fail_str

diff --git a/pyserini/2cr/msmarco.py b/pyserini/2cr/msmarco.py
@@ -15,6 +15,7 @@
 #
 
 import argparse
+import importlib.resources
 import math
 import os
 import re
@@ -24,7 +25,6 @@
 from datetime import datetime
 from string import Template
 
-import importlib.resources
 import yaml
 
 from ._base import run_eval_and_return_metric, ok_str, okish_str, fail_str

diff --git a/pyserini/2cr/odqa.py b/pyserini/2cr/odqa.py
@@ -15,6 +15,7 @@
 #
 
 import argparse
+import importlib.resources
 import math
 import os
 import sys
@@ -23,7 +24,6 @@
 from datetime import datetime
 from string import Template
 
-import importlib.resources
 import yaml
 
 from ._base import run_dpr_retrieval_eval_and_return_metric, convert_trec_run_to_dpr_retrieval_json, run_fusion, ok_str, \

diff --git a/pyserini/analysis/__init__.py b/pyserini/analysis/__init__.py
@@ -15,5 +15,3 @@
 #
 
 from ._base import get_lucene_analyzer, Analyzer, JAnalyzer, JAnalyzerUtils, JDefaultEnglishAnalyzer, JWhiteSpaceAnalyzer
-
-__all__ = ['get_lucene_analyzer', 'Analyzer', 'JAnalyzer', 'JAnalyzerUtils', 'JDefaultEnglishAnalyzer', 'JWhiteSpaceAnalyzer']
diff --git a/pyserini/analysis/_base.py b/pyserini/analysis/_base.py
@@ -16,7 +16,7 @@
 
 from typing import List
 
-from ..pyclass import autoclass
+from pyserini.pyclass import autoclass
 
 # Wrappers around Lucene classes
 JAnalyzer = autoclass('org.apache.lucene.analysis.Analyzer')

diff --git a/pyserini/collection/__init__.py b/pyserini/collection/__init__.py
@@ -16,5 +16,3 @@
 
 from ._base import Collection, FileSegment, SourceDocument
 from ._collection_support import Cord19Article
-
-__all__ = ['Collection', 'FileSegment', 'SourceDocument', 'Cord19Article']
diff --git a/pyserini/demo/acl.py b/pyserini/demo/acl.py
@@ -23,14 +23,16 @@
     --port [PORT] --hits [Number of hits]
     --k1 [BM25 k1] --b [BM25 b] --device [cpu, cuda]
 """
-import json
+
 import logging
 from argparse import ArgumentParser
 from functools import partial
 from typing import Callable, Optional, Tuple, Union
 
-from flask import Flask, render_template, request, flash, jsonify
-from pyserini.search import LuceneSearcher, FaissSearcher, AutoQueryEncoder
+from flask import Flask, render_template, request, flash
+
+from pyserini.search.faiss import FaissSearcher
+from pyserini.search.lucene import LuceneSearcher
 
 logging.basicConfig(
     format='%(asctime)s | %(levelname)s | %(name)s | %(message)s',

diff --git a/pyserini/demo/atomic.py b/pyserini/demo/atomic.py
@@ -23,14 +23,16 @@
     --port [PORT] --hits [Number of hits] --index [BM25 or {dense retrieval flag}]
     --k1 [BM25 k1] --b [BM25 b] --device [cpu, cuda]
 """
+
 import json
 from argparse import ArgumentParser
 from functools import partial
 from typing import Callable, Optional, Tuple, Union
 
 from flask import Flask, render_template, request, flash, jsonify
-from pyserini.search import LuceneSearcher, FaissSearcher
 
+from pyserini.search.faiss import FaissSearcher
+from pyserini.search.lucene import LuceneSearcher
 
 INDEX_NAMES = (
     'atomic_image_v0.2_small_validation',

diff --git a/pyserini/demo/dpr.py b/pyserini/demo/dpr.py
@@ -18,15 +18,15 @@
 import json
 import random
 
-from pyserini.search.lucene import LuceneSearcher
+from pyserini.search import get_topics
 from pyserini.search.faiss import FaissSearcher, DprQueryEncoder
 from pyserini.search.hybrid import HybridSearcher
-from pyserini import search
+from pyserini.search.lucene import LuceneSearcher
 
 
 class DPRDemo(cmd.Cmd):
-    nq_dev_topics = list(search.get_topics('dpr-nq-dev').values())
-    trivia_dev_topics = list(search.get_topics('dpr-trivia-dev').values())
+    nq_dev_topics = list(get_topics('dpr-nq-dev').values())
+    trivia_dev_topics = list(get_topics('dpr-trivia-dev').values())
 
     ssearcher = LuceneSearcher.from_prebuilt_index('wikipedia-dpr')
     searcher = ssearcher

diff --git a/pyserini/demo/miracl.py b/pyserini/demo/miracl.py
@@ -23,14 +23,18 @@
     --port [PORT] --hits [Number of hits] --index [BM25 or mdpr-tied-pft-msmarco]
     --k1 [BM25 k1] --b [BM25 b] --device [cpu, cuda]
 """
+
 import json
 import logging
 from argparse import ArgumentParser
 from functools import partial
 from typing import Callable, Optional, Tuple, Union
 
 from flask import Flask, render_template, request, flash, jsonify
-from pyserini.search import LuceneSearcher, FaissSearcher, AutoQueryEncoder
+
+from pyserini.encode import AutoQueryEncoder
+from pyserini.search.faiss import FaissSearcher
+from pyserini.search.lucene import LuceneSearcher
 
 logging.basicConfig(
     format='%(asctime)s | %(levelname)s | %(name)s | %(message)s',

diff --git a/pyserini/demo/msmarco.py b/pyserini/demo/msmarco.py
@@ -16,17 +16,16 @@
 
 import cmd
 import json
-import os
 import random
 
-from pyserini.search.lucene import LuceneSearcher
+from pyserini.search import get_topics
 from pyserini.search.faiss import FaissSearcher, TctColBertQueryEncoder, AnceQueryEncoder
 from pyserini.search.hybrid import HybridSearcher
-from pyserini import search
+from pyserini.search.lucene import LuceneSearcher
 
 
 class MsMarcoDemo(cmd.Cmd):
-    dev_topics = list(search.get_topics('msmarco-passage-dev-subset').values())
+    dev_topics = list(get_topics('msmarco-passage-dev-subset').values())
 
     ssearcher = LuceneSearcher.from_prebuilt_index('msmarco-passage')
     dsearcher = None

diff --git a/pyserini/dsearch.py b/pyserini/dsearch.py
@@ -20,20 +20,20 @@
 import os
 import sys
 
-import pyserini.search.faiss
-from pyserini.search.faiss import TctColBertQueryEncoder
+from pyserini.search.faiss import FaissSearcher
+from pyserini.search.faiss._searcher import TctColBertQueryEncoder, BinaryDenseSearcher
 
 __all__ = ['SimpleDenseSearcher', 'BinaryDenseSearcher', 'TctColBertQueryEncoder']
 
 
-class SimpleDenseSearcher(pyserini.search.faiss.FaissSearcher):
+class SimpleDenseSearcher(FaissSearcher):
     def __new__(cls, *args, **kwargs):
         print('pyserini.dsearch.SimpleDenseSearcher class has been deprecated, '
               'please use FaissSearcher from pyserini.search.faiss instead')
         return super().__new__(cls)
 
 
-class BinaryDenseSearcher(pyserini.search.faiss.BinaryDenseSearcher):
+class BinaryDenseSearcher(BinaryDenseSearcher):
     def __new__(cls, *args, **kwargs):
         print('pyserini.dsearch.BinaryDenseSearcher class has been deprecated, '
               'please use BinaryDenseSearcher from pyserini.search.faiss instead')
Original file line number	Diff line number	Diff line change
Expand Up		@@ -15,5 +15,3 @@
		#

		from ._base import get_lucene_analyzer, Analyzer, JAnalyzer, JAnalyzerUtils, JDefaultEnglishAnalyzer, JWhiteSpaceAnalyzer

		__all__ = ['get_lucene_analyzer', 'Analyzer', 'JAnalyzer', 'JAnalyzerUtils', 'JDefaultEnglishAnalyzer', 'JWhiteSpaceAnalyzer']
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,5 +16,3 @@

		from ._base import Collection, FileSegment, SourceDocument
		from ._collection_support import Cord19Article

		__all__ = ['Collection', 'FileSegment', 'SourceDocument', 'Cord19Article']