Rename the dir name and add the unittest dir (#48)

Signed-off-by: SimFG <bang.fu@zilliz.com>
zilliztech · Apr 4, 2023 · aa1fe25 · aa1fe25
1 parent c67178f
commit aa1fe25
Show file tree

Hide file tree

Showing 40 changed files with 64 additions and 62 deletions.
diff --git a/.gitignore b/.gitignore
@@ -132,5 +132,7 @@ dmypy.json
 **/data_map**.txt
 **/faiss**.index
 **/sqlite**.db
+**/gpt_cache**.db
 **/example.py
 **/example.db
+**/.chroma
diff --git a/docs/contributing.md b/docs/contributing.md
@@ -22,7 +22,7 @@ First check which part you want to contribute:
 # The __init__.py file of the same directory under the new file
 __all__ = ['Milvus']
 
-from gptcache.util.lazy_import import LazyImport
+from gptcache.utils.lazy_import import LazyImport
 
 milvus = LazyImport('milvus', globals(), 'gptcache.cache.vector_data.milvus')
 
@@ -89,16 +89,16 @@ refer to the implementation of [MapDataManager, SSDataManager or SIDataManager](
 
 ## Add a embedding function
 
-refer to the implementation of [towhee](../gptcache/encoder/towhee.py) or [openai](../gptcache/encoder/openai.py).
+refer to the implementation of [towhee](../gptcache/embedding/towhee.py) or [openai](../gptcache/embedding/openai.py).
 
-1. Add a new python file to [embedding](../gptcache/encoder) directory
+1. Add a new python file to [embedding](../gptcache/embedding) directory
 2. Make sure the newly added third-party libraries are lazy imported and automatic installation
 3. Implement the embedding function and **make sure** your output dimension
 4. Add a usage example to [example](../examples) directory and add the corresponding content to [example.md](../examples/example.md) [README.md](../README.md)
 
 ## Add a similarity evaluation function
 
-refer to the implementation of [pair_evaluation](../gptcache/ranker/simple.py) or [towhee](../gptcache/ranker/towhee.py)
+refer to the implementation of [pair_evaluation](../gptcache/similarity_evaluation/simple.py) or [towhee](../gptcache/similarity_evaluation/towhee.py)
 
 1. Make sure the input params, you can learn more about in the [user view](../gptcache/adapter/openai.py) model
 ```python

diff --git a/examples/benchmark/benchmark_sqlite_faiss_towhee.py b/examples/benchmark/benchmark_sqlite_faiss_towhee.py
@@ -5,9 +5,9 @@
 from gptcache.adapter import openai
 from gptcache.core import cache, Config
 from gptcache.cache.factory import get_ss_data_manager
-from gptcache.ranker import Towhee as EvaluationTowhee
-from gptcache.encoder import Towhee as EmbeddingTowhee
-from gptcache.ranker.simple import SearchDistanceEvaluation
+from gptcache.similarity_evaluation import Towhee as EvaluationTowhee
+from gptcache.embedding import Towhee as EmbeddingTowhee
+from gptcache.similarity_evaluation.simple import SearchDistanceEvaluation
 
 
 def run():
@@ -42,7 +42,7 @@ def evaluation(self, src_dict, cache_dict, **kwargs):
         def range(self):
             return 0.0, 1.0
 
-    sqlite_file = "sqlite.db"
+    sqlite_file = "gptcache.db"
     faiss_file = "faiss.index"
     has_data = os.path.isfile(sqlite_file) and os.path.isfile(faiss_file)
 

diff --git a/examples/langchian_llms_mock/langchian_llms_mock.py b/examples/langchian_llms_mock/langchian_llms_mock.py
@@ -4,7 +4,7 @@
 from langchain.llms import OpenAI
 from gptcache.core import cache, Config
 from gptcache.cache.factory import get_ss_data_manager
-from gptcache.ranker.simple import pair_evaluation
+from gptcache.similarity_evaluation.simple import pair_evaluation
 from gptcache.processor.post import nop as postnop
 from gptcache.processor.pre import nop as prenop
 import numpy as np
@@ -18,7 +18,7 @@ def mock_embeddings(data, **kwargs):
 
 
 def run():
-    sqlite_file = "sqlite.db"
+    sqlite_file = "gptcache.db"
     faiss_file = "faiss.index"
     has_data = os.path.isfile(sqlite_file) and os.path.isfile(faiss_file)
     data_manager = get_ss_data_manager("sqlite", "faiss",

diff --git a/examples/openai_request/openai_request.py b/examples/openai_request/openai_request.py
@@ -3,8 +3,8 @@
 
 from gptcache.cache.factory import get_data_manager, get_ss_data_manager
 from gptcache.core import cache, Cache, Config
-from gptcache.encoder import Towhee
-from gptcache.ranker.simple import SearchDistanceEvaluation
+from gptcache.embedding import Towhee
+from gptcache.similarity_evaluation.simple import SearchDistanceEvaluation
 from gptcache.adapter import openai
 
 

diff --git a/examples/sqlite_chromadb_mock/sqlite_chromadb_mock.py b/examples/sqlite_chromadb_mock/sqlite_chromadb_mock.py
@@ -4,7 +4,7 @@
 from gptcache.adapter import openai
 from gptcache.core import cache, Config
 from gptcache.cache.factory import get_ss_data_manager
-from gptcache.ranker.simple import pair_evaluation
+from gptcache.similarity_evaluation.simple import SearchDistanceEvaluation
 import numpy as np
 
 
@@ -15,17 +15,16 @@ def mock_embeddings(data, **kwargs):
 
 
 def run():
-    sqlite_file = "sqlite.db"
+    sqlite_file = "gptcache.db"
     has_data = os.path.isfile(sqlite_file)
     # milvus
     data_manager = get_ss_data_manager("sqlite", "chromadb", max_size=8, clean_size=2, client_settings={})
 
     cache.init(embedding_func=mock_embeddings,
                data_manager=data_manager,
-               evaluation_func=pair_evaluation,
+               similarity_evaluation=SearchDistanceEvaluation(),
                config=Config(
-                       similarity_threshold=10000,
-                       similarity_positive=False,
+                       similarity_threshold=0,
                    ),
                )
 

diff --git a/examples/sqlite_faiss_mock/sqlite_faiss_mock.py b/examples/sqlite_faiss_mock/sqlite_faiss_mock.py
@@ -3,7 +3,7 @@
 from gptcache.adapter import openai
 from gptcache.core import cache, Config
 from gptcache.cache.factory import get_ss_data_manager
-from gptcache.ranker.simple import SearchDistanceEvaluation
+from gptcache.similarity_evaluation.simple import SearchDistanceEvaluation
 import numpy as np
 
 
@@ -15,7 +15,7 @@ def mock_embeddings(data, **kwargs):
 
 
 def run():
-    sqlite_file = "sqlite.db"
+    sqlite_file = "gptcache.db"
     faiss_file = "faiss.index"
     has_data = os.path.isfile(sqlite_file) and os.path.isfile(faiss_file)
     data_manager = get_ss_data_manager("sqlite", "faiss",

diff --git a/examples/sqlite_faiss_towhee/sqlite_faiss_towhee.py b/examples/sqlite_faiss_towhee/sqlite_faiss_towhee.py
@@ -4,16 +4,16 @@
 from gptcache.adapter import openai
 from gptcache.core import cache, Config
 from gptcache.cache.factory import get_ss_data_manager
-from gptcache.ranker.simple import SearchDistanceEvaluation
-from gptcache.encoder import Towhee
+from gptcache.similarity_evaluation.simple import SearchDistanceEvaluation
+from gptcache.embedding import Towhee
 
 
 def run():
     towhee = Towhee()
     # chinese model
     # towhee = Towhee(model="uer/albert-base-chinese-cluecorpussmall")
 
-    sqlite_file = "sqlite.db"
+    sqlite_file = "gptcache.db"
     faiss_file = "faiss.index"
     has_data = os.path.isfile(sqlite_file) and os.path.isfile(faiss_file)
     data_manager = get_ss_data_manager("sqlite", "faiss",

diff --git a/examples/sqlite_milvus_mock/sqlite_milvus_mock.py b/examples/sqlite_milvus_mock/sqlite_milvus_mock.py
@@ -3,7 +3,7 @@
 from gptcache.adapter import openai
 from gptcache.core import cache, Config
 from gptcache.cache.factory import get_ss_data_manager
-from gptcache.ranker.simple import SearchDistanceEvaluation
+from gptcache.similarity_evaluation.simple import SearchDistanceEvaluation
 import numpy as np
 
 
@@ -15,7 +15,7 @@ def mock_embeddings(data, **kwargs):
 
 
 def run():
-    sqlite_file = "sqlite.db"
+    sqlite_file = "gptcache.db"
     has_data = os.path.isfile(sqlite_file)
     # milvus
     data_manager = get_ss_data_manager("sqlite", "milvus", dimension=d, max_size=8, clean_size=2)

diff --git a/gptcache/adapter/adapter.py b/gptcache/adapter/adapter.py
@@ -1,7 +1,7 @@
 import logging
 import copy
 from ..core import cache, time_cal
-from ..util.error import NotInitError
+from ..utils.error import NotInitError
 
 
 def adapt(llm_handler, cache_data_convert, update_cache_callback, *args, **kwargs):

diff --git a/gptcache/cache/data_manager.py b/gptcache/cache/data_manager.py
@@ -7,7 +7,7 @@
 from .scalar_data.base import ScalarStorage
 from .vector_data.base import VectorBase, ClearStrategy
 from .eviction import EvictionManager
-from ..util.error import CacheError
+from ..utils.error import CacheError
 
 
 class DataManager(metaclass=ABCMeta):

diff --git a/gptcache/cache/factory.py b/gptcache/cache/factory.py
@@ -1,7 +1,7 @@
 from .data_manager import DataManager, SSDataManager
 from .scalar_data import SQLDataBase, SQL_URL
 from .vector_data import Milvus, Faiss, Chromadb
-from ..util.error import NotFoundStoreError, ParamError
+from ..utils.error import NotFoundStoreError, ParamError
 
 
 def get_data_manager(data_manager_name: str, **kwargs) -> DataManager:

diff --git a/gptcache/cache/scalar_data/__init__.py b/gptcache/cache/scalar_data/__init__.py
@@ -1,6 +1,6 @@
 __all__ = ['SQLDataBase', 'SQL_URL']
 
-from gptcache.util.lazy_import import LazyImport
+from gptcache.utils.lazy_import import LazyImport
 
 sql_database = LazyImport('milvus', globals(), 'gptcache.cache.scalar_data.sqlalchemy')
 

diff --git a/gptcache/cache/scalar_data/sqlalchemy.py b/gptcache/cache/scalar_data/sqlalchemy.py
@@ -1,4 +1,4 @@
-from gptcache.util import import_sqlalchemy
+from gptcache.utils import import_sqlalchemy
 import_sqlalchemy()
 
 import numpy as np

diff --git a/gptcache/cache/vector_data/__init__.py b/gptcache/cache/vector_data/__init__.py
@@ -1,6 +1,6 @@
 __all__ = ['Milvus', 'Faiss', 'Chromadb']
 
-from gptcache.util.lazy_import import LazyImport
+from gptcache.utils.lazy_import import LazyImport
 
 milvus = LazyImport('milvus', globals(), 'gptcache.cache.vector_data.milvus')
 faiss = LazyImport('faiss', globals(), 'gptcache.cache.vector_data.faiss')

diff --git a/gptcache/cache/vector_data/chroma.py b/gptcache/cache/vector_data/chroma.py
@@ -1,4 +1,4 @@
-from gptcache.util import import_chromadb
+from gptcache.utils import import_chromadb
 import_chromadb()
 
 import chromadb

diff --git a/gptcache/cache/vector_data/faiss.py b/gptcache/cache/vector_data/faiss.py
@@ -1,4 +1,4 @@
-from gptcache.util import import_faiss
+from gptcache.utils import import_faiss
 import_faiss()
 
 import os

diff --git a/gptcache/cache/vector_data/milvus.py b/gptcache/cache/vector_data/milvus.py
@@ -1,4 +1,4 @@
-from gptcache.util import import_pymilvus
+from gptcache.utils import import_pymilvus
 import_pymilvus()
 
 from uuid import uuid4

diff --git a/gptcache/core.py b/gptcache/core.py
@@ -3,14 +3,14 @@
 import time
 
 import openai
-from .encoder.string import to_embeddings as string_embedding
+from .embedding.string import to_embeddings as string_embedding
 from .cache.data_manager import DataManager
 from .cache.factory import get_data_manager
 from .processor.post import first
 from .processor.pre import last_content
-from .ranker.similarity_evaluation import SimilarityEvaluation
-from .ranker.string import AbsoluteEvaluation
-from .util.error import CacheError
+from .similarity_evaluation.similarity_evaluation import SimilarityEvaluation
+from .similarity_evaluation.string import AbsoluteEvaluation
+from .utils.error import CacheError
 
 
 def cache_all(*args, **kwargs):

diff --git a/gptcache/embedding/__init__.py b/gptcache/embedding/__init__.py
@@ -0,0 +1,9 @@
+__all__ = ['Towhee']
+
+from gptcache.utils.lazy_import import LazyImport
+
+towhee = LazyImport('towhee', globals(), 'gptcache.embedding.towhee')
+
+
+def Towhee(model="paraphrase-albert-small-v2"):
+    return towhee.Towhee(model)
diff --git a/gptcache/encoder/openai.py → gptcache/embedding/openai.py b/gptcache/encoder/openai.py → gptcache/embedding/openai.py
diff --git a/gptcache/encoder/string.py → gptcache/embedding/string.py b/gptcache/encoder/string.py → gptcache/embedding/string.py
diff --git a/gptcache/encoder/towhee.py → gptcache/embedding/towhee.py b/gptcache/encoder/towhee.py → gptcache/embedding/towhee.py
@@ -1,4 +1,4 @@
-from gptcache.util import import_towhee
+from gptcache.utils import import_towhee
 import_towhee()
 
 import numpy as np

diff --git a/gptcache/encoder/__init__.py b/gptcache/encoder/__init__.py
diff --git a/gptcache/ranker/__init__.py b/gptcache/ranker/__init__.py
diff --git a/gptcache/similarity_evaluation/__init__.py b/gptcache/similarity_evaluation/__init__.py
@@ -0,0 +1,9 @@
+__all__ = ['Towhee']
+
+from gptcache.utils.lazy_import import LazyImport
+
+towhee = LazyImport('towhee', globals(), 'gptcache.similarity_evaluation.towhee')
+
+
+def Towhee():
+    return towhee.Towhee()
diff --git a/gptcache/ranker/np.py → gptcache/similarity_evaluation/np.py b/gptcache/ranker/np.py → gptcache/similarity_evaluation/np.py
diff --git a/gptcache/ranker/similarity_evaluation.py → ...arity_evaluation/similarity_evaluation.py b/gptcache/ranker/similarity_evaluation.py → ...arity_evaluation/similarity_evaluation.py
diff --git a/gptcache/ranker/simple.py → gptcache/similarity_evaluation/simple.py b/gptcache/ranker/simple.py → gptcache/similarity_evaluation/simple.py
diff --git a/gptcache/ranker/string.py → gptcache/similarity_evaluation/string.py b/gptcache/ranker/string.py → gptcache/similarity_evaluation/string.py
diff --git a/gptcache/ranker/towhee.py → gptcache/similarity_evaluation/towhee.py b/gptcache/ranker/towhee.py → gptcache/similarity_evaluation/towhee.py
diff --git a/gptcache/util/__init__.py → gptcache/utils/__init__.py b/gptcache/util/__init__.py → gptcache/utils/__init__.py
@@ -1,4 +1,5 @@
-__all__ = ['import_pymilvus', 'import_towhee', 'import_faiss']
+__all__ = ['import_pymilvus', 'import_towhee',
+           'import_faiss', 'import_sqlalchemy']
 
 from .dependency_control import prompt_install
 

diff --git a/gptcache/util/dependency_control.py → gptcache/utils/dependency_control.py b/gptcache/util/dependency_control.py → gptcache/utils/dependency_control.py
diff --git a/gptcache/util/error.py → gptcache/utils/error.py b/gptcache/util/error.py → gptcache/utils/error.py
diff --git a/gptcache/util/lazy_import.py → gptcache/utils/lazy_import.py b/gptcache/util/lazy_import.py → gptcache/utils/lazy_import.py
diff --git a/scripts/manage_conda_env.sh b/scripts/manage_conda_env.sh
@@ -22,7 +22,7 @@ if [[ "$1" == "create" ]]; then
     conda activate "$env_name"
     echo "conda environment '$env_name' activated."
 elif [[ "$1" == "remove" ]]; then
-    conda deactivate "$env_name"
+    conda deactivate
     # 删除 conda 环境
     if [[ -n "$2" ]]; then
         env_name="$2"

diff --git a/scripts/remove_example_cache.sh b/scripts/remove_example_cache.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 parent_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-find "$parent_dir/examples" \( -path "$parent_dir/examples/benchmark" -path "$parent_dir/examples/sqlite_milvus_mock" \) -prune -o \( -type f \( -name 'data_map*.txt' -or -name 'faiss.index' -or -name 'sqlite.db' \) -delete \)
+find "$parent_dir/examples" \( -path "$parent_dir/examples/benchmark" -path "$parent_dir/examples/sqlite_milvus_mock" \) -prune -o \( -type f \( -name 'data_map*.txt' -or -name 'faiss.index' -or -name 'gptcache.db' \) -delete \)
diff --git a/tests/integration_tests/test_sqlite_faiss_towhee.py b/tests/integration_tests/test_sqlite_faiss_towhee.py
@@ -4,10 +4,10 @@
 from gptcache.adapter import openai
 from gptcache.cache.factory import get_ss_data_manager
 from gptcache.core import cache, Config
-from gptcache.encoder import Towhee
-from gptcache.ranker.simple import SearchDistanceEvaluation
+from gptcache.embedding import Towhee
+from gptcache.similarity_evaluation.simple import SearchDistanceEvaluation
 
-sqlite_file = "sqlite.db"
+sqlite_file = "gptcache.db"
 faiss_file = "faiss.index"
 
 

diff --git a/..._tests/cache/vector_data/test_chromadb.py → tests/unit_tests/cache/test_chromadb.py b/..._tests/cache/vector_data/test_chromadb.py → tests/unit_tests/cache/test_chromadb.py
diff --git a/tests/unit_tests/test_core.py b/tests/unit_tests/test_core.py