Merge pull request HamaWhiteGG#67 from HamaWhiteGG/dev

Support Milvus HamaWhiteGG#38
RazorChen · Aug 4, 2023 · 7780bd9 · 7780bd9
2 parents fd74c51 + 00f4ba5
commit 7780bd9
Show file tree

Hide file tree

Showing 11 changed files with 689 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -16,11 +16,12 @@ The following example can view in the [langchain-example](langchain-examples/src
 
 - [SQL Chains](langchain-examples/src/main/java/com/hw/langchain/examples/chains/SqlChainExample.java)
 - [API Chains](langchain-examples/src/main/java/com/hw/langchain/examples/chains/ApiChainExample.java)
-- [Spark SQL Agent](langchain-bigdata/langchain-spark/src/test/java/com/hw/langchain/agents/toolkits/spark/sql/toolkit/SparkSqlToolkitTest.java)
-- [Flink SQL Agent](langchain-bigdata/langchain-flink/src/test/java/com/hw/langchain/agents/toolkits/flink/sql/toolkit/FlinkSqlToolkitTest.java)
+- [QA-Milvus](langchain-examples/src/main/java/com/hw/langchain/examples/chains/MilvusExample.java)
+- [QA-Pinecone](langchain-examples/src/main/java/com/hw/langchain/examples/chains/RetrievalQaExample.java)
+- [QA-Pinecone-Markdown](langchain-examples/src/main/java/com/hw/langchain/examples/chains/RetrievalMarkdownExample.java)
 - [Agent with Google Search](langchain-examples/src/main/java/com/hw/langchain/examples/agents/LlmAgentExample.java)
-- [Question answering over documents](langchain-examples/src/main/java/com/hw/langchain/examples/chains/RetrievalQaExample.java)
-- [Context aware text splitting and QA](langchain-examples/src/main/java/com/hw/langchain/examples/chains/RetrievalMarkdownExample.java)
+- [Spark SQL AI](langchain-bigdata/langchain-spark/src/test/java/com/hw/langchain/agents/toolkits/spark/sql/toolkit/SparkSqlToolkitTest.java)
+- [Flink SQL AI](langchain-bigdata/langchain-flink/src/test/java/com/hw/langchain/agents/toolkits/flink/sql/toolkit/FlinkSqlToolkitTest.java)
 
 ## 3. Quickstart Guide
 This tutorial gives you a quick walkthrough about building an end-to-end language model application with LangChain.

diff --git a/langchain-core/pom.xml b/langchain-core/pom.xml
@@ -29,6 +29,11 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>io.milvus</groupId>
+            <artifactId>milvus-sdk-java</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>

diff --git a/langchain-core/src/main/java/com/hw/langchain/chains/query/constructor/JsonUtils.java b/langchain-core/src/main/java/com/hw/langchain/chains/query/constructor/JsonUtils.java
@@ -58,6 +58,15 @@ public static String toJsonStringWithIndent(Object object) {
         return toJsonStringWithIndent(object, 4);
     }
 
+    public static String writeValueAsString(Object object) {
+        try {
+            ObjectMapper objectMapper = new ObjectMapper();
+            return objectMapper.writeValueAsString(object);
+        } catch (JsonProcessingException e) {
+            throw new LangChainException("Failed to format attribute info.", e);
+        }
+    }
+
     public static <T> T convertFromJsonStr(String jsonStr, Class<T> clazz) {
         try {
             return OBJECT_MAPPER.readValue(jsonStr, clazz);

diff --git a/langchain-core/src/main/java/com/hw/langchain/schema/Document.java b/langchain-core/src/main/java/com/hw/langchain/schema/Document.java
@@ -18,6 +18,8 @@
 
 package com.hw.langchain.schema;
 
+import com.google.common.collect.Maps;
+
 import lombok.Data;
 
 import java.util.Map;
@@ -38,4 +40,8 @@ public Document(String pageContent, Map<String, Object> metadata) {
         this.pageContent = pageContent;
         this.metadata = metadata;
     }
+
+    public Document(String pageContent) {
+        this(pageContent, Maps.newHashMap());
+    }
 }
diff --git a/langchain-core/src/main/java/com/hw/langchain/vectorstores/base/VectorStore.java b/langchain-core/src/main/java/com/hw/langchain/vectorstores/base/VectorStore.java
@@ -42,11 +42,9 @@ public abstract class VectorStore {
      *
      * @param texts     Iterable of strings to add to the vectorStore.
      * @param metadatas list of metadatas associated with the texts.
-     * @param kwargs    vectorStore specific parameters
      * @return List of ids from adding the texts into the vectorStore.
      */
-    public abstract List<String> addTexts(List<String> texts, List<Map<String, Object>> metadatas,
-            Map<String, Object> kwargs);
+    public abstract List<String> addTexts(List<String> texts, List<Map<String, Object>> metadatas);
 
     /**
      * Delete by vector ID.
@@ -66,7 +64,7 @@ public abstract List<String> addTexts(List<String> texts, List<Map<String, Objec
     public List<String> addDocuments(List<Document> documents, Map<String, Object> kwargs) {
         var texts = documents.stream().map(Document::getPageContent).toList();
         var metadatas = documents.stream().map(Document::getMetadata).toList();
-        return addTexts(texts, metadatas, kwargs);
+        return addTexts(texts, metadatas);
     }
 
     public List<Document> search(String query, SearchType searchType, Map<String, Object> filter) {
@@ -125,7 +123,7 @@ public List<Pair<Document, Float>> similaritySearchWithRelevanceScores(String qu
      * @return List of Tuples of (doc, similarityScore)
      */
     public List<Pair<Document, Float>> similaritySearchWithRelevanceScores(String query, int k) {
-        List<Pair<Document, Float>> docsAndSimilarities = _similaritySearchWithRelevanceScores(query, k);
+        List<Pair<Document, Float>> docsAndSimilarities = innerSimilaritySearchWithRelevanceScores(query, k);
 
         // Check relevance scores and filter by threshold
         if (docsAndSimilarities.stream().anyMatch(pair -> pair.getRight() < 0.0f || pair.getRight() > 1.0f)) {
@@ -141,7 +139,7 @@ public List<Pair<Document, Float>> similaritySearchWithRelevanceScores(String qu
      * @param k     Number of Documents to return.
      * @return List of Tuples of (doc, similarityScore)
      */
-    protected abstract List<Pair<Document, Float>> _similaritySearchWithRelevanceScores(String query, int k);
+    protected abstract List<Pair<Document, Float>> innerSimilaritySearchWithRelevanceScores(String query, int k);
 
     /**
      * Return docs most similar to embedding vector.