resolve merge conflict

TheSnowGuru · Jul 3, 2023 · 62231cb · 62231cb
2 parents c3091db + e90c57f
commit 62231cb
Show file tree

Hide file tree

Showing 10 changed files with 403 additions and 13 deletions.
diff --git a/.github/ISSUE_TEMPLATE/1.BUG_REPORT.yml b/.github/ISSUE_TEMPLATE/1.BUG_REPORT.yml
@@ -0,0 +1,149 @@
+name: Bug report 
+description: Create a bug report for SuperAGI.
+labels: ['status: needs triage']
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ### ⚠️ Issue Creation Guideline
+        * Check out our [roadmap] and join our [discord] to discuss what's going on
+        * If you need help, you can ask in the [#general] section or in [#support]
+        * **Thoroughly search the [existing issues] before creating a new one**
+        * Read through our docs:
+        [roadmap]: https://github.com/users/TransformerOptimus/projects/1
+        [discord]: https://discord.gg/dXbRe5BHJC
+        [#general]: https://discord.com/channels/1107593006032355359/1107642413993959505
+        [#support]: https://discord.com/channels/1107593006032355359/1107645922797703198
+        [existing issues]: https://github.com/TransformerOptimus/SuperAGI/issues
+
+  - type: checkboxes
+    attributes:
+      label: ⚠️ Check for existing issues before proceeding. ⚠️
+      description: >
+        Please [search the history](https://github.com/TransformerOptimus/SuperAGI/issues)
+        to see if an issue already exists for the same problem.
+      options:
+        - label: I have searched the existing issues, and there is no existing issue for my problem
+          required: true
+
+  - type: markdown
+    attributes:
+      value: |
+        Please confirm that the issue you have is described well and precise in the title above ⬆️.
+        Think like this: What would you type if you were searching for the issue?
+        
+        For example:
+        ❌ - my SuperAGI agent keeps looping
+        ✅ - After performing Write Tool, SuperAGI goes into a loop where it keeps trying to write the file.
+        
+        Please help us help you by following these steps:
+        - Search for existing issues, adding a comment when you have the same or similar issue is tidier than "new issue" and 
+          newer issues will not be reviewed earlier, this is dependent on the current priorities set by our wonderful team
+        - Ask on our Discord if your issue is known when you are unsure (https://discord.gg/dXbRe5BHJC)
+        - Provide relevant info:
+          - Provide Docker Logs(docker compose logs) whenever possible.
+          - If it's a pip/packages issue, mention this in the title and provide pip version, python version.
+  - type: dropdown
+    attributes:
+      label: Where are you using SuperAGI?
+      description: >
+        Please select the operating system you were using to run SuperAGI when this problem occurred.
+      options:
+        - Windows
+        - Linux
+        - MacOS
+        - Codespaces
+        - Web Version
+        - Other
+    validations:
+      required: true
+      nested_fields:
+        - type: text
+          attributes:
+            label: Specify the system
+            description: Please specify the system you are working on.
+
+  - type: dropdown
+    attributes:
+      label: Which branch of SuperAGI are you using?
+      description: |
+        Please select which version of SuperAGI you were using when this issue occurred.
+        If installed with git you can run `git branch` to see which version of Auto-GPT you are running.
+      options:
+        - Main
+        - Dev (branch)
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Do you use OpenAI GPT-3.5 or GPT-4?
+      description: >
+        If you are using SuperAGI with GPT-3.5, your problems may be caused by
+        the limitations of GPT-3.5 like incorrect Tool selection thus causing looping in the agent feed.
+      options:
+        - GPT-3.5
+        - GPT-3.5(16k)
+        - GPT-4
+        - GPT-4(32k)
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Which area covers your issue best?
+      description: >
+        Select the area related to the issue you are reporting.
+      options:
+        - Installation and setup
+        - Resource Manager
+        - Action Console
+        - Performance
+        - Marketplace
+        - Prompt
+        - Tools
+        - Agents
+        - Documentation
+        - Logging
+        - Other
+    validations:
+      required: true
+      autolabels: true
+      nested_fields:
+        - type: text
+          attributes:
+            label: Specify the area
+            description: Please specify the area you think is best related to the issue.
+
+  - type: textarea
+    attributes:
+      label: Describe your issue.
+      description: Describe the problem you are experiencing. Try to describe only the issue and phrase it short but clear. ⚠️ Provide NO other data in this field  
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+     label: How to replicate your Issue?
+     description: |
+      Mention Agent Name, Agent Description and Agent Goals, along with Model selected. 
+      Provide any other data which might be relevant for us to replicate this issue.
+      ⚠️ Provide NO other data in this field  
+    validations:
+     required: false
+
+  - type: markdown
+    attributes:
+      value: |
+        ⚠️ Please keep in mind that the log files may contain personal information such as credentials. Make sure you hide them before copy/pasting it! ⚠️        
+  - type: input
+    attributes:
+      label: Upload Error Log Content
+      description: |
+        Upload the error log content, this can help us understand the issue better. 
+        To do this, you can simply copy the logs from the terminal with which you did 'docker compose up' or in a new terminal, 
+        enter 'docker compose logs' and copy/paste the error contents to this field. 
+        ⚠️ The activity log may contain personal data given to SuperAGI by you in prompt or input as well as 
+        any personal information that SuperAGI collected out of files during last run. Please hide them before sharing. ⚠️
+    validations:
+      required: True
diff --git a/gui/pages/Content/Agents/AgentCreate.js b/gui/pages/Content/Agents/AgentCreate.js
@@ -57,7 +57,7 @@ export default function AgentCreate({sendAgentData, selectedProjectId, fetchAgen
   const rollingRef = useRef(null);
   const [rollingDropdown, setRollingDropdown] = useState(false);
 
-  const databases = ["Pinecone"]
+  const databases = ["Pinecone", "LanceDB"]
   const [database, setDatabase] = useState(databases[0]);
   const databaseRef = useRef(null);
   const [databaseDropdown, setDatabaseDropdown] = useState(false);
@@ -195,7 +195,8 @@ export default function AgentCreate({sendAgentData, selectedProjectId, fetchAgen
     setLocalStorageArray("tool_names_" + String(internalId), updatedToolNames, setToolNames);
     setSearchValue('');
   }
-
+
+
   const removeTool = (indexToDelete) => {
     const updatedToolIds = [...selectedTools];
     updatedToolIds.splice(indexToDelete, 1);

diff --git a/requirements.txt b/requirements.txt
@@ -66,6 +66,7 @@ jsonmerge==1.9.0
 jsonschema==4.17.3
 kombu==5.2.4
 llama-index==0.6.35
+lancedb==0.1.8
 log-symbols==0.0.14
 loguru==0.7.0
 lxml==4.9.2

diff --git a/superagi/jobs/agent_executor.py b/superagi/jobs/agent_executor.py
@@ -30,6 +30,7 @@
 from superagi.vector_store.vector_factory import VectorFactory
 from superagi.types.vector_store_types import VectorStoreType
 from superagi.models.agent_config import AgentConfiguration
+import yaml
 
 # from superagi.helper.tool_helper import get_tool_config_by_key
 
@@ -179,11 +180,11 @@ def execute_next_action(self, agent_execution_id):
             if parsed_config["LTM_DB"] == "Pinecone":
                 memory = VectorFactory.get_vector_storage(VectorStoreType.PINECONE, "super-agent-index1",
                                                           OpenAiEmbedding(model_api_key))
-            else:
-                memory = VectorFactory.get_vector_storage(VectorStoreType.PINECONE, "super-agent-index1",
+            elif parsed_config["LTM_DB"] == "LanceDB":
+                memory = VectorFactory.get_vector_storage(VectorStoreType.LANCEDB, "super-agent-index1",
                                                           OpenAiEmbedding(model_api_key))
         except:
-            logger.info("Unable to setup the pinecone connection...")
+            logger.info("Unable to setup the connection...")
             memory = None
 
         user_tools = session.query(Tool).filter(Tool.id.in_(parsed_config["tools"])).all()
@@ -197,7 +198,6 @@ def execute_next_action(self, agent_execution_id):
                                               model_api_key=model_api_key,
                                               resource_description=resource_summary,
                                               session=session)
-
         spawned_agent = SuperAgi(ai_name=parsed_config["name"], ai_role=parsed_config["description"],
                                  llm=OpenAi(model=parsed_config["model"], api_key=model_api_key), tools=tools,
                                  memory=memory,

diff --git a/superagi/tool_manager.py b/superagi/tool_manager.py
@@ -64,7 +64,7 @@ def download_and_extract_tools():
     tools_config = load_tools_config()
 
     for tool_name, tool_url in tools_config.items():
-        tool_folder = os.path.join("", "tools", tool_name)
+        tool_folder = os.path.join("superagi", "tools", tool_name)
         if not os.path.exists(tool_folder):
             os.makedirs(tool_folder)
         download_tool(tool_url, tool_folder)

diff --git a/superagi/types/vector_store_types.py b/superagi/types/vector_store_types.py
@@ -7,6 +7,7 @@ class VectorStoreType(Enum):
     CHROMA = 'chroma'
     WEAVIATE = 'weaviate'
     QDRANT = 'qdrant'
+    LANCEDB = 'LanceDB'
 
     @classmethod
     def get_vector_store_type(cls, store):

diff --git a/superagi/vector_store/lancedb.py b/superagi/vector_store/lancedb.py
@@ -0,0 +1,119 @@
+import uuid
+
+from superagi.vector_store.document import Document
+from superagi.vector_store.base import VectorStore
+from typing import Any, Callable, Optional, Iterable, List
+
+from superagi.vector_store.embedding.openai import BaseEmbedding
+
+
+class LanceDB(VectorStore):
+    """
+    LanceDB vector store.
+
+    Attributes:
+        db : The LanceDB connected database.
+        embedding_model : The embedding model.
+        text_field : The text field is the name of the field where the corresponding text for an embedding is stored.
+    """
+    def __init__(
+            self,
+            db: Any,
+            embedding_model: BaseEmbedding,
+            text_field: str,
+    ):
+        try:
+            import lancedb
+        except ImportError:
+            raise ValueError("Please install LanceDB to use this vector store.")
+
+        self.db = db
+        self.embedding_model = embedding_model
+        self.text_field = text_field
+
+    def add_texts(
+            self,
+            texts: Iterable[str],
+            metadatas: Optional['list[dict]'] = None,
+            ids: Optional['list[str]'] = None,
+            table_name: Optional[str] = None,
+    ) -> 'list[str]':
+        """
+        Add texts to the vector store.
+
+        Args:
+            texts : The texts to add.
+            metadatas: The metadatas to add.
+            ids : The ids to add.
+            table_name : The table to add.
+        Returns:
+            The list of ids vectors stored in LanceDB.
+        """
+
+        vectors = []
+        ids = ids or [str(uuid.uuid4()) for _ in texts]
+        if len(ids) < len(texts):
+            raise ValueError("Number of ids must match number of texts.")
+
+        for text, id in zip(texts, ids):
+            vector = {}
+            metadata = metadatas.pop(0) if metadatas else {}
+            metadata[self.text_field] = text
+
+            vector["id"] = id
+            vector["vector"] = self.embedding_model.get_embedding(text)
+            for key, value in metadata.items():
+                vector[key] = value
+
+            vectors.append(vector)
+
+        try:
+            tbl = self.db.create_table(table_name, data=vectors)
+        except:
+            tbl = self.db.open_table(table_name)
+            tbl.add(vectors)
+
+        return ids
+
+    def get_matching_text(self, query: str, top_k: int = 5, **kwargs: Any) -> List[Document]:
+        """
+        Return docs most similar to query using specified search type.
+
+        Args:
+            query : The query to search.
+            top_k : The top k to search.
+            **kwargs : The keyword arguments to search.
+
+        Returns:
+            The list of documents most similar to the query
+        """
+        namespace = kwargs.get("namespace", "None")
+
+        for table in self.db.table_names():
+            if table == namespace:
+                tbl = self.db.open_table(table)
+
+        try:
+            tbl
+        except:
+            raise ValueError(namespace + " Table not found in LanceDB. Please call this function with a valid table name.")
+
+        embed_text = self.embedding_model.get_embedding(query)
+        res = tbl.search(embed_text).limit(top_k).to_df()
+
+        documents = []
+
+        for i in range(len(res)):
+            meta = {}
+            for col in res:
+                if col != 'vector' and col != 'id' and col != 'score':
+                    meta[col] = res[col][i]
+
+            documents.append(
+                Document(
+                    text_content=res[self.text_field][i],
+                    metadata=meta,
+                )
+            )
+
+        return documents
diff --git a/superagi/vector_store/vector_factory.py b/superagi/vector_store/vector_factory.py
@@ -1,11 +1,13 @@
 import pinecone
+import lancedb
 from pinecone import UnauthorizedException
 
+from superagi.vector_store.pinecone import Pinecone
+from superagi.vector_store.lancedb import LanceDB
+from superagi.vector_store import weaviate
 from superagi.config.config import get_config
 from superagi.lib.logger import logger
 from superagi.types.vector_store_types import VectorStoreType
-from superagi.vector_store import weaviate
-from superagi.vector_store.pinecone import Pinecone
 
 
 class VectorFactory:
@@ -47,9 +49,19 @@ def get_vector_storage(cls, vector_store: VectorStoreType, index_name, embedding
                 return Pinecone(index, embedding_model, 'text')
             except UnauthorizedException:
                 raise ValueError("PineCone API key not found")
-
+
+        if vector_store == VectorStoreType.LANCEDB:
+            try:
+                # connect lancedb to local directory /lancedb/index_name
+                uri = "/lancedb/" + index_name
+                db = lancedb.connect(uri)
+
+                return LanceDB(db, embedding_model, 'text')
+            except:
+                raise ValueError("VectorStore setup for LanceDB failed")
+
         if vector_store == VectorStoreType.WEAVIATE:
-            
+
             use_embedded = bool(get_config("WEAVIATE_USE_EMBEDDED"))
             url = get_config("WEAVIATE_URL")
             api_key = get_config("WEAVIATE_API_KEY")