Added web interactor tool functionality (TransformerOptimus#350)

The SuperAGI Web Interactor Tool with Playwright equips users to interact and manipulate web content efficiently, paving the way for diverse and compelling use cases. Features: 1. Load Webpages: With SuperAGI's Web Interactor Tool, you can effortlessly load any webpage and view its content, ensuring you can access and interact with any website you need. 2. Interact with Web Elements: The tool lets you interact with various elements on a webpage, such as links, buttons, and forms, providing a highly interactive web experience. 3. Scrape Web Content: This feature allows the tool to extract specific content from webpages, useful for tasks like data collection and analysis. 4. Navigate Through Webpages: The tool helps you seamlessly navigate through webpages by clicking on links or using the browser's forward and back buttons. Authored-by: COLONAYUSH <ayushkaps9462@gmail.com>
ashwin-athappan · Jun 14, 2023 · f6bbc7c · f6bbc7c
1 parent 7c151bb
commit f6bbc7c
Show file tree

Hide file tree

Showing 16 changed files with 669 additions and 9 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -6,6 +6,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y git wget libpq-
 RUN pip install --upgrade pip
 
 RUN pip install --no-cache-dir -r requirements.txt
+RUN playwright install --with-deps chromium
 
 WORKDIR /app
 COPY . .

diff --git a/DockerfileCelery b/DockerfileCelery
@@ -7,9 +7,18 @@ RUN pip install --upgrade pip
 
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
+RUN apt-get update && \
+    apt-get install -y xvfb xauth x11vnc && \
+    apt-get clean
+RUN apt-get update && playwright install --with-deps chromium
+
+
+COPY entrypoint_celery.sh /entrypoint_celery.sh
+RUN chmod +x /entrypoint_celery.sh
 
 WORKDIR /app
 COPY . .
 COPY config.yaml .
 
+ENTRYPOINT ["/entrypoint_celery.sh"]
 CMD ["celery", "-A", "superagi.worker", "worker", "--loglevel=info"]
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,5 +1,4 @@
 version: '3.8'
-
 services:
   backend:
     volumes:
@@ -12,7 +11,6 @@ services:
       - super__postgres
     networks:
       - super_network
-
   celery:
     volumes:
       - "./:/app"
@@ -24,7 +22,10 @@ services:
       - super__postgres
     networks:
       - super_network
-
+    environment:
+      - DISPLAY=:0
+    ports:
+      - "5900:5900"
   gui:
     build: ./gui
     ports:
@@ -37,12 +38,10 @@ services:
       - ./gui:/app
       - /app/node_modules
       - /app/.next
-
   super__redis:
     image: "docker.io/library/redis:latest"
     networks:
       - super_network
-
   super__postgres:
     image: "docker.io/library/postgres:latest"
     environment:
@@ -55,10 +54,8 @@ services:
       - super_network
     ports:
       - "5432:5432"
-
 networks:
   super_network:
     driver: bridge
-
 volumes:
   superagi_postgres_data:
diff --git a/entrypoint_celery.sh b/entrypoint_celery.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+Xvfb :0 -screen 0 1280x1024x24 &
+x11vnc -display :0 -N -forever -shared &
+
+exec "$@"
diff --git a/requirements.txt b/requirements.txt
@@ -83,10 +83,8 @@ psycopg2==2.9.6
 pycparser==2.21
 pydantic==1.10.8
 pydantic-sqlalchemy==0.0.9
-pyee==8.2.2
 PyJWT==1.7.1
 PyPDF2==3.0.1
-pyppeteer==1.0.2
 pyquery==2.0.0
 pyrsistent==0.19.3
 python-dateutil==2.8.2
@@ -130,5 +128,7 @@ yarl==1.9.2
 zipp==3.15.0
 tiktoken==0.4.0
 psycopg2==2.9.6
+playwright==1.34.0
+boto3~=1.26.146
 slack-sdk==3.21.3
 pytest==7.3.2
diff --git a/superagi/helper/browser_wrapper.py b/superagi/helper/browser_wrapper.py
@@ -0,0 +1,33 @@
+from playwright.sync_api import sync_playwright
+
+class BrowserWrapper:
+    def __init__(self):
+        self.page = None
+        self.client = None
+        self.page_element_buffer = None
+
+    def start_browser_and_goto_page(self, url: str) -> str:
+        global browser
+
+        browser = (
+            sync_playwright()
+            .start()
+            .chromium.launch(
+                headless=False,
+            )
+        )
+
+        self.page = browser.new_page()
+        self.page.set_viewport_size({"width": 1280, "height": 1080})
+
+        try:
+            self.page.goto(url=url if "://" in url else "http://" + url)
+            self.client = self.page.context.new_cdp_session(self.page)
+            self.page_element_buffer = {}
+        except:
+            return "Failed to go to URL, please try again and make sure the URL is correct."
+
+        return f"Browser successfully started and navigated to {url}!"
+
+
+browser_wrapper = BrowserWrapper()
diff --git a/superagi/tools/web_interactor/__init__.py b/superagi/tools/web_interactor/__init__.py
diff --git a/superagi/tools/web_interactor/playwright_click_element_by_id.py b/superagi/tools/web_interactor/playwright_click_element_by_id.py
@@ -0,0 +1,51 @@
+from superagi.tools.base_tool import BaseTool
+from pydantic import BaseModel, Field
+from superagi.helper.browser_wrapper import browser_wrapper
+
+from typing import Type, Optional, List
+
+from pydantic import BaseModel, Field
+
+from superagi.tools.base_tool import BaseTool
+from pydantic import BaseModel, Field, PrivateAttr
+black_listed_elements = set(["html", "head", "title", "meta", "iframe", "body", "style", "script", "path", "svg", "br", "::marker",])
+# global page_element_buffer
+# page_element_buffer = {}
+
+
+class ClickElementByIdSchema(BaseModel):
+    element_id: str = Field(
+        ..., 
+        description="The ID of the HTML element to click."
+        )
+
+
+class ClickElementByIdTool(BaseTool):
+    name = "PlaywrightClickElementByID"
+    description = "A tool for clicking an element by its ID using Playwright.clicks an element. Specify the id with the unique id received from the get_dom command. CRITICAL: The ID must be the integer id from the get_dom command. It should execute after getting the DOM"
+    args_schema: Type[ClickElementByIdSchema] = ClickElementByIdSchema
+
+    def _execute(self, element_id: str) -> str:
+        page_element_buffer = browser_wrapper.page_element_buffer
+        client = browser_wrapper.client
+        page = browser_wrapper.page
+
+        # Inject javascript into the page which removes the target= attribute from all links
+        js = """
+        links = document.getElementsByTagName("a");
+        for (var i = 0; i < links.length; i++) {
+            links[i].removeAttribute("target");
+        }
+        """
+        page.evaluate(js)
+
+        element = page_element_buffer.get(int(element_id))
+        if element:
+            x = element.get("center_x")
+            y = element.get("center_y")
+
+            page.mouse.click(x, y)
+        else:
+            return "Could not find element"
+
+        return "Successfully clicked!"
diff --git a/superagi/tools/web_interactor/playwright_enter.py b/superagi/tools/web_interactor/playwright_enter.py
@@ -0,0 +1,24 @@
+from superagi.tools.base_tool import BaseTool
+from pydantic import BaseModel, Field
+from superagi.helper.browser_wrapper import browser_wrapper
+
+from typing import Type, Optional, List
+
+from pydantic import BaseModel, Field
+
+from superagi.tools.base_tool import BaseTool
+from pydantic import BaseModel, Field, PrivateAttr
+class EnterSchema(BaseModel):
+    pass
+
+
+class EnterTool(BaseTool):
+    name = "PlaywrightEnter"
+    description = "A tool for pressing the Enter key using Playwright."
+    args_schema: Type[EnterSchema] = EnterSchema
+
+    def _execute(self) -> str:
+        page = browser_wrapper.page
+
+        page.keyboard.press("Enter")
+        return "Pressed enter!"