feat: add VSCode to OpenHands runtime and UI (#4745)

Co-authored-by: openhands <openhands@all-hands.dev> Co-authored-by: Robert Brennan <accounts@rbren.io>
All-Hands-AI · Nov 13, 2024 · fd81670 · fd81670
1 parent 79ed4e3
commit fd81670
Show file tree

Hide file tree

Showing 22 changed files with 469 additions and 69 deletions.
diff --git a/frontend/src/api/open-hands.ts b/frontend/src/api/open-hands.ts
@@ -8,6 +8,7 @@ import {
   GitHubAccessTokenResponse,
   ErrorResponse,
   GetConfigResponse,
+  GetVSCodeUrlResponse,
 } from "./open-hands.types";
 
 class OpenHands {
@@ -174,6 +175,14 @@ class OpenHands {
       true,
     );
   }
+
+  /**
+   * Get the VSCode URL
+   * @returns VSCode URL
+   */
+  static async getVSCodeUrl(): Promise<GetVSCodeUrlResponse> {
+    return request(`/api/vscode-url`, {}, false, false, 1);
+  }
 }
 
 export default OpenHands;
diff --git a/frontend/src/api/open-hands.types.ts b/frontend/src/api/open-hands.types.ts
@@ -46,3 +46,8 @@ export interface GetConfigResponse {
   GITHUB_CLIENT_ID: string;
   POSTHOG_CLIENT_KEY: string;
 }
+
+export interface GetVSCodeUrlResponse {
+  vscode_url: string | null;
+  error?: string;
+}
diff --git a/frontend/src/assets/vscode-alt.svg b/frontend/src/assets/vscode-alt.svg
diff --git a/frontend/src/components/file-explorer/FileExplorer.tsx b/frontend/src/components/file-explorer/FileExplorer.tsx
@@ -12,6 +12,7 @@ import { useTranslation } from "react-i18next";
 import { twMerge } from "tailwind-merge";
 import AgentState from "#/types/AgentState";
 import { setRefreshID } from "#/state/codeSlice";
+import { addAssistantMessage } from "#/state/chatSlice";
 import IconButton from "../IconButton";
 import ExplorerTree from "./ExplorerTree";
 import toast from "#/utils/toast";
@@ -20,6 +21,7 @@ import { I18nKey } from "#/i18n/declaration";
 import OpenHands from "#/api/open-hands";
 import { useFiles } from "#/context/files";
 import { isOpenHandsErrorResponse } from "#/api/open-hands.utils";
+import VSCodeIcon from "#/assets/vscode-alt.svg?react";
 
 interface ExplorerActionsProps {
   onRefresh: () => void;
@@ -168,6 +170,35 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
     }
   };
 
+  const handleVSCodeClick = async (e: React.MouseEvent) => {
+    e.preventDefault();
+    try {
+      const response = await OpenHands.getVSCodeUrl();
+      if (response.vscode_url) {
+        dispatch(
+          addAssistantMessage(
+            "You opened VS Code. Please inform the agent of any changes you made to the workspace or environment. To avoid conflicts, it's best to pause the agent before making any changes.",
+          ),
+        );
+        window.open(response.vscode_url, "_blank");
+      } else {
+        toast.error(
+          `open-vscode-error-${new Date().getTime()}`,
+          t(I18nKey.EXPLORER$VSCODE_SWITCHING_ERROR_MESSAGE, {
+            error: response.error,
+          }),
+        );
+      }
+    } catch (exp_error) {
+      toast.error(
+        `open-vscode-error-${new Date().getTime()}`,
+        t(I18nKey.EXPLORER$VSCODE_SWITCHING_ERROR_MESSAGE, {
+          error: String(exp_error),
+        }),
+      );
+    }
+  };
+
   React.useEffect(() => {
     refreshWorkspace();
   }, [curAgentState]);
@@ -210,7 +241,7 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
           !isOpen ? "w-12" : "w-60",
         )}
       >
-        <div className="flex flex-col relative h-full px-3 py-2">
+        <div className="flex flex-col relative h-full px-3 py-2 overflow-hidden">
           <div className="sticky top-0 bg-neutral-800">
             <div
               className={twMerge(
@@ -232,7 +263,7 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
             </div>
           </div>
           {!error && (
-            <div className="overflow-auto flex-grow">
+            <div className="overflow-auto flex-grow min-h-0">
               <div style={{ display: !isOpen ? "none" : "block" }}>
                 <ExplorerTree files={paths} />
               </div>
@@ -243,6 +274,27 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
               <p className="text-neutral-300 text-sm">{error}</p>
             </div>
           )}
+          {isOpen && (
+            <button
+              type="button"
+              onClick={handleVSCodeClick}
+              disabled={
+                curAgentState === AgentState.INIT ||
+                curAgentState === AgentState.LOADING
+              }
+              className={twMerge(
+                "mt-auto mb-2 w-full h-10 text-white rounded flex items-center justify-center gap-2 transition-colors",
+                curAgentState === AgentState.INIT ||
+                  curAgentState === AgentState.LOADING
+                  ? "bg-neutral-600 cursor-not-allowed"
+                  : "bg-[#4465DB] hover:bg-[#3451C7]",
+              )}
+              aria-label="Open in VS Code"
+            >
+              <VSCodeIcon width={20} height={20} />
+              Open in VS Code
+            </button>
+          )}
         </div>
         <input
           data-testid="file-input"

diff --git a/frontend/src/i18n/translation.json b/frontend/src/i18n/translation.json
@@ -678,6 +678,16 @@
     "tr": "Sunucudan beklenmeyen yanıt yapısı",
     "no": "Uventet responsstruktur fra serveren"
   },
+  "EXPLORER$VSCODE_SWITCHING_MESSAGE": {
+    "en": "Switching to VS Code in 3 seconds...\nImportant: Please inform the agent of any changes you make in VS Code. To avoid conflicts, wait for the assistant to complete its work before making your own changes.",
+    "zh-CN": "3 秒后切换到 VS Code\n重要提示：请告知 OpenHands 您在 VS Code 中进行的任何更改。为了避免冲突，请在 OpenHands 完成工作后再进行自己的更改。",
+    "zh-TW": "3 秒後切換到 VS Code\n重要提示：請告知 OpenHands 您在 VS Code 中進行的任何更改。為避免衝突，請在 OpenHands 完成工作後再進行自己的更改。"
+  },
+  "EXPLORER$VSCODE_SWITCHING_ERROR_MESSAGE": {
+    "en": "Error switching to VS Code: {{error}}",
+    "zh-CN": "切换到 VS Code 时发生错误: {{error}}",
+    "zh-TW": "切換到 VS Code 時發生錯誤: {{error}}"
+  },
   "LOAD_SESSION$MODAL_TITLE": {
     "en": "Return to existing session?",
     "de": "Zurück zu vorhandener Sitzung?",

diff --git a/frontend/src/utils/toast.tsx b/frontend/src/utils/toast.tsx
@@ -10,7 +10,6 @@ export default {
       style: {
         background: "#ef4444",
         color: "#fff",
-        lineBreak: "anywhere",
       },
       iconTheme: {
         primary: "#ef4444",
@@ -19,25 +18,20 @@ export default {
     });
     idMap.set(id, toastId);
   },
-  success: (id: string, msg: string) => {
-    const toastId = idMap.get(id);
-    if (toastId === undefined) return;
-    if (toastId) {
-      toast.success(msg, {
-        id: toastId,
-        duration: 4000,
-        style: {
-          background: "#333",
-          color: "#fff",
-          lineBreak: "anywhere",
-        },
-        iconTheme: {
-          primary: "#333",
-          secondary: "#fff",
-        },
-      });
-    }
-    idMap.delete(id);
+  success: (id: string, msg: string, duration: number = 4000) => {
+    if (idMap.has(id)) return; // prevent duplicate toast
+    const toastId = toast.success(msg, {
+      duration,
+      style: {
+        background: "#333",
+        color: "#fff",
+      },
+      iconTheme: {
+        primary: "#333",
+        secondary: "#fff",
+      },
+    });
+    idMap.set(id, toastId);
   },
   settingsChanged: (msg: string) => {
     toast(msg, {
@@ -48,7 +42,6 @@ export default {
       style: {
         background: "#333",
         color: "#fff",
-        lineBreak: "anywhere",
       },
     });
   },

diff --git a/openhands/core/cli.py b/openhands/core/cli.py
@@ -116,6 +116,7 @@ async def main():
         event_stream=event_stream,
         sid=sid,
         plugins=agent_cls.sandbox_plugins,
+        headless_mode=True,
     )
 
     controller = AgentController(

diff --git a/openhands/core/main.py b/openhands/core/main.py
@@ -54,11 +54,14 @@ def read_task_from_stdin() -> str:
 def create_runtime(
     config: AppConfig,
     sid: str | None = None,
+    headless_mode: bool = True,
 ) -> Runtime:
     """Create a runtime for the agent to run on.
 
     config: The app config.
     sid: The session id.
+    headless_mode: Whether the agent is run in headless mode. `create_runtime` is typically called within evaluation scripts,
+        where we don't want to have the VSCode UI open, so it defaults to True.
     """
     # if sid is provided on the command line, use it as the name of the event stream
     # otherwise generate it on the basis of the configured jwt_secret
@@ -80,6 +83,7 @@ def create_runtime(
         event_stream=event_stream,
         sid=session_id,
         plugins=agent_cls.sandbox_plugins,
+        headless_mode=headless_mode,
     )
 
     return runtime
@@ -122,7 +126,7 @@ async def run_controller(
     sid = sid or generate_sid(config)
 
     if runtime is None:
-        runtime = create_runtime(config, sid=sid)
+        runtime = create_runtime(config, sid=sid, headless_mode=headless_mode)
         await runtime.connect()
 
     event_stream = runtime.event_stream

diff --git a/openhands/runtime/action_execution_server.py b/openhands/runtime/action_execution_server.py
@@ -47,14 +47,11 @@
 from openhands.events.serialization import event_from_dict, event_to_dict
 from openhands.runtime.browser import browse
 from openhands.runtime.browser.browser_env import BrowserEnv
-from openhands.runtime.plugins import (
-    ALL_PLUGINS,
-    JupyterPlugin,
-    Plugin,
-)
+from openhands.runtime.plugins import ALL_PLUGINS, JupyterPlugin, Plugin, VSCodePlugin
 from openhands.runtime.utils.bash import BashSession
 from openhands.runtime.utils.files import insert_lines, read_lines
 from openhands.runtime.utils.runtime_init import init_user_and_working_directory
+from openhands.runtime.utils.system import check_port_available
 from openhands.utils.async_utils import wait_all
 
 
@@ -116,7 +113,10 @@ def initial_pwd(self):
         return self._initial_pwd
 
     async def ainit(self):
-        await wait_all(self._init_plugin(plugin) for plugin in self.plugins_to_load)
+        await wait_all(
+            (self._init_plugin(plugin) for plugin in self.plugins_to_load),
+            timeout=30,
+        )
 
         # This is a temporary workaround
         # TODO: refactor AgentSkills to be part of JupyterPlugin
@@ -345,6 +345,8 @@ def close(self):
     )
     # example: python client.py 8000 --working-dir /workspace --plugins JupyterRequirement
     args = parser.parse_args()
+    os.environ['VSCODE_PORT'] = str(int(args.port) + 1)
+    assert check_port_available(int(os.environ['VSCODE_PORT']))
 
     plugins_to_load: list[Plugin] = []
     if args.plugins:
@@ -527,6 +529,19 @@ async def download_file(path: str):
     async def alive():
         return {'status': 'ok'}
 
+    # ================================
+    # VSCode-specific operations
+    # ================================
+
+    @app.get('/vscode/connection_token')
+    async def get_vscode_connection_token():
+        assert client is not None
+        if 'vscode' in client.plugins:
+            plugin: VSCodePlugin = client.plugins['vscode']  # type: ignore
+            return {'token': plugin.vscode_connection_token}
+        else:
+            return {'token': None}
+
     # ================================
     # File-specific operations for UI
     # ================================