feat: add bash/browser agent tools + Docker support

- bash-tool.ts: execute shell commands via tool-server HTTP API
- browser-tool.ts: Playwright browser automation (navigate, click, type, screenshot)
- tool-server.mjs: Node.js HTTP server for bash exec + Playwright control (port 7700)
- Dockerfile + docker-compose.yml for containerised deployment
- Register tools in agent toolchain (main.ts, index.ts)
- Add dev:all script to run Vite + tool-server concurrently

packages/web-ui/example/Dockerfile

@@ -0,0 +1,29 @@
+FROM node:20-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git curl ca-certificates \
+    libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 \
+    libdrm2 libdbus-1-3 libxkbcommon0 libatspi2.0-0 \
+    libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libpango-1.0-0 \
+    libasound2 libwayland-client0 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm ci --production
+
+RUN npx playwright install chromium
+
+COPY . .
+
+RUN npm run build 2>/dev/null || true
+
+EXPOSE 5173 7700
+
+ENV TOOL_SERVER_PORT=7700
+
+CMD ["sh", "-c", "node server/tool-server.mjs & npx vite --host 0.0.0.0 --port 5173"]

packages/web-ui/example/docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  jae-web:
+    build: .
+    ports:
+      - "5173:5173"
+      - "7700:7700"
+    environment:
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - OPENAI_BASE_URL=${OPENAI_BASE_URL:-https://api.venice.ai/api/v1}
+      - TOOL_SERVER_PORT=7700
+    volumes:
+      - workspace:/workspace
+    restart: unless-stopped
+
+volumes:
+  workspace:

packages/web-ui/example/package.json

@@ -11,7 +11,10 @@
     "clean": "shx rm -rf dist",
     "terminal-server": "node server/terminal-server.mjs",
     "browser-server": "node server/browser-server.mjs",
-    "dev:all": "concurrently -k -n VITE,TERM,BROWSER -c cyan,green,magenta \"npm run dev\" \"npm run terminal-server\" \"npm run browser-server\""
+    "dev:all": "concurrently \"npm run dev\" \"npm run dev:tools\"",
+    "dev:tools": "node server/tool-server.mjs",
+    "docker:build": "docker build -t jae-web .",
+    "docker:run": "docker run -p 5173:5173 -p 7700:7700 -e OPENAI_API_KEY=$OPENAI_API_KEY jae-web"
   },
   "dependencies": {
     "@jaeswift/jae-ai": "file:../../ai",

packages/web-ui/example/server/tool-server.mjs

@@ -0,0 +1,155 @@
+import http from 'http';
+import { exec } from 'child_process';
+import { chromium } from 'playwright';
+
+const PORT = parseInt(process.env.TOOL_SERVER_PORT || '7700');
+let browser = null;
+let context = null;
+let page = null;
+
+const cors = {
+  'Access-Control-Allow-Origin': '*',
+  'Access-Control-Allow-Methods': 'POST, GET, OPTIONS',
+  'Access-Control-Allow-Headers': 'Content-Type',
+  'Content-Type': 'application/json',
+};
+
+function parseBody(req) {
+  return new Promise((resolve) => {
+    let body = '';
+    req.on('data', c => body += c);
+    req.on('end', () => {
+      try { resolve(JSON.parse(body || '{}')); } catch { resolve({}); }
+    });
+  });
+}
+
+async function launchBrowser() {
+  if (!browser) {
+    browser = await chromium.launch({
+      headless: true,
+      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
+    });
+  }
+  return browser;
+}
+
+async function getPage() {
+  if (!page || page.isClosed()) {
+    const b = await launchBrowser();
+    if (context) await context.close().catch(() => {});
+    context = await b.newContext({ viewport: { width: 1280, height: 800 } });
+    page = await context.newPage();
+  }
+  return page;
+}
+
+async function snap() {
+  const p = await getPage();
+  const buf = await p.screenshot({ type: 'jpeg', quality: 70, fullPage: false });
+  return { screenshot: buf.toString('base64'), url: p.url(), title: await p.title() };
+}
+
+async function handleBash(body) {
+  const { command, timeout = 30000 } = body;
+  if (!command) return { error: 'No command provided' };
+  return new Promise((resolve) => {
+    exec(command, {
+      timeout,
+      maxBuffer: 10 * 1024 * 1024,
+      cwd: process.env.WORKSPACE || process.env.HOME || '/root',
+      env: { ...process.env, TERM: 'dumb', COLUMNS: '200' },
+    }, (error, stdout, stderr) => {
+      resolve({
+        stdout: stdout || '',
+        stderr: stderr || '',
+        exitCode: error ? (error.code ?? 1) : 0,
+        output: (stdout || '') + (stderr ? '\nSTDERR: ' + stderr : ''),
+      });
+    });
+  });
+}
+
+async function handleNavigate(body) {
+  const { url } = body;
+  if (!url) return { error: 'No URL' };
+  const p = await getPage();
+  const target = url.startsWith('http') ? url : 'https://' + url;
+  await p.goto(target, { timeout: 30000, waitUntil: 'domcontentloaded' });
+  return snap();
+}
+
+async function handleClick(body) {
+  const p = await getPage();
+  await p.mouse.click(body.x || 0, body.y || 0);
+  await p.waitForTimeout(500);
+  return snap();
+}
+
+async function handleType(body) {
+  const p = await getPage();
+  if (body.selector) await p.fill(body.selector, body.text || '');
+  else await p.keyboard.type(body.text || '');
+  await p.waitForTimeout(300);
+  return snap();
+}
+
+async function handleScroll(body) {
+  const p = await getPage();
+  await p.mouse.wheel(0, body.dy || 300);
+  await p.waitForTimeout(300);
+  return snap();
+}
+
+async function handleBack() {
+  const p = await getPage();
+  await p.goBack({ timeout: 10000 }).catch(() => {});
+  return snap();
+}
+
+async function handleText() {
+  const p = await getPage();
+  const text = await p.evaluate(() => document.body.innerText);
+  return { url: p.url(), title: await p.title(), text: text.slice(0, 8000) };
+}
+
+async function handleEval(body) {
+  const p = await getPage();
+  const result = await p.evaluate(body.script || 'null');
+  const ss = await snap();
+  return { ...ss, evalResult: String(result) };
+}
+
+const routes = {
+  '/api/bash': handleBash,
+  '/api/browser/navigate': handleNavigate,
+  '/api/browser/click': handleClick,
+  '/api/browser/type': handleType,
+  '/api/browser/scroll': handleScroll,
+  '/api/browser/back': handleBack,
+  '/api/browser/screenshot': () => snap(),
+  '/api/browser/text': handleText,
+  '/api/browser/eval': handleEval,
+};
+
+http.createServer(async (req, res) => {
+  if (req.method === 'OPTIONS') { res.writeHead(204, cors); res.end(); return; }
+  if (req.url === '/health') { res.writeHead(200, cors); res.end(JSON.stringify({ ok: true })); return; }
+  const handler = routes[req.url];
+  if (req.method === 'POST' && handler) {
+    try {
+      const body = await parseBody(req);
+      const result = await handler(body);
+      res.writeHead(200, cors);
+      res.end(JSON.stringify(result));
+    } catch (err) {
+      res.writeHead(500, cors);
+      res.end(JSON.stringify({ error: String(err) }));
+    }
+    return;
+  }
+  res.writeHead(404, cors);
+  res.end(JSON.stringify({ error: 'Not found' }));
+}).listen(PORT, () => {
+  console.log(`[tool-server] listening on :${'PORT'}`);
+});

packages/web-ui/example/src/main.ts

@@ -22,7 +22,7 @@ import {
 import { html, render } from "lit";
 import { Brain, Download, History, Keyboard, Plus, Settings } from "lucide";
 import "./app.css";
-import { createImageGenTool, createMemoryTools, createTTSTool, createWebSearchTool } from "@jaeswift/jae-web-ui";
+import { createImageGenTool, createMemoryTools, createTTSTool, createWebSearchTool, createBashTool, createBrowserTool } from "@jaeswift/jae-web-ui";
 import { icon } from "@mariozechner/mini-lit";
 import { Button } from "@mariozechner/mini-lit/dist/Button.js";
 import { Input } from "@mariozechner/mini-lit/dist/Input.js";
@@ -351,7 +351,9 @@ IMPORTANT RULES:
 		createTools: async (runtimeProvidersFactory: any) => {
 			const replTool = createJavaScriptReplTool();
 			replTool.runtimeProvidersFactory = runtimeProvidersFactory;
-			return [replTool, createWebSearchTool(), createImageGenTool(), createTTSTool(), ...createMemoryTools()];
+			return [replTool, createWebSearchTool(),
+    createBashTool(),
+    createBrowserTool(), createImageGenTool(), createTTSTool(), ...createMemoryTools()];
 		},
 	});
 	costTracker.bindAgent(agent);

packages/web-ui/src/index.ts

@@ -106,6 +106,7 @@ export { createImageGenTool, imageGenTool } from "./tools/image-gen.js";
 // Tools
 export { getToolRenderer, registerToolRenderer, renderTool, setShowJsonMode } from "./tools/index.js";
 export { createJavaScriptReplTool, javascriptReplTool } from "./tools/javascript-repl.js";
+export { createMemoryTools, recallMemoryTool, saveMemoryTool } from "./tools/memory-tool.js";
 export { renderCollapsibleHeader, renderHeader } from "./tools/renderer-registry.js";
 export { BashRenderer } from "./tools/renderers/BashRenderer.js";
 export { CalculateRenderer } from "./tools/renderers/CalculateRenderer.js";
@@ -118,7 +119,8 @@ export type { ToolRenderer, ToolRenderResult } from "./tools/types.js";
 export { createTTSTool, ttsTool } from "./tools/voice-tts.js";
 // Venice / community tools
 export { createWebSearchTool, webSearchTool } from "./tools/web-search.js";
-export { createMemoryTools, saveMemoryTool, recallMemoryTool } from "./tools/memory-tool.js";
+export { createBashTool, bashTool } from "./tools/bash-tool.js";
+export { createBrowserTool, browserTool } from "./tools/browser-tool.js";
 export type { Attachment } from "./utils/attachment-utils.js";
 // Utils
 export { loadAttachment } from "./utils/attachment-utils.js";

packages/web-ui/src/tools/bash-tool.ts

@@ -0,0 +1,89 @@
+import type { AgentTool } from "@jaeswift/jae-agent-core";
+import type { ToolResultMessage } from "@jaeswift/jae-ai";
+import { Type } from "@sinclair/typebox";
+import { html } from "lit";
+import { Terminal } from "lucide";
+import { registerToolRenderer, renderHeader } from "./renderer-registry.js";
+import type { ToolRenderer, ToolRenderResult } from "./types.js";
+
+const TOOL_SERVER = typeof window !== "undefined"
+  ? (window as any).__JAE_TOOL_SERVER__ || "http://localhost:7700"
+  : "http://localhost:7700";
+
+const bashSchema = Type.Object({
+  command: Type.String({ description: "Shell command to execute" }),
+  timeout: Type.Optional(Type.Number({ description: "Timeout in ms (default: 30000)" })),
+});
+
+export interface BashDetails {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+  command: string;
+}
+
+export const bashTool: AgentTool<typeof bashSchema, BashDetails> = {
+  name: "bash",
+  label: "Terminal",
+  description: "Execute a shell command on the server. Use for file operations, installing packages, running scripts, git commands, etc.",
+  parameters: bashSchema,
+  async execute(toolCallId, params, signal) {
+    const { command, timeout = 30000 } = params;
+    try {
+      const res = await fetch(TOOL_SERVER + "/api/bash", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ command, timeout }),
+        signal,
+      });
+      const data = await res.json() as any;
+      if (data.error) {
+        return {
+          content: [{ type: "text" as const, text: "Error: " + data.error }],
+          details: { stdout: "", stderr: data.error, exitCode: 1, command },
+        };
+      }
+      const output = data.output || data.stdout || "";
+      return {
+        content: [{ type: "text" as const, text: output.slice(0, 10000) || "(no output)" }],
+        details: { stdout: data.stdout, stderr: data.stderr, exitCode: data.exitCode, command },
+      };
+    } catch (err: any) {
+      return {
+        content: [{ type: "text" as const, text: "Bash tool error: " + err.message }],
+        details: { stdout: "", stderr: err.message, exitCode: 1, command },
+      };
+    }
+  },
+};
+
+class BashToolRenderer implements ToolRenderer<{ command: string }, BashDetails> {
+  render(
+    params: { command: string } | undefined,
+    result: ToolResultMessage<BashDetails> | undefined,
+  ): ToolRenderResult {
+    const state = result ? (result.isError ? "error" : "complete") : "inprogress";
+    const cmd = result?.details?.command || params?.command || "...";
+    const exitCode = result?.details?.exitCode;
+    const icon = state === "error" ? "text-red-500" : exitCode === 0 ? "text-green-500" : "";
+    if (result?.details) {
+      const d = result.details;
+      return {
+        content: html`
+          <div class="flex flex-col gap-2">
+            ${renderHeader(state, Terminal, "$ " + cmd)}
+            <pre class="text-xs bg-black text-green-400 p-3 rounded overflow-auto max-h-64 whitespace-pre-wrap font-mono">${d.stdout}${d.stderr ? "\nSTDERR: " + d.stderr : ""}</pre>
+            <span class="text-xs ${icon}">Exit code: ${d.exitCode}</span>
+          </div>`,
+        isCustom: false,
+      };
+    }
+    return { content: renderHeader(state, Terminal, "$ " + cmd), isCustom: false };
+  }
+}
+
+registerToolRenderer("bash", new BashToolRenderer());
+
+export function createBashTool(): AgentTool<typeof bashSchema, BashDetails> {
+  return bashTool;
+}

packages/web-ui/src/tools/browser-tool.ts

@@ -0,0 +1,142 @@
+import type { AgentTool } from "@jaeswift/jae-agent-core";
+import type { ToolResultMessage } from "@jaeswift/jae-ai";
+import { Type } from "@sinclair/typebox";
+import { html } from "lit";
+import { Globe } from "lucide";
+import { registerToolRenderer, renderHeader } from "./renderer-registry.js";
+import type { ToolRenderer, ToolRenderResult } from "./types.js";
+
+const TOOL_SERVER = typeof window !== "undefined"
+  ? (window as any).__JAE_TOOL_SERVER__ || "http://localhost:7700"
+  : "http://localhost:7700";
+
+const browserSchema = Type.Object({
+  action: Type.Union([
+    Type.Literal("navigate"),
+    Type.Literal("click"),
+    Type.Literal("type"),
+    Type.Literal("scroll"),
+    Type.Literal("back"),
+    Type.Literal("screenshot"),
+    Type.Literal("text"),
+    Type.Literal("eval"),
+  ], { description: "Browser action to perform" }),
+  url: Type.Optional(Type.String({ description: "URL to navigate to" })),
+  x: Type.Optional(Type.Number({ description: "Click X coordinate" })),
+  y: Type.Optional(Type.Number({ description: "Click Y coordinate" })),
+  text: Type.Optional(Type.String({ description: "Text to type" })),
+  selector: Type.Optional(Type.String({ description: "CSS selector to type into" })),
+  dy: Type.Optional(Type.Number({ description: "Scroll delta Y pixels" })),
+  script: Type.Optional(Type.String({ description: "JavaScript to evaluate in page" })),
+});
+
+export interface BrowserDetails {
+  action: string;
+  url?: string;
+  title?: string;
+  screenshot?: string;
+  text?: string;
+  evalResult?: string;
+  error?: string;
+}
+
+export const browserTool: AgentTool<typeof browserSchema, BrowserDetails> = {
+  name: "browser",
+  label: "Browser",
+  description: "Control a headless browser. Actions: navigate (url), click (x,y), type (text, optional selector), scroll (dy), back, screenshot, text (get page text), eval (run JS).",
+  parameters: browserSchema,
+  async execute(toolCallId, params, signal) {
+    const { action, ...rest } = params;
+    const endpoint = action === "navigate" ? "/api/browser/navigate"
+      : action === "click" ? "/api/browser/click"
+      : action === "type" ? "/api/browser/type"
+      : action === "scroll" ? "/api/browser/scroll"
+      : action === "back" ? "/api/browser/back"
+      : action === "screenshot" ? "/api/browser/screenshot"
+      : action === "text" ? "/api/browser/text"
+      : action === "eval" ? "/api/browser/eval"
+      : null;
+    if (!endpoint) {
+      return {
+        content: [{ type: "text" as const, text: "Unknown action: " + action }],
+        details: { action, error: "Unknown action" },
+      };
+    }
+    try {
+      const res = await fetch(TOOL_SERVER + endpoint, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(rest),
+        signal,
+      });
+      const data = await res.json() as any;
+      if (data.error) {
+        return {
+          content: [{ type: "text" as const, text: "Browser error: " + data.error }],
+          details: { action, error: data.error },
+        };
+      }
+      // Build text response for LLM
+      let textParts: string[] = [];
+      if (data.url) textParts.push("URL: " + data.url);
+      if (data.title) textParts.push("Title: " + data.title);
+      if (data.text) textParts.push("Page text:\n" + data.text);
+      if (data.evalResult) textParts.push("Eval result: " + data.evalResult);
+      if (data.screenshot) textParts.push("[Screenshot captured]");
+      if (textParts.length === 0) textParts.push("Action completed.");
+      // Include screenshot as image content if available
+      const content: any[] = [{ type: "text" as const, text: textParts.join("\n") }];
+      if (data.screenshot) {
+        content.push({ type: "image" as const, mimeType: "image/jpeg", data: data.screenshot });
+      }
+      return {
+        content,
+        details: { action, url: data.url, title: data.title, screenshot: data.screenshot, text: data.text, evalResult: data.evalResult },
+      };
+    } catch (err: any) {
+      return {
+        content: [{ type: "text" as const, text: "Browser tool error: " + err.message }],
+        details: { action, error: err.message },
+      };
+    }
+  },
+};
+
+class BrowserToolRenderer implements ToolRenderer<any, BrowserDetails> {
+  render(params: any | undefined, result: ToolResultMessage<BrowserDetails> | undefined): ToolRenderResult {
+    const state = result ? (result.isError ? "error" : "complete") : "inprogress";
+    const action = result?.details?.action || params?.action || "...";
+    const url = result?.details?.url || params?.url || "";
+    const label = url ? action + ": " + url : action;
+    if (result?.details?.screenshot) {
+      return {
+        content: html`
+          <div class="flex flex-col gap-2">
+            ${renderHeader(state, Globe, label)}
+            <img src="data:image/jpeg;base64,${result.details.screenshot}"
+                 class="rounded border border-border max-w-full" style="max-height:400px"
+                 alt="Browser screenshot" />
+            ${result.details.title ? html`<span class="text-xs text-muted-foreground">${result.details.title}</span>` : html``}
+          </div>`,
+        isCustom: false,
+      };
+    }
+    if (result?.details?.text) {
+      return {
+        content: html`
+          <div class="flex flex-col gap-2">
+            ${renderHeader(state, Globe, label)}
+            <pre class="text-xs p-3 rounded border border-border overflow-auto max-h-48 whitespace-pre-wrap">${result.details.text}</pre>
+          </div>`,
+        isCustom: false,
+      };
+    }
+    return { content: renderHeader(state, Globe, label), isCustom: false };
+  }
+}
+
+registerToolRenderer("browser", new BrowserToolRenderer());
+
+export function createBrowserTool(): AgentTool<typeof browserSchema, BrowserDetails> {
+  return browserTool;
+}

packages/web-ui/src/tools/index.ts

@@ -2,6 +2,8 @@ import type { ToolResultMessage } from "@jaeswift/jae-ai";
 import "./javascript-repl.js"; // Auto-registers the renderer
 import "./extract-document.js";
 import "./memory-tool.js"; // Auto-registers the renderer
+import "./bash-tool.js"; // Auto-registers bash tool renderer
+import "./browser-tool.js"; // Auto-registers browser tool renderer
 import { getToolRenderer, registerToolRenderer } from "./renderer-registry.js";
 import { BashRenderer } from "./renderers/BashRenderer.js";
 import { DefaultRenderer } from "./renderers/DefaultRenderer.js";
@@ -47,6 +49,9 @@ export function renderTool(
 export { getToolRenderer, registerToolRenderer };
 
 export { createImageGenTool, type ImageGenDetails, imageGenTool } from "./image-gen.js";
+export { createMemoryTools, recallMemoryTool, saveMemoryTool } from "./memory-tool.js";
 export { createTTSTool, type TTSDetails, ttsTool } from "./voice-tts.js";
 export { createWebSearchTool, type WebSearchDetails, type WebSearchResult, webSearchTool } from "./web-search.js";
-export { createMemoryTools, saveMemoryTool, recallMemoryTool } from "./memory-tool.js";
+
+export { createBashTool, type BashDetails, bashTool } from "./bash-tool.js";
+export { createBrowserTool, type BrowserDetails, browserTool } from "./browser-tool.js";