diff --git a/packages/web-ui/example/Dockerfile b/packages/web-ui/example/Dockerfile new file mode 100644 index 0000000..e843458 --- /dev/null +++ b/packages/web-ui/example/Dockerfile @@ -0,0 +1,29 @@ +FROM node:20-slim + +ENV DEBIAN_FRONTEND=noninteractive +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl ca-certificates \ + libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 \ + libdrm2 libdbus-1-3 libxkbcommon0 libatspi2.0-0 \ + libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libpango-1.0-0 \ + libasound2 libwayland-client0 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY package*.json ./ +RUN npm ci --production + +RUN npx playwright install chromium + +COPY . . + +RUN npm run build 2>/dev/null || true + +EXPOSE 5173 7700 + +ENV TOOL_SERVER_PORT=7700 + +CMD ["sh", "-c", "node server/tool-server.mjs & npx vite --host 0.0.0.0 --port 5173"] diff --git a/packages/web-ui/example/docker-compose.yml b/packages/web-ui/example/docker-compose.yml new file mode 100644 index 0000000..de0b547 --- /dev/null +++ b/packages/web-ui/example/docker-compose.yml @@ -0,0 +1,16 @@ +services: + jae-web: + build: . + ports: + - "5173:5173" + - "7700:7700" + environment: + - OPENAI_API_KEY=${OPENAI_API_KEY} + - OPENAI_BASE_URL=${OPENAI_BASE_URL:-https://api.venice.ai/api/v1} + - TOOL_SERVER_PORT=7700 + volumes: + - workspace:/workspace + restart: unless-stopped + +volumes: + workspace: diff --git a/packages/web-ui/example/package.json b/packages/web-ui/example/package.json index 1bc0de7..421c1d2 100644 --- a/packages/web-ui/example/package.json +++ b/packages/web-ui/example/package.json @@ -11,7 +11,10 @@ "clean": "shx rm -rf dist", "terminal-server": "node server/terminal-server.mjs", "browser-server": "node server/browser-server.mjs", - "dev:all": "concurrently -k -n VITE,TERM,BROWSER -c cyan,green,magenta \"npm run dev\" \"npm run terminal-server\" \"npm run browser-server\"" + "dev:all": "concurrently \"npm run dev\" \"npm run dev:tools\"", + "dev:tools": "node server/tool-server.mjs", + "docker:build": "docker build -t jae-web .", + "docker:run": "docker run -p 5173:5173 -p 7700:7700 -e OPENAI_API_KEY=$OPENAI_API_KEY jae-web" }, "dependencies": { "@jaeswift/jae-ai": "file:../../ai", diff --git a/packages/web-ui/example/server/tool-server.mjs b/packages/web-ui/example/server/tool-server.mjs new file mode 100644 index 0000000..db9a86a --- /dev/null +++ b/packages/web-ui/example/server/tool-server.mjs @@ -0,0 +1,155 @@ +import http from 'http'; +import { exec } from 'child_process'; +import { chromium } from 'playwright'; + +const PORT = parseInt(process.env.TOOL_SERVER_PORT || '7700'); +let browser = null; +let context = null; +let page = null; + +const cors = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'POST, GET, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type', + 'Content-Type': 'application/json', +}; + +function parseBody(req) { + return new Promise((resolve) => { + let body = ''; + req.on('data', c => body += c); + req.on('end', () => { + try { resolve(JSON.parse(body || '{}')); } catch { resolve({}); } + }); + }); +} + +async function launchBrowser() { + if (!browser) { + browser = await chromium.launch({ + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'] + }); + } + return browser; +} + +async function getPage() { + if (!page || page.isClosed()) { + const b = await launchBrowser(); + if (context) await context.close().catch(() => {}); + context = await b.newContext({ viewport: { width: 1280, height: 800 } }); + page = await context.newPage(); + } + return page; +} + +async function snap() { + const p = await getPage(); + const buf = await p.screenshot({ type: 'jpeg', quality: 70, fullPage: false }); + return { screenshot: buf.toString('base64'), url: p.url(), title: await p.title() }; +} + +async function handleBash(body) { + const { command, timeout = 30000 } = body; + if (!command) return { error: 'No command provided' }; + return new Promise((resolve) => { + exec(command, { + timeout, + maxBuffer: 10 * 1024 * 1024, + cwd: process.env.WORKSPACE || process.env.HOME || '/root', + env: { ...process.env, TERM: 'dumb', COLUMNS: '200' }, + }, (error, stdout, stderr) => { + resolve({ + stdout: stdout || '', + stderr: stderr || '', + exitCode: error ? (error.code ?? 1) : 0, + output: (stdout || '') + (stderr ? '\nSTDERR: ' + stderr : ''), + }); + }); + }); +} + +async function handleNavigate(body) { + const { url } = body; + if (!url) return { error: 'No URL' }; + const p = await getPage(); + const target = url.startsWith('http') ? url : 'https://' + url; + await p.goto(target, { timeout: 30000, waitUntil: 'domcontentloaded' }); + return snap(); +} + +async function handleClick(body) { + const p = await getPage(); + await p.mouse.click(body.x || 0, body.y || 0); + await p.waitForTimeout(500); + return snap(); +} + +async function handleType(body) { + const p = await getPage(); + if (body.selector) await p.fill(body.selector, body.text || ''); + else await p.keyboard.type(body.text || ''); + await p.waitForTimeout(300); + return snap(); +} + +async function handleScroll(body) { + const p = await getPage(); + await p.mouse.wheel(0, body.dy || 300); + await p.waitForTimeout(300); + return snap(); +} + +async function handleBack() { + const p = await getPage(); + await p.goBack({ timeout: 10000 }).catch(() => {}); + return snap(); +} + +async function handleText() { + const p = await getPage(); + const text = await p.evaluate(() => document.body.innerText); + return { url: p.url(), title: await p.title(), text: text.slice(0, 8000) }; +} + +async function handleEval(body) { + const p = await getPage(); + const result = await p.evaluate(body.script || 'null'); + const ss = await snap(); + return { ...ss, evalResult: String(result) }; +} + +const routes = { + '/api/bash': handleBash, + '/api/browser/navigate': handleNavigate, + '/api/browser/click': handleClick, + '/api/browser/type': handleType, + '/api/browser/scroll': handleScroll, + '/api/browser/back': handleBack, + '/api/browser/screenshot': () => snap(), + '/api/browser/text': handleText, + '/api/browser/eval': handleEval, +}; + +http.createServer(async (req, res) => { + if (req.method === 'OPTIONS') { res.writeHead(204, cors); res.end(); return; } + if (req.url === '/health') { res.writeHead(200, cors); res.end(JSON.stringify({ ok: true })); return; } + const handler = routes[req.url]; + if (req.method === 'POST' && handler) { + try { + const body = await parseBody(req); + const result = await handler(body); + res.writeHead(200, cors); + res.end(JSON.stringify(result)); + } catch (err) { + res.writeHead(500, cors); + res.end(JSON.stringify({ error: String(err) })); + } + return; + } + res.writeHead(404, cors); + res.end(JSON.stringify({ error: 'Not found' })); +}).listen(PORT, () => { + console.log(`[tool-server] listening on :${'PORT'}`); +}); diff --git a/packages/web-ui/example/src/main.ts b/packages/web-ui/example/src/main.ts index 29b3997..bc91a05 100644 --- a/packages/web-ui/example/src/main.ts +++ b/packages/web-ui/example/src/main.ts @@ -22,7 +22,7 @@ import { import { html, render } from "lit"; import { Brain, Download, History, Keyboard, Plus, Settings } from "lucide"; import "./app.css"; -import { createImageGenTool, createMemoryTools, createTTSTool, createWebSearchTool } from "@jaeswift/jae-web-ui"; +import { createImageGenTool, createMemoryTools, createTTSTool, createWebSearchTool, createBashTool, createBrowserTool } from "@jaeswift/jae-web-ui"; import { icon } from "@mariozechner/mini-lit"; import { Button } from "@mariozechner/mini-lit/dist/Button.js"; import { Input } from "@mariozechner/mini-lit/dist/Input.js"; @@ -351,7 +351,9 @@ IMPORTANT RULES: createTools: async (runtimeProvidersFactory: any) => { const replTool = createJavaScriptReplTool(); replTool.runtimeProvidersFactory = runtimeProvidersFactory; - return [replTool, createWebSearchTool(), createImageGenTool(), createTTSTool(), ...createMemoryTools()]; + return [replTool, createWebSearchTool(), + createBashTool(), + createBrowserTool(), createImageGenTool(), createTTSTool(), ...createMemoryTools()]; }, }); costTracker.bindAgent(agent); diff --git a/packages/web-ui/src/index.ts b/packages/web-ui/src/index.ts index b3c18b3..0ffd49d 100644 --- a/packages/web-ui/src/index.ts +++ b/packages/web-ui/src/index.ts @@ -106,6 +106,7 @@ export { createImageGenTool, imageGenTool } from "./tools/image-gen.js"; // Tools export { getToolRenderer, registerToolRenderer, renderTool, setShowJsonMode } from "./tools/index.js"; export { createJavaScriptReplTool, javascriptReplTool } from "./tools/javascript-repl.js"; +export { createMemoryTools, recallMemoryTool, saveMemoryTool } from "./tools/memory-tool.js"; export { renderCollapsibleHeader, renderHeader } from "./tools/renderer-registry.js"; export { BashRenderer } from "./tools/renderers/BashRenderer.js"; export { CalculateRenderer } from "./tools/renderers/CalculateRenderer.js"; @@ -118,7 +119,8 @@ export type { ToolRenderer, ToolRenderResult } from "./tools/types.js"; export { createTTSTool, ttsTool } from "./tools/voice-tts.js"; // Venice / community tools export { createWebSearchTool, webSearchTool } from "./tools/web-search.js"; -export { createMemoryTools, saveMemoryTool, recallMemoryTool } from "./tools/memory-tool.js"; +export { createBashTool, bashTool } from "./tools/bash-tool.js"; +export { createBrowserTool, browserTool } from "./tools/browser-tool.js"; export type { Attachment } from "./utils/attachment-utils.js"; // Utils export { loadAttachment } from "./utils/attachment-utils.js"; diff --git a/packages/web-ui/src/tools/bash-tool.ts b/packages/web-ui/src/tools/bash-tool.ts new file mode 100644 index 0000000..f1c8d46 --- /dev/null +++ b/packages/web-ui/src/tools/bash-tool.ts @@ -0,0 +1,89 @@ +import type { AgentTool } from "@jaeswift/jae-agent-core"; +import type { ToolResultMessage } from "@jaeswift/jae-ai"; +import { Type } from "@sinclair/typebox"; +import { html } from "lit"; +import { Terminal } from "lucide"; +import { registerToolRenderer, renderHeader } from "./renderer-registry.js"; +import type { ToolRenderer, ToolRenderResult } from "./types.js"; + +const TOOL_SERVER = typeof window !== "undefined" + ? (window as any).__JAE_TOOL_SERVER__ || "http://localhost:7700" + : "http://localhost:7700"; + +const bashSchema = Type.Object({ + command: Type.String({ description: "Shell command to execute" }), + timeout: Type.Optional(Type.Number({ description: "Timeout in ms (default: 30000)" })), +}); + +export interface BashDetails { + stdout: string; + stderr: string; + exitCode: number; + command: string; +} + +export const bashTool: AgentTool = { + name: "bash", + label: "Terminal", + description: "Execute a shell command on the server. Use for file operations, installing packages, running scripts, git commands, etc.", + parameters: bashSchema, + async execute(toolCallId, params, signal) { + const { command, timeout = 30000 } = params; + try { + const res = await fetch(TOOL_SERVER + "/api/bash", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ command, timeout }), + signal, + }); + const data = await res.json() as any; + if (data.error) { + return { + content: [{ type: "text" as const, text: "Error: " + data.error }], + details: { stdout: "", stderr: data.error, exitCode: 1, command }, + }; + } + const output = data.output || data.stdout || ""; + return { + content: [{ type: "text" as const, text: output.slice(0, 10000) || "(no output)" }], + details: { stdout: data.stdout, stderr: data.stderr, exitCode: data.exitCode, command }, + }; + } catch (err: any) { + return { + content: [{ type: "text" as const, text: "Bash tool error: " + err.message }], + details: { stdout: "", stderr: err.message, exitCode: 1, command }, + }; + } + }, +}; + +class BashToolRenderer implements ToolRenderer<{ command: string }, BashDetails> { + render( + params: { command: string } | undefined, + result: ToolResultMessage | undefined, + ): ToolRenderResult { + const state = result ? (result.isError ? "error" : "complete") : "inprogress"; + const cmd = result?.details?.command || params?.command || "..."; + const exitCode = result?.details?.exitCode; + const icon = state === "error" ? "text-red-500" : exitCode === 0 ? "text-green-500" : ""; + if (result?.details) { + const d = result.details; + return { + content: html` +
+ ${renderHeader(state, Terminal, "$ " + cmd)} + 
${d.stdout}${d.stderr ? "\nSTDERR: " + d.stderr : ""}
+ Exit code: ${d.exitCode} +
`, + isCustom: false, + }; + } + return { content: renderHeader(state, Terminal, "$ " + cmd), isCustom: false }; + } +} + +registerToolRenderer("bash", new BashToolRenderer()); + +export function createBashTool(): AgentTool { + return bashTool; +} diff --git a/packages/web-ui/src/tools/browser-tool.ts b/packages/web-ui/src/tools/browser-tool.ts new file mode 100644 index 0000000..51c4f6e --- /dev/null +++ b/packages/web-ui/src/tools/browser-tool.ts @@ -0,0 +1,142 @@ +import type { AgentTool } from "@jaeswift/jae-agent-core"; +import type { ToolResultMessage } from "@jaeswift/jae-ai"; +import { Type } from "@sinclair/typebox"; +import { html } from "lit"; +import { Globe } from "lucide"; +import { registerToolRenderer, renderHeader } from "./renderer-registry.js"; +import type { ToolRenderer, ToolRenderResult } from "./types.js"; + +const TOOL_SERVER = typeof window !== "undefined" + ? (window as any).__JAE_TOOL_SERVER__ || "http://localhost:7700" + : "http://localhost:7700"; + +const browserSchema = Type.Object({ + action: Type.Union([ + Type.Literal("navigate"), + Type.Literal("click"), + Type.Literal("type"), + Type.Literal("scroll"), + Type.Literal("back"), + Type.Literal("screenshot"), + Type.Literal("text"), + Type.Literal("eval"), + ], { description: "Browser action to perform" }), + url: Type.Optional(Type.String({ description: "URL to navigate to" })), + x: Type.Optional(Type.Number({ description: "Click X coordinate" })), + y: Type.Optional(Type.Number({ description: "Click Y coordinate" })), + text: Type.Optional(Type.String({ description: "Text to type" })), + selector: Type.Optional(Type.String({ description: "CSS selector to type into" })), + dy: Type.Optional(Type.Number({ description: "Scroll delta Y pixels" })), + script: Type.Optional(Type.String({ description: "JavaScript to evaluate in page" })), +}); + +export interface BrowserDetails { + action: string; + url?: string; + title?: string; + screenshot?: string; + text?: string; + evalResult?: string; + error?: string; +} + +export const browserTool: AgentTool = { + name: "browser", + label: "Browser", + description: "Control a headless browser. Actions: navigate (url), click (x,y), type (text, optional selector), scroll (dy), back, screenshot, text (get page text), eval (run JS).", + parameters: browserSchema, + async execute(toolCallId, params, signal) { + const { action, ...rest } = params; + const endpoint = action === "navigate" ? "/api/browser/navigate" + : action === "click" ? "/api/browser/click" + : action === "type" ? "/api/browser/type" + : action === "scroll" ? "/api/browser/scroll" + : action === "back" ? "/api/browser/back" + : action === "screenshot" ? "/api/browser/screenshot" + : action === "text" ? "/api/browser/text" + : action === "eval" ? "/api/browser/eval" + : null; + if (!endpoint) { + return { + content: [{ type: "text" as const, text: "Unknown action: " + action }], + details: { action, error: "Unknown action" }, + }; + } + try { + const res = await fetch(TOOL_SERVER + endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(rest), + signal, + }); + const data = await res.json() as any; + if (data.error) { + return { + content: [{ type: "text" as const, text: "Browser error: " + data.error }], + details: { action, error: data.error }, + }; + } + // Build text response for LLM + let textParts: string[] = []; + if (data.url) textParts.push("URL: " + data.url); + if (data.title) textParts.push("Title: " + data.title); + if (data.text) textParts.push("Page text:\n" + data.text); + if (data.evalResult) textParts.push("Eval result: " + data.evalResult); + if (data.screenshot) textParts.push("[Screenshot captured]"); + if (textParts.length === 0) textParts.push("Action completed."); + // Include screenshot as image content if available + const content: any[] = [{ type: "text" as const, text: textParts.join("\n") }]; + if (data.screenshot) { + content.push({ type: "image" as const, mimeType: "image/jpeg", data: data.screenshot }); + } + return { + content, + details: { action, url: data.url, title: data.title, screenshot: data.screenshot, text: data.text, evalResult: data.evalResult }, + }; + } catch (err: any) { + return { + content: [{ type: "text" as const, text: "Browser tool error: " + err.message }], + details: { action, error: err.message }, + }; + } + }, +}; + +class BrowserToolRenderer implements ToolRenderer { + render(params: any | undefined, result: ToolResultMessage | undefined): ToolRenderResult { + const state = result ? (result.isError ? "error" : "complete") : "inprogress"; + const action = result?.details?.action || params?.action || "..."; + const url = result?.details?.url || params?.url || ""; + const label = url ? action + ": " + url : action; + if (result?.details?.screenshot) { + return { + content: html` +
+ ${renderHeader(state, Globe, label)} + Browser screenshot + ${result.details.title ? html`${result.details.title}` : html``} +
`, + isCustom: false, + }; + } + if (result?.details?.text) { + return { + content: html` +
+ ${renderHeader(state, Globe, label)} + 
${result.details.text}
+
`, + isCustom: false, + }; + } + return { content: renderHeader(state, Globe, label), isCustom: false }; + } +} + +registerToolRenderer("browser", new BrowserToolRenderer()); + +export function createBrowserTool(): AgentTool { + return browserTool; +} diff --git a/packages/web-ui/src/tools/index.ts b/packages/web-ui/src/tools/index.ts index 8cb94bd..aa4049a 100644 --- a/packages/web-ui/src/tools/index.ts +++ b/packages/web-ui/src/tools/index.ts @@ -2,6 +2,8 @@ import type { ToolResultMessage } from "@jaeswift/jae-ai"; import "./javascript-repl.js"; // Auto-registers the renderer import "./extract-document.js"; import "./memory-tool.js"; // Auto-registers the renderer +import "./bash-tool.js"; // Auto-registers bash tool renderer +import "./browser-tool.js"; // Auto-registers browser tool renderer import { getToolRenderer, registerToolRenderer } from "./renderer-registry.js"; import { BashRenderer } from "./renderers/BashRenderer.js"; import { DefaultRenderer } from "./renderers/DefaultRenderer.js"; @@ -47,6 +49,9 @@ export function renderTool( export { getToolRenderer, registerToolRenderer }; export { createImageGenTool, type ImageGenDetails, imageGenTool } from "./image-gen.js"; +export { createMemoryTools, recallMemoryTool, saveMemoryTool } from "./memory-tool.js"; export { createTTSTool, type TTSDetails, ttsTool } from "./voice-tts.js"; export { createWebSearchTool, type WebSearchDetails, type WebSearchResult, webSearchTool } from "./web-search.js"; -export { createMemoryTools, saveMemoryTool, recallMemoryTool } from "./memory-tool.js"; + +export { createBashTool, type BashDetails, bashTool } from "./bash-tool.js"; +export { createBrowserTool, type BrowserDetails, browserTool } from "./browser-tool.js";