diff --git a/package-lock.json b/package-lock.json index 9f99d2f..6fdfaf9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4524,6 +4524,16 @@ "alien-signals": "^2.0.6" } }, + "node_modules/@xterm/addon-fit": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@xterm/addon-fit/-/addon-fit-0.11.0.tgz", + "integrity": "sha512-jYcgT6xtVYhnhgxh3QgYDnnNMYTcf8ElbxxFzX0IZo+vabQqSPAjC3c1wJrKB5E19VwQei89QCiZZP86DCPF7g==" + }, + "node_modules/@xterm/addon-web-links": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@xterm/addon-web-links/-/addon-web-links-0.12.0.tgz", + "integrity": "sha512-4Smom3RPyVp7ZMYOYDoC/9eGJJJqYhnPLGGqJ6wOBfB8VxPViJNSKdgRYb8NpaM6YSelEKbA2SStD7lGyqaobw==" + }, "node_modules/@xterm/headless": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/@xterm/headless/-/headless-5.5.0.tgz", @@ -8012,6 +8022,47 @@ "pathe": "^2.0.1" } }, + "node_modules/playwright": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", + "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "dependencies": { + "playwright-core": "1.58.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", + "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/points-on-curve": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", @@ -9950,15 +10001,24 @@ "@jaeswift/jae-web-ui": "file:../", "@mariozechner/mini-lit": "^0.2.0", "@tailwindcss/vite": "^4.1.17", + "@xterm/addon-fit": "^0.11.0", + "@xterm/addon-web-links": "^0.12.0", + "@xterm/xterm": "^6.0.0", "diff2html": "^3.4.56", "lit": "^3.3.1", "lucide": "^0.544.0", - "mermaid": "^11.13.0" + "mermaid": "^11.13.0", + "playwright": "^1.58.2" }, "devDependencies": { "typescript": "^5.7.3", "vite": "^7.1.6" } + }, + "packages/web-ui/node_modules/@xterm/xterm": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-6.0.0.tgz", + "integrity": "sha512-TQwDdQGtwwDt+2cgKDLn0IRaSxYu1tSUjgKarSDkUM0ZNiSRXFpjxEsvc/Zgc5kq5omJ+V0a8/kIM2WD3sMOYg==" } } } diff --git a/packages/web-ui/example/package.json b/packages/web-ui/example/package.json index 30fdaca..5ca4e2b 100644 --- a/packages/web-ui/example/package.json +++ b/packages/web-ui/example/package.json @@ -8,20 +8,27 @@ "build": "vite build", "preview": "vite preview", "check": "tsgo --noEmit", - "clean": "shx rm -rf dist" + "clean": "shx rm -rf dist", + "terminal-server": "node server/terminal-server.mjs", + "browser-server": "node server/browser-server.mjs", + "dev:all": "concurrently -k -n VITE,TERM,BROWSER -c cyan,green,magenta \"npm run dev\" \"npm run terminal-server\" \"npm run browser-server\"" }, "dependencies": { "@jaeswift/jae-ai": "file:../../ai", "@jaeswift/jae-web-ui": "file:../", "@mariozechner/mini-lit": "^0.2.0", "@tailwindcss/vite": "^4.1.17", + "@xterm/addon-fit": "^0.11.0", + "@xterm/addon-web-links": "^0.12.0", + "@xterm/xterm": "^6.0.0", "diff2html": "^3.4.56", "lit": "^3.3.1", "lucide": "^0.544.0", - "mermaid": "^11.13.0" + "mermaid": "^11.13.0", + "playwright": "^1.58.2" }, "devDependencies": { "typescript": "^5.7.3", "vite": "^7.1.6" } -} +} \ No newline at end of file diff --git a/packages/web-ui/example/server/browser-server.mjs b/packages/web-ui/example/server/browser-server.mjs new file mode 100644 index 0000000..5d96f98 --- /dev/null +++ b/packages/web-ui/example/server/browser-server.mjs @@ -0,0 +1,65 @@ +import { WebSocketServer } from 'ws'; +import { chromium } from 'playwright'; + +const PORT = 7702; +const wss = new WebSocketServer({ port: PORT }); +console.log(`Browser WS server on ws://localhost:${PORT}`); + +let browser = null; + +async function getBrowser() { + if (!browser) browser = await chromium.launch({ headless: true, args: ['--no-sandbox','--disable-setuid-sandbox'] }); + return browser; +} + +wss.on('connection', async (ws) => { + let context = null; + let page = null; + + async function screenshot() { + if (!page) return; + try { + const buf = await page.screenshot({ type: 'jpeg', quality: 70, fullPage: false }); + ws.send(JSON.stringify({ type: 'screenshot', data: buf.toString('base64'), url: page.url() })); + } catch(e) { ws.send(JSON.stringify({ type: 'error', msg: String(e) })); } + } + + async function navigate(url) { + try { + if (!context) { + const b = await getBrowser(); + context = await b.newContext({ viewport: { width: 1280, height: 800 } }); + page = await context.newPage(); + } + if (!url.startsWith('http')) url = 'https://' + url; + ws.send(JSON.stringify({ type: 'loading' })); + await page.goto(url, { timeout: 30000, waitUntil: 'domcontentloaded' }); + await screenshot(); + } catch(e) { ws.send(JSON.stringify({ type: 'error', msg: String(e) })); } + } + + ws.on('message', async (msg) => { + try { + const m = JSON.parse(msg.toString()); + if (m.type === 'navigate') await navigate(m.url); + if (m.type === 'screenshot') await screenshot(); + if (m.type === 'click') { + if (page) { await page.mouse.click(m.x, m.y); await screenshot(); } + } + if (m.type === 'scroll') { + if (page) { await page.mouse.wheel(0, m.dy); await screenshot(); } + } + if (m.type === 'type') { + if (page) { await page.keyboard.type(m.text); await screenshot(); } + } + if (m.type === 'back') { if (page) { await page.goBack(); await screenshot(); } } + if (m.type === 'fwd') { if (page) { await page.goForward(); await screenshot(); } } + if (m.type === 'reload'){ if (page) { await page.reload(); await screenshot(); } } + } catch(e) { ws.send(JSON.stringify({ type: 'error', msg: String(e) })); } + }); + + ws.on('close', async () => { if (context) await context.close().catch(()=>{}); context = null; page = null; }); + + // Send welcome screenshot placeholder + ws.send(JSON.stringify({ type: 'ready' })); +}); diff --git a/packages/web-ui/example/server/terminal-server.mjs b/packages/web-ui/example/server/terminal-server.mjs new file mode 100644 index 0000000..95dda1e --- /dev/null +++ b/packages/web-ui/example/server/terminal-server.mjs @@ -0,0 +1,27 @@ +import { WebSocketServer } from 'ws'; +import { spawn } from 'child_process'; + +const PORT = 7701; +const wss = new WebSocketServer({ port: PORT }); +console.log(`Terminal WS server on ws://localhost:${PORT}`); + +wss.on('connection', (ws) => { + const shell = spawn('/bin/bash', [], { + env: { ...process.env, TERM: 'xterm-256color', COLORTERM: 'truecolor' }, + cwd: process.env.HOME || '/root', + }); + + shell.stdout.on('data', (d) => { try { ws.send(JSON.stringify({ type:'data', data: d.toString('binary') })); } catch{} }); + shell.stderr.on('data', (d) => { try { ws.send(JSON.stringify({ type:'data', data: d.toString('binary') })); } catch{} }); + shell.on('close', (code) => { try { ws.send(JSON.stringify({ type:'exit', code })); ws.close(); } catch{} }); + + ws.on('message', (msg) => { + try { + const m = JSON.parse(msg.toString()); + if (m.type === 'input') shell.stdin.write(m.data); + if (m.type === 'resize') { /* no node-pty resize without pty, best effort */ } + } catch{} + }); + + ws.on('close', () => { shell.kill(); }); +}); diff --git a/packages/web-ui/example/src/components/browser-panel.ts b/packages/web-ui/example/src/components/browser-panel.ts new file mode 100644 index 0000000..f789157 --- /dev/null +++ b/packages/web-ui/example/src/components/browser-panel.ts @@ -0,0 +1,126 @@ +import { LitElement, html } from 'lit'; +import { customElement, state } from 'lit/decorators.js'; + +@customElement('jae-browser-panel') +export class JaeBrowserPanel extends LitElement { + @state() private url = ''; + @state() private inputUrl = ''; + @state() private screenshot = ''; + @state() private loading = false; + @state() private connected = false; + @state() private error = ''; + + private ws: WebSocket | null = null; + private imgEl: HTMLImageElement | null = null; + + createRenderRoot() { return this; } + + override connectedCallback() { + super.connectedCallback(); + this.style.display = 'flex'; + this.style.flexDirection = 'column'; + this.style.height = '100%'; + this.style.minHeight = '0'; + this.connect(); + } + + override disconnectedCallback() { + super.disconnectedCallback(); + this.ws?.close(); + } + + connect() { + this.ws = new WebSocket('ws://localhost:7702'); + this.ws.onopen = () => { this.connected = true; this.requestUpdate(); }; + this.ws.onclose = () => { this.connected = false; this.requestUpdate(); }; + this.ws.onerror = () => { this.connected = false; this.error = 'Browser server not running.'; this.requestUpdate(); }; + this.ws.onmessage = (e) => { + const m = JSON.parse(e.data); + if (m.type === 'screenshot') { + this.loading = false; + this.screenshot = `data:image/jpeg;base64,${m.data}`; + this.url = m.url; + this.inputUrl = m.url; + this.error = ''; + this.requestUpdate(); + } + if (m.type === 'loading') { this.loading = true; this.requestUpdate(); } + if (m.type === 'error') { this.loading = false; this.error = m.msg; this.requestUpdate(); } + }; + } + + navigate(url?: string) { + const target = url || this.inputUrl; + if (!target) return; + this.ws?.send(JSON.stringify({ type: 'navigate', url: target })); + this.loading = true; + this.requestUpdate(); + } + + private handleImgClick(e: MouseEvent) { + const img = e.currentTarget as HTMLImageElement; + const rect = img.getBoundingClientRect(); + const scaleX = 1280 / rect.width; + const scaleY = 800 / rect.height; + const x = (e.clientX - rect.left) * scaleX; + const y = (e.clientY - rect.top) * scaleY; + this.ws?.send(JSON.stringify({ type: 'click', x, y })); + this.loading = true; + this.requestUpdate(); + } + + private handleScroll(e: WheelEvent) { + e.preventDefault(); + this.ws?.send(JSON.stringify({ type: 'scroll', dy: e.deltaY })); + } + + override render() { + return html` +
+ + + + { this.inputUrl = (e.target as HTMLInputElement).value; }} + @keydown=${(e: KeyboardEvent) => { if (e.key === 'Enter') this.navigate(); }} + placeholder="Enter URL and press Enter..." + /> +
+
+
+ ${this.loading ? html` +
+
Loading...
+
+ ` : html``} + ${this.error ? html` +
${this.error}
+ ` : html``} + ${this.screenshot ? html` + browser + ` : !this.error ? html` +
+ +

Enter a URL above to browse

+
+ ` : html``} +
+ `; + } +} diff --git a/packages/web-ui/example/src/components/terminal-panel.ts b/packages/web-ui/example/src/components/terminal-panel.ts new file mode 100644 index 0000000..22b32db --- /dev/null +++ b/packages/web-ui/example/src/components/terminal-panel.ts @@ -0,0 +1,115 @@ +import { LitElement, html } from 'lit'; +import { customElement, state } from 'lit/decorators.js'; +import { Terminal } from '@xterm/xterm'; +import { FitAddon } from '@xterm/addon-fit'; +import { WebLinksAddon } from '@xterm/addon-web-links'; +import '@xterm/xterm/css/xterm.css'; + +@customElement('jae-terminal-panel') +export class JaeTerminalPanel extends LitElement { + @state() private connected = false; + @state() private connecting = false; + + private term: Terminal | null = null; + private fitAddon: FitAddon | null = null; + private ws: WebSocket | null = null; + private container: HTMLElement | null = null; + private resizeObs: ResizeObserver | null = null; + + createRenderRoot() { return this; } + + override connectedCallback() { + super.connectedCallback(); + this.style.display = 'flex'; + this.style.flexDirection = 'column'; + this.style.height = '100%'; + this.style.minHeight = '0'; + } + + override disconnectedCallback() { + super.disconnectedCallback(); + this.destroyTerminal(); + } + + private destroyTerminal() { + this.resizeObs?.disconnect(); + this.ws?.close(); + this.term?.dispose(); + this.term = null; this.ws = null; + } + + async connect() { + if (this.connected || this.connecting) return; + this.connecting = true; + this.requestUpdate(); + await this.updateComplete; + + this.container = this.querySelector('#xterm-container') as HTMLElement; + if (!this.container) { this.connecting = false; return; } + + const isDark = document.documentElement.classList.contains('dark'); + this.term = new Terminal({ + cursorBlink: true, + fontFamily: '"Fira Code", "Cascadia Code", monospace', + fontSize: 13, + theme: isDark + ? { background: '#09090b', foreground: '#e4e4e7', cursor: '#a1a1aa' } + : { background: '#ffffff', foreground: '#18181b', cursor: '#52525b' }, + }); + this.fitAddon = new FitAddon(); + this.term.loadAddon(this.fitAddon); + this.term.loadAddon(new WebLinksAddon()); + this.term.open(this.container); + this.fitAddon.fit(); + + this.ws = new WebSocket('ws://localhost:7701'); + this.ws.onopen = () => { + this.connected = true; this.connecting = false; + this.requestUpdate(); + }; + this.ws.onclose = () => { + this.connected = false; this.connecting = false; + this.term?.write('\r\n\x1b[31m[disconnected]\x1b[0m\r\n'); + this.requestUpdate(); + }; + this.ws.onerror = () => { + this.connecting = false; this.connected = false; + this.term?.write('\r\n\x1b[31m[connection error - is terminal server running?]\x1b[0m\r\n'); + this.requestUpdate(); + }; + this.ws.onmessage = (e) => { + const m = JSON.parse(e.data); + if (m.type === 'data') this.term?.write(m.data); + if (m.type === 'exit') this.term?.write(`\r\n\x1b[33m[process exited: ${m.code}]\x1b[0m\r\n`); + }; + this.term.onData((d) => { this.ws?.send(JSON.stringify({ type: 'input', data: d })); }); + + this.resizeObs = new ResizeObserver(() => { + requestAnimationFrame(() => { + this.fitAddon?.fit(); + if (this.term && this.ws?.readyState === WebSocket.OPEN) { + this.ws.send(JSON.stringify({ type: 'resize', cols: this.term.cols, rows: this.term.rows })); + } + }); + }); + this.resizeObs.observe(this.container); + } + + disconnect() { this.destroyTerminal(); this.connected = false; this.requesting = false; this.requestUpdate(); } + + override render() { + return html` +
+
+ bash +
+ ${!this.connected && !this.connecting + ? html`` + : html``} + ${this.connecting ? html`Connecting...` : html``} + ${this.connected ? html`` : html``} +
+
+ `; + } +} diff --git a/packages/web-ui/example/src/main.ts b/packages/web-ui/example/src/main.ts index de7dc21..973085e 100644 --- a/packages/web-ui/example/src/main.ts +++ b/packages/web-ui/example/src/main.ts @@ -42,6 +42,11 @@ import "./components/utility-toggle.js"; import { JaeSessionSidebar } from "./components/session-sidebar.js"; import "./components/session-sidebar.js"; +import { JaeTerminalPanel } from './components/terminal-panel.js'; +import { JaeBrowserPanel } from './components/browser-panel.js'; +import './components/terminal-panel.js'; +import './components/browser-panel.js'; + registerCustomMessageRenderers(); const settings = new SettingsStore(); @@ -75,6 +80,9 @@ let currentSessionId: string | undefined; let currentTitle = ""; let isEditingTitle = false; let agent: Agent; +let rightPanel: 'none' | 'terminal' | 'browser' = 'none'; +let terminalPanel: JaeTerminalPanel | null = null; +let browserPanel: JaeBrowserPanel | null = null; let chatPanel: ChatPanel; let agentUnsubscribe: (() => void) | undefined; @@ -267,11 +275,24 @@ const newSession = () => { const handleSuggestion = (e: Event) => { const text = (e as CustomEvent).detail; - if (chatPanel && (chatPanel as any).agentInterface) { - (chatPanel as any).agentInterface.setInput(text); + if (!text) return; + // Try ChatPanel.agentInterface.setInput first + if (chatPanel?.agentInterface) { + chatPanel.agentInterface.setInput(text); + // Focus the textarea after injection + requestAnimationFrame(() => { + const ta = document.querySelector("message-editor textarea") as HTMLTextAreaElement + || document.querySelector("textarea") as HTMLTextAreaElement; + if (ta) ta.focus(); + }); } else { - const textarea = document.querySelector("textarea") as HTMLTextAreaElement; - if (textarea) { textarea.value = text; textarea.dispatchEvent(new Event("input", { bubbles: true })); textarea.focus(); } + const ta = document.querySelector("message-editor textarea") as HTMLTextAreaElement + || document.querySelector("textarea") as HTMLTextAreaElement; + if (ta) { + ta.value = text; + ta.dispatchEvent(new Event("input", { bubbles: true })); + ta.focus(); + } } }; @@ -300,23 +321,45 @@ const renderApp = () => { ${Button({ variant: "ghost", size: "sm", children: html`⌘K`, onClick: () => commandPalette.show(), title: "Command Palette (Ctrl+K)" })} ${utilityToggle} - ${Button({ variant: "ghost", size: "sm", children: icon(Settings, "sm"), onClick: () => SettingsDialog.open([new ProvidersModelsTab(), new ProxyTab()]), title: "Settings" })} + ${Button({ variant: "ghost", size: "sm", children: html``, onClick: () => { rightPanel = rightPanel === 'terminal' ? 'none' : 'terminal'; renderApp(); if (rightPanel === 'terminal') requestAnimationFrame(() => { terminalPanel = document.querySelector('jae-terminal-panel') as JaeTerminalPanel; terminalPanel?.connect(); }); }, title: "Toggle Terminal" })} +${Button({ variant: "ghost", size: "sm", children: html``, onClick: () => { rightPanel = rightPanel === 'browser' ? 'none' : 'browser'; renderApp(); }, title: "Toggle Browser" })} +${Button({ variant: "ghost", size: "sm", children: icon(Settings, "sm"), onClick: () => SettingsDialog.open([new ProvidersModelsTab(), new ProxyTab()]), title: "Settings" })}
- ${sidebar} -
- ${!hasMessages ? html` -
- -
- ` : html``} -
- ${chatPanel} -
-
-
-`, app); +${sidebar} +
+${!hasMessages ? html` +
+ +
+` : html``} +
+${chatPanel} +
+
+${rightPanel !== 'none' ? html` +
+
+ + +
+ +
+${rightPanel === 'terminal' ? html`` : html``} +${rightPanel === 'browser' ? html`` : html``} +
+` : html``} + +`, app); }; async function initApp() {