diff --git a/apps/desktop/package.json b/apps/desktop/package.json index b21be619de..b9956ef30f 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -96,6 +96,7 @@ "@icons-pack/react-simple-icons": "13.11.1", "@lezer/highlight": "1.2.3", "@nanostores/react": "1.1.0", + "@novnc/novnc": "1.7.0", "@nous-research/ui": "0.18.2", "@streamdown/code": "1.1.1", "@streamdown/math": "1.0.2", diff --git a/apps/desktop/src/lib/sibling-ws-url.test.ts b/apps/desktop/src/lib/sibling-ws-url.test.ts new file mode 100644 index 0000000000..06ac27e43e --- /dev/null +++ b/apps/desktop/src/lib/sibling-ws-url.test.ts @@ -0,0 +1,49 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { resolveSiblingWsUrl } from './sibling-ws-url' + +// A sibling stream (voice PCM, Bot Screen RFB) must dial the SAME (connection, +// profile) backend chat uses. The bare v1 getConnection pair answers for the +// local primary — the wrong machine when a registry remote rides over a local +// install — so registry routes must go through the *For bridges. +describe('resolveSiblingWsUrl', () => { + const remoteWsUrl = 'wss://gateway.example/api/ws?ticket=fresh' + const localWsUrl = 'ws://127.0.0.1:5151/api/ws?token=local' + + let getConnection: ReturnType + let getConnectionFor: ReturnType + let getGatewayWsUrl: ReturnType + let getGatewayWsUrlFor: ReturnType + + beforeEach(() => { + getConnection = vi.fn(async () => ({ authMode: 'token', baseUrl: 'http://127.0.0.1:5151', wsUrl: localWsUrl })) + getConnectionFor = vi.fn(async () => ({ authMode: 'token', baseUrl: 'https://gateway.example', wsUrl: remoteWsUrl })) + getGatewayWsUrl = vi.fn(async () => ({ ok: true, wsUrl: localWsUrl })) + getGatewayWsUrlFor = vi.fn(async () => ({ ok: true, wsUrl: remoteWsUrl })) + Object.defineProperty(window, 'hermesDesktop', { + configurable: true, + value: { getConnection, getConnectionFor, getGatewayWsUrl, getGatewayWsUrlFor } + }) + }) + + afterEach(() => { + Reflect.deleteProperty(window, 'hermesDesktop') + }) + + it('routes a registry-scoped profile through the *For bridges and swaps only the path', async () => { + const url = await resolveSiblingWsUrl({ connectionId: 'gw-tailscale', profile: 'research' }, '/api/display/ws') + + expect(url).toBe('wss://gateway.example/api/display/ws?ticket=fresh') + expect(getConnectionFor).toHaveBeenCalledWith({ connectionId: 'gw-tailscale', profile: 'research' }) + expect(getConnection).not.toHaveBeenCalled() + expect(getGatewayWsUrl).not.toHaveBeenCalled() + }) + + it('strips the spent gateway credential when the sibling route authenticates itself', async () => { + const url = new URL(await resolveSiblingWsUrl({ profile: null }, 'api/display/ws', { stripGatewayCredential: true })) + + expect(url.origin + url.pathname).toBe('ws://127.0.0.1:5151/api/display/ws') + expect(url.searchParams.has('token')).toBe(false) + expect(url.searchParams.has('ticket')).toBe(false) + }) +}) diff --git a/apps/desktop/src/lib/sibling-ws-url.ts b/apps/desktop/src/lib/sibling-ws-url.ts new file mode 100644 index 0000000000..7ba0be8045 --- /dev/null +++ b/apps/desktop/src/lib/sibling-ws-url.ts @@ -0,0 +1,86 @@ +/** + * Sibling WebSocket URLs beside `/api/ws` for a (connection, profile) route. + * + * Some gateway streams are not JSON-RPC and cannot share the gateway socket: + * voice PCM (`/api/audio/speak-stream`) and the Bot Screen's raw RFB + * (`/api/display/ws`). They ride the SAME authenticated origin the route's + * `/api/ws` uses — a fresh credential for OAuth remotes, the registry-scoped + * `*For` bridges for a remote riding over a local install (the bare + * getConnection/getGatewayWsUrl pair answers for the v1 primary backend, which + * would be the wrong machine). One resolver so every sibling stream routes the + * way chat does. + */ + +import { resolveGatewayWsUrl } from '@hermes/shared' + +const RESOLVE_TIMEOUT_MS = 15_000 + +function withTimeout(promise: Promise, ms: number, label: string): Promise { + return new Promise((resolve, reject) => { + const timer = window.setTimeout(() => reject(new Error(label)), ms) + promise.then( + value => { + window.clearTimeout(timer) + resolve(value) + }, + error => { + window.clearTimeout(timer) + reject(error instanceof Error ? error : new Error(String(error))) + } + ) + }) +} + +export interface SiblingWsRoute { + connectionId?: null | string + profile?: null | string +} + +/** + * Resolve `ws(s):///` for `route`. `stripGatewayCredential` + * drops the `?ticket=`/`?token=` the gateway URL carried when the sibling route + * authenticates on its own credential (a one-shot ticket must not be spent + * twice; the display bridge mints its own). + */ +export async function resolveSiblingWsUrl( + route: SiblingWsRoute, + path: string, + options: { stripGatewayCredential?: boolean } = {} +): Promise { + const desktop = window.hermesDesktop + + if (!desktop?.getConnection) { + throw new Error('Hermes Desktop connection bridge unavailable') + } + + const connectionId = route.connectionId?.trim() || null + const profile = route.profile?.trim() || null + + const conn = + connectionId && desktop.getConnectionFor + ? await withTimeout(desktop.getConnectionFor({ connectionId, profile }), RESOLVE_TIMEOUT_MS, `Timed out connecting to profile "${profile}"`) + : await withTimeout(desktop.getConnection(profile), RESOLVE_TIMEOUT_MS, `Timed out connecting to profile "${profile}"`) + + const wsDeps = + connectionId && desktop.getGatewayWsUrlFor + ? { getGatewayWsUrl: () => desktop.getGatewayWsUrlFor!({ connectionId, profile }) } + : connectionId + ? {} + : desktop + + const wsUrl = await withTimeout(resolveGatewayWsUrl(wsDeps, conn), RESOLVE_TIMEOUT_MS, 'Timed out minting the gateway WebSocket URL') + const url = new URL(wsUrl) + + if (!url.pathname.endsWith('/api/ws')) { + throw new Error(`Unexpected gateway WebSocket path: ${url.pathname}`) + } + + url.pathname = url.pathname.replace(/\/api\/ws$/, path.startsWith('/') ? path : `/${path}`) + + if (options.stripGatewayCredential) { + url.searchParams.delete('ticket') + url.searchParams.delete('token') + } + + return url.toString() +} diff --git a/apps/desktop/src/plugins/hermes-bots/bot-row.tsx b/apps/desktop/src/plugins/hermes-bots/bot-row.tsx index f21a121026..ab4ed24372 100644 --- a/apps/desktop/src/plugins/hermes-bots/bot-row.tsx +++ b/apps/desktop/src/plugins/hermes-bots/bot-row.tsx @@ -72,6 +72,7 @@ import { useTurnBusy, workerActiveAt } from './row-helpers' +import { openBotScreen } from './screen-open' import type { GroupMember, RosterRow, SidebarRowLabels } from './types' import { $botSections, $draggingBot, BOT_DRAG_MIME, botSectionId, moveBotsToSection } from './user-sections' @@ -310,6 +311,7 @@ export function BotRow({ bot, onDelete, onEdit, onGroup, onNewSection, showHandl {row} void openRosterBot(bot)}>{b.bot.openBotChat} + openBotScreen(bot, meta)}>{b.screen.menu} { diff --git a/apps/desktop/src/plugins/hermes-bots/i18n.ts b/apps/desktop/src/plugins/hermes-bots/i18n.ts index d6b96cf348..12e63a6b71 100644 --- a/apps/desktop/src/plugins/hermes-bots/i18n.ts +++ b/apps/desktop/src/plugins/hermes-bots/i18n.ts @@ -223,6 +223,31 @@ type BotsMessages = { noMcpServers: string } + /** Bot Screen: the bot's headless desktop on the gateway host, live in a pane. */ + screen: { + title: string + menu: string + unsupportedTitle: string + unsupportedBody: string + notInstalledTitle: string + notInstalledBody: string + installHint: string + recheck: string + stoppedTitle: string + stoppedBody: string + start: string + attaching: string + streamLost: string + reconnect: string + takeOver: string + handBack: string + youControl: string + otherControls: string + agentControls: string + controlTaken: string + handoffRequested: string + } + /** Bot-scoped scheduled jobs. Generic scheduling chrome (weekday names, * Daily/Hourly, the job verbs) resolves against core's `cron` section. */ cron: { @@ -446,6 +471,29 @@ const en: BotsMessages = { searchHub: 'Search the hub (community + well-known sources)…', noMcpServers: 'No MCP servers configured or in the catalog.' }, + screen: { + title: 'Screen', + menu: 'Open Screen', + unsupportedTitle: 'No bot screen on this host', + unsupportedBody: 'Bot screens run on Linux gateway hosts. This bot uses the host\u2019s own display.', + notInstalledTitle: 'Screen packages missing', + notInstalledBody: 'The gateway host needs TigerVNC and the Xfce core to give this bot a screen. Run on the host:', + installHint: 'or: hermes computer-use screen install', + recheck: 'Check again', + stoppedTitle: 'Screen is off', + stoppedBody: 'Start this bot\u2019s desktop to watch what it does and take over when it needs you.', + start: 'Start screen', + attaching: 'Connecting to the screen\u2026', + streamLost: 'Screen stream ended', + reconnect: 'Reconnect', + takeOver: 'Take over', + handBack: 'Hand back', + youControl: 'You are in control', + otherControls: 'Another viewer is in control', + agentControls: 'Bot is in control', + controlTaken: 'Another viewer took control. Watching only.', + handoffRequested: 'Bot needs you' + }, cron: { filterHint: 'Scheduled jobs exist in this profile but none are tagged for this bot. Name a job "[bot:] …" to show it here, or see them in Cron below.', @@ -665,6 +713,29 @@ const ja: BotsMessages = { searchHub: 'ハブを検索(コミュニティと既知のソース)…', noMcpServers: '設定済みまたはカタログ内の MCP サーバーはありません。' }, + screen: { + title: '画面', + menu: '画面を開く', + unsupportedTitle: 'このホストにはボット画面がありません', + unsupportedBody: 'ボット画面は Linux のゲートウェイホストで動作します。このボットはホスト自身のディスプレイを使います。', + notInstalledTitle: '画面パッケージが不足しています', + notInstalledBody: 'このボットに画面を与えるには、ゲートウェイホストに TigerVNC と Xfce コアが必要です。ホストで実行:', + installHint: 'または: hermes computer-use screen install', + recheck: '再確認', + stoppedTitle: '画面はオフです', + stoppedBody: 'このボットのデスクトップを起動すると、動作を見守り、必要なときに操作を引き継げます。', + start: '画面を起動', + attaching: '画面に接続中…', + streamLost: '画面ストリームが終了しました', + reconnect: '再接続', + takeOver: '引き継ぐ', + handBack: '戻す', + youControl: 'あなたが操作中', + otherControls: '別のビューアが操作中', + agentControls: 'ボットが操作中', + controlTaken: '別のビューアが操作を引き継ぎました。閲覧のみ。', + handoffRequested: 'ボットが助けを求めています' + }, cron: { filterHint: 'このプロファイルには定期実行ジョブがありますが、このボット向けのタグが付いたものはありません。ジョブ名を「[bot:<名前>] …」にするとここに表示されます。下のCronでも確認できます。', @@ -879,6 +950,29 @@ const zh: BotsMessages = { searchHub: '搜索技能中心(社区和常见来源)…', noMcpServers: '未配置 MCP 服务器,目录中也没有。' }, + screen: { + title: '屏幕', + menu: '打开屏幕', + unsupportedTitle: '此主机没有机器人屏幕', + unsupportedBody: '机器人屏幕在 Linux 网关主机上运行。此机器人使用主机自身的显示器。', + notInstalledTitle: '缺少屏幕软件包', + notInstalledBody: '网关主机需要 TigerVNC 和 Xfce 核心组件才能为此机器人提供屏幕。在主机上运行:', + installHint: '或: hermes computer-use screen install', + recheck: '重新检查', + stoppedTitle: '屏幕已关闭', + stoppedBody: '启动此机器人的桌面,观看它的操作,并在需要时接管。', + start: '启动屏幕', + attaching: '正在连接屏幕…', + streamLost: '屏幕流已结束', + reconnect: '重新连接', + takeOver: '接管', + handBack: '交还', + youControl: '你正在控制', + otherControls: '另一位查看者正在控制', + agentControls: '机器人正在控制', + controlTaken: '另一位查看者已接管控制。仅可观看。', + handoffRequested: '机器人需要你' + }, cron: { filterHint: '此配置档案中有定时任务,但没有一个标记给这个机器人。将任务命名为“[bot:<名称>] …”即可显示在这里,也可以在下方的 Cron 中查看。', @@ -1093,6 +1187,29 @@ const zhHant: BotsMessages = { searchHub: '搜尋技能中心(社群和常見來源)…', noMcpServers: '未設定 MCP 伺服器,目錄中也沒有。' }, + screen: { + title: '螢幕', + menu: '開啟螢幕', + unsupportedTitle: '此主機沒有機器人螢幕', + unsupportedBody: '機器人螢幕在 Linux 閘道主機上執行。此機器人使用主機自身的顯示器。', + notInstalledTitle: '缺少螢幕套件', + notInstalledBody: '閘道主機需要 TigerVNC 與 Xfce 核心元件才能為此機器人提供螢幕。在主機上執行:', + installHint: '或: hermes computer-use screen install', + recheck: '重新檢查', + stoppedTitle: '螢幕已關閉', + stoppedBody: '啟動此機器人的桌面,觀看它的操作,並在需要時接手。', + start: '啟動螢幕', + attaching: '正在連線至螢幕…', + streamLost: '螢幕串流已結束', + reconnect: '重新連線', + takeOver: '接手', + handBack: '交還', + youControl: '你正在控制', + otherControls: '另一位檢視者正在控制', + agentControls: '機器人正在控制', + controlTaken: '另一位檢視者已接手控制。僅可觀看。', + handoffRequested: '機器人需要你' + }, cron: { filterHint: '此設定檔中有排程工作,但沒有任何一個標記給這個機器人。將工作命名為「[bot:<名稱>] …」即可顯示在這裡,也可以在下方的 Cron 中查看。', diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts new file mode 100644 index 0000000000..247a79ec0e --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts @@ -0,0 +1,81 @@ +/** + * Bot Screen — connection plumbing for a bot's Bot Desktop (the headless Xfce + * screen its computer_use drives on the gateway host). + * + * Two legs share one authenticated origin: JSON-RPC (`display.*`) rides the + * bot's pooled gateway socket through `host.requestProfile`; the RFB stream + * rides a SIBLING WebSocket to `/api/display/ws`, minted per attach by + * `display.observe` (single-use, 30 s). noVNC's Websock takes ownership of the + * socket it is handed, so it can never share the JSON-RPC one — same reason + * voice playback opens `/api/audio/speak-stream` beside `/api/ws`. + */ + +import { host, resolveSiblingWsUrl } from '@hermes/plugin-sdk' +import type { PluginProfileRoute } from '@hermes/plugin-sdk' + +import { botConnectionRoute } from './routing' +import type { RosterRow } from './types' + +export interface DisplayLease { + holder: 'agent' | 'human' + viewer_id: null | string + since: number + reason: string + pending_handoff: null | string +} + +export interface DisplayStatus { + profile: string + profile_key: string + supported: boolean + installed: boolean + missing: string[] + running: boolean + pid: null | number + display: null | string + socket: null | string + geometry: string + install_command: null | string + lease: DisplayLease +} + +export interface DisplayObserveResult extends DisplayStatus { + ticket: string + path: string + viewer_id: string +} + +/** Stable per-window viewer identity: the lease names who holds control, and a + * reload must NOT silently inherit a stale holder's authority. */ +export const VIEWER_ID = `desktop-${Math.random().toString(36).slice(2, 10)}` + +/** Bare-profile fallback so a v1 local bot (no registry route) still resolves. */ +export function botScreenRoute(bot: RosterRow): PluginProfileRoute | string { + return botConnectionRoute(bot) ?? bot.name +} + +export function displayRequest(bot: RosterRow, method: string, params: Record = {}): Promise { + return host.requestProfile(botScreenRoute(bot), method, params) +} + +/** + * Resolve the RFB WebSocket URL for `bot`: the bot's gateway `/api/ws` origin + * (fresh credential for OAuth remotes) with the path swapped for the display + * bridge and the single-use display ticket attached. + */ +export async function resolveScreenWsUrl(bot: RosterRow, ticket: string): Promise { + const route = botConnectionRoute(bot) + + // The /api/ws credential authenticated the RPC that minted the display ticket; + // the bridge authenticates on the ticket alone, so the gateway credential is + // dropped rather than spending a second one-shot ticket. + const url = new URL( + await resolveSiblingWsUrl({ connectionId: route?.connectionId ?? null, profile: route?.profile ?? bot.name }, '/api/display/ws', { + stripGatewayCredential: true + }) + ) + + url.searchParams.set('display_ticket', ticket) + + return url.toString() +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-open.tsx b/apps/desktop/src/plugins/hermes-bots/screen-open.tsx new file mode 100644 index 0000000000..ae0ff946c6 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-open.tsx @@ -0,0 +1,45 @@ +/** + * Open a bot's Screen as a main-window workspace tab (host.openWorkspace), + * one tab per bot; a second open refocuses the existing tab. + */ + +import { host } from '@hermes/plugin-sdk' + +import { botSelectionKey } from './data' +import { displayName } from './labels' +import { BotScreenPane } from './screen-pane' +import { ID } from './shared' +import type { BotMeta, RosterRow } from './types' + +const openTabs = new Map void>() + +export function screenPaneId(bot: RosterRow): string { + return `plugin-workspace:${ID}:screen:${botSelectionKey(bot)}` +} + +export function openBotScreen(bot: RosterRow, meta?: BotMeta | null): void { + if (typeof host.openWorkspace !== 'function') { + host.notify({ kind: 'info', message: 'Update Hermes Desktop to open bot screens.' }) + + return + } + + const key = botSelectionKey(bot) + + if (openTabs.has(key)) { + host.revealPane(screenPaneId(bot)) + + return + } + + const close = host.openWorkspace(`${ID}:screen:${key}`, { + title: `${displayName(bot, meta ?? null)} · Screen`, + minWidth: '28rem', + render: () => , + onClose: () => { + openTabs.delete(key) + } + }) + + openTabs.set(key, close) +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx new file mode 100644 index 0000000000..27d4190f9a --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -0,0 +1,306 @@ +/** + * Bot Screen pane — live view of a bot's headless desktop with Take over / Hand back. + * + * State authority: the BACKEND owns runtime + lease (`display.status`, pushed + * as `display.lease` events); this pane paints a cache of it. The RFB stream + * is a sibling WebSocket handed to noVNC's RFB; `viewOnly` here is UX only — + * the gateway drops input from anyone but the lease holder. + * + * Every attach spends a single-use ticket, so a lease flip that the bridge + * answers with close 4000 (`control-taken`) simply re-attaches in watch mode. + */ + +import { Button, Codicon, EmptyState, GlyphSpinner, host, useValue } from '@hermes/plugin-sdk' +import type { RpcEvent } from '@hermes/plugin-sdk' +import { useCallback, useEffect, useRef, useState } from 'react' + +import { useBots } from './i18n' +import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, resolveScreenWsUrl, VIEWER_ID } from './screen-connection' +import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' +import type { RosterRow } from './types' + +type RfbLike = { + viewOnly: boolean + scaleViewport: boolean + resizeSession: boolean + focusOnClick: boolean + background: string + qualityLevel: number + addEventListener: (type: string, handler: (event: { detail?: { clean?: boolean; reason?: string } }) => void) => void + disconnect: () => void + focus: () => void +} + +type ConnState = 'idle' | 'attaching' | 'live' | 'control-taken' | 'error' + +async function loadRfb(): Promise) => RfbLike> { + const mod = (await import('@novnc/novnc')) as unknown as { default: new (...args: never[]) => RfbLike } + + return mod.default as unknown as new (target: HTMLElement, socket: WebSocket, options?: Record) => RfbLike +} + +export function BotScreenPane({ bot }: { bot: RosterRow }) { + const t = useBots() + const screen = useValue($screenState) + const state = screenStateFor(screen, bot) + const status = state?.status ?? null + const lease = state?.lease ?? status?.lease ?? null + const iHold = lease?.holder === 'human' && lease.viewer_id === VIEWER_ID + + const canvasHost = useRef(null) + const rfb = useRef(null) + const socket = useRef(null) + const [conn, setConn] = useState('idle') + const [error, setError] = useState(null) + const [busy, setBusy] = useState(false) + const attachGeneration = useRef(0) + + const refresh = useCallback(async () => { + try { + const next = await displayRequest(bot, 'display.status') + setScreenStatus(bot, next) + setError(null) + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } + }, [bot]) + + useEffect(() => { + void refresh() + + return host.onEvent('display.lease', (event: RpcEvent) => { + const payload = event.payload as { profile_key?: string; lease?: DisplayLease } | undefined + + if (payload?.lease && payload.profile_key && payload.profile_key === status?.profile_key) { + setScreenLease(bot, payload.lease) + } + }) + }, [bot, refresh, status?.profile_key]) + + const detach = useCallback(() => { + attachGeneration.current += 1 + rfb.current?.disconnect() + rfb.current = null + socket.current?.close() + socket.current = null + }, []) + + const attach = useCallback(async () => { + if (!canvasHost.current) { + return + } + + detach() + const generation = attachGeneration.current + setConn('attaching') + setError(null) + + try { + // Load the client BEFORE dialing: noVNC's Websock installs its own `onopen`, so a socket that + // opened while the dynamic import was still in flight never hands it the open event. + const Rfb = await loadRfb() + const observe = await displayRequest(bot, 'display.observe', { viewer_id: VIEWER_ID }) + setScreenStatus(bot, observe) + const url = await resolveScreenWsUrl(bot, observe.ticket) + + if (generation !== attachGeneration.current || !canvasHost.current) { + return + } + + const ws = new WebSocket(url) + ws.binaryType = 'arraybuffer' + socket.current = ws + const client = new Rfb(canvasHost.current, ws, { shared: true }) + client.scaleViewport = true + client.resizeSession = false + client.focusOnClick = true + client.background = 'transparent' + client.qualityLevel = 7 + client.viewOnly = !(observe.lease.holder === 'human' && observe.lease.viewer_id === VIEWER_ID) + client.addEventListener('connect', () => { + if (generation === attachGeneration.current) { + setConn('live') + } + }) + client.addEventListener('disconnect', event => { + if (generation !== attachGeneration.current) { + return + } + + const reason = event.detail?.reason ?? '' + + if (reason.includes('control-taken')) { + setConn('control-taken') + } else if (event.detail?.clean) { + setConn('idle') + } else { + setConn('error') + setError(reason || t.screen.streamLost) + } + + void refresh() + }) + rfb.current = client + } catch (err) { + if (generation === attachGeneration.current) { + setConn('error') + setError(err instanceof Error ? err.message : String(err)) + } + } + }, [bot, detach, refresh, t.screen.streamLost]) + + // Visibility is not lifecycle: the stream stays attached while the pane is + // hidden; only unmount tears it down (and hands control back server-side). + useEffect(() => () => detach(), [detach]) + + useEffect(() => { + if (status?.running && conn === 'idle') { + void attach() + } + }, [attach, conn, status?.running]) + + useEffect(() => { + if (rfb.current) { + rfb.current.viewOnly = !iHold + + if (iHold) { + rfb.current.focus() + } + } + }, [iHold]) + + const start = useCallback(async () => { + setBusy(true) + + try { + const next = await displayRequest(bot, 'display.start') + setScreenStatus(bot, next) + setConn('idle') + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setBusy(false) + } + }, [bot]) + + const takeOver = useCallback(async () => { + setBusy(true) + + try { + const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.acquire', { viewer_id: VIEWER_ID }) + setScreenLease(bot, result.lease) + + if (conn !== 'live') { + void attach() + } + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setBusy(false) + } + }, [attach, bot, conn]) + + const handBack = useCallback(async () => { + setBusy(true) + + try { + const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.release', { viewer_id: VIEWER_ID }) + setScreenLease(bot, result.lease) + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setBusy(false) + } + }, [bot]) + + if (status && !status.supported) { + return + } + + if (status && !status.installed) { + return ( +
+
+
{t.screen.notInstalledTitle}
+
{t.screen.notInstalledBody}
+ {status.install_command ? ( + {status.install_command} + ) : null} +
{t.screen.installHint}
+ +
+
+ ) + } + + if (status && !status.running) { + return ( +
+
+
{t.screen.stoppedTitle}
+
{t.screen.stoppedBody}
+ + {error ?
{error}
: null} +
+
+ ) + } + + const humanOther = lease?.holder === 'human' && !iHold + + return ( +
+
+ + {t.screen.title} + {status?.display ? {status.display} · {status.geometry} : null} + + {lease?.pending_handoff ? ( + + {t.screen.handoffRequested} + + ) : null} + {iHold ? ( + {t.screen.youControl} + ) : humanOther ? ( + {t.screen.otherControls} + ) : ( + {t.screen.agentControls} + )} + {iHold ? ( + + ) : ( + + )} + +
+
+ {/* data-terminal: the same keyboard-ownership marker the terminal pane uses, so the app's + type-to-focus / bare-key shortcuts never steal keystrokes meant for the remote screen. */} +
+ {conn === 'attaching' ? ( +
+ {t.screen.attaching} +
+ ) : null} + {conn === 'control-taken' ? ( +
{t.screen.controlTaken}
+ ) : null} + {conn === 'error' && error ? ( +
{error}
+ ) : null} +
+
+ ) +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-state.ts b/apps/desktop/src/plugins/hermes-bots/screen-state.ts new file mode 100644 index 0000000000..e93ddea0ef --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-state.ts @@ -0,0 +1,40 @@ +/** + * Per-bot Bot Screen cache: backend truth (`display.status`) plus the last + * `display.lease` event, keyed by the bot's roster identity. Renderer-owned + * cache of backend state — never the authority. + */ + +import { atom } from 'nanostores' + +import { botSelectionKey } from './data' +import type { DisplayLease, DisplayStatus } from './screen-connection' +import type { RosterRow } from './types' + +export interface BotScreenState { + status: DisplayStatus | null + lease: DisplayLease | null +} + +export const $screenState = atom>({}) + +export function screenStateFor(all: Record, bot: RosterRow): BotScreenState | null { + return all[botSelectionKey(bot)] ?? null +} + +export function setScreenStatus(bot: RosterRow, status: DisplayStatus): void { + const key = botSelectionKey(bot) + const current = $screenState.get() + $screenState.set({ ...current, [key]: { status, lease: status.lease ?? current[key]?.lease ?? null } }) +} + +export function setScreenLease(bot: RosterRow, lease: DisplayLease): void { + const key = botSelectionKey(bot) + const current = $screenState.get() + const prev = current[key] + + if (prev?.lease && prev.lease.holder === lease.holder && prev.lease.viewer_id === lease.viewer_id && prev.lease.pending_handoff === lease.pending_handoff) { + return + } + + $screenState.set({ ...current, [key]: { status: prev?.status ?? null, lease } }) +} diff --git a/apps/desktop/src/sdk/index.ts b/apps/desktop/src/sdk/index.ts index 4ce472aa50..b3b2f85abf 100644 --- a/apps/desktop/src/sdk/index.ts +++ b/apps/desktop/src/sdk/index.ts @@ -1721,6 +1721,9 @@ export const TITLEBAR_AREAS = { center: 'titleBar.center', left: 'titleBar.left' * setup.runtime_check, reconciled) — pass `host.request`. Don't hand-roll * readiness from raw RPC shapes. */ export { evaluateRuntimeReadiness, type RuntimeReadinessResult } from '@/lib/runtime-readiness' +/** A sibling WebSocket beside the route's `/api/ws` (voice PCM, Bot Screen RFB): + * same origin, same auth resolution as chat. */ +export { resolveSiblingWsUrl, type SiblingWsRoute } from '@/lib/sibling-ws-url' /** Canonical time formatting — every surface pulls from here so timestamps read * the same app-wide. For a row's AGE, bucket with `coarseElapsed` and render * the compact suffixes (`t.sidebar.row.ageMin` → "52m"), which is what the diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 5273d0beb3..5e5b6c0c07 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -2255,6 +2255,13 @@ DEFAULT_CONFIG = { "paste_collapse_threshold_fallback": 5, "paste_collapse_char_threshold": 2000, + # Bot Desktop: a headless Xfce screen per profile on the gateway host (Linux), streamed to Hermes + # Desktop where a human can watch, take over (logins, 2FA, CAPTCHAs) and hand back. `hermes desktop`. + "bot_desktop": { + "geometry": "1440x900", + # Start the screen automatically the first time computer_use or a headed browser needs a display. + "auto_start": True, + }, "computer_use": { # cua-driver's upstream PostHog telemetry defaults ON; Hermes sets # CUA_DRIVER_RS_TELEMETRY_ENABLED=0 in every child env unless this is true. diff --git a/hermes_cli/dashboard_auth/ws_tickets.py b/hermes_cli/dashboard_auth/ws_tickets.py index aef1456b84..65391ae239 100644 --- a/hermes_cli/dashboard_auth/ws_tickets.py +++ b/hermes_cli/dashboard_auth/ws_tickets.py @@ -33,11 +33,13 @@ class TicketInvalid(Exception): """Ticket missing, expired, or already consumed.""" -def mint_ticket(*, user_id: str, provider: str) -> str: +def mint_ticket(*, user_id: str, provider: str, extra: Optional[Dict[str, Any]] = None) -> str: """One-shot base64url ticket (32 random bytes) bound to this identity; ``consume_ticket`` - hands the ``info`` dict back to the WS handler.""" + hands the ``info`` dict back to the WS handler. ``extra`` rides along for routes that need + server-chosen context (the Bot Desktop bridge pins the RFB socket's profile home here so a + client can never pick another profile's screen).""" ticket = secrets.token_urlsafe(32) - info = {"user_id": user_id, "provider": provider, "minted_at": int(time.time())} + info = {"user_id": user_id, "provider": provider, "minted_at": int(time.time()), **(extra or {})} with _lock: _tickets[ticket] = (int(time.time()) + TTL_SECONDS, info) _gc_expired_locked() diff --git a/hermes_cli/subcommands/computer_use.py b/hermes_cli/subcommands/computer_use.py index ac0d52f4b8..22e0effcd6 100644 --- a/hermes_cli/subcommands/computer_use.py +++ b/hermes_cli/subcommands/computer_use.py @@ -174,6 +174,12 @@ def build_computer_use_parser(subparsers) -> None: "grant", help="Request the grants (opens the dialog attributed to CuaDriver)") _perms_actions = {"grant": _cu_perms_grant, "status": _cu_perms_status} + from hermes_cli.subcommands.computer_use_screen import build_screen_parser + build_screen_parser(computer_use_sub, add_json_flag) + + def _cu_screen(args): + return args.screen_func(args) + def _cu_permissions(args): handler = _perms_actions.get(getattr(args, "computer_use_perms_action", None)) if handler is not None: @@ -181,7 +187,7 @@ def build_computer_use_parser(subparsers) -> None: computer_use_perms.print_help() _actions = {"install": _cu_install, "status": _cu_status, "doctor": _cu_doctor, - "permissions": _cu_permissions} + "permissions": _cu_permissions, "screen": _cu_screen} def cmd_computer_use(args): handler = _actions.get(getattr(args, "computer_use_action", None)) diff --git a/hermes_cli/subcommands/computer_use_screen.py b/hermes_cli/subcommands/computer_use_screen.py new file mode 100644 index 0000000000..5c2dc52606 --- /dev/null +++ b/hermes_cli/subcommands/computer_use_screen.py @@ -0,0 +1,108 @@ +"""``hermes computer-use screen`` — the Bot Desktop screen a profile's ``computer_use`` drives on a +headless Linux gateway host, viewable from Hermes Desktop. ``status`` / ``start`` / ``stop`` / +``install`` mirror the Desktop pane's controls for ops shells and cloud images.""" + +from __future__ import annotations + +import json +import subprocess +import sys + + +def _screen_status(args) -> int: + from tools.bot_desktop import lease, runtime + st = runtime.status() + if bool(getattr(args, "json", False)): + print(json.dumps({**st.as_dict(), "lease": lease.get().as_dict()}, indent=2, sort_keys=True)) + return 0 if st.running else 1 + if not st.supported: + print("Bot Desktop screens run on Linux gateway hosts only (this host keeps its real display).") + return 1 + if not st.installed: + print("Bot Desktop: packages missing → " + ", ".join(st.missing)) + print(" Install: " + (st.install_command or "hermes computer-use screen install")) + return 1 + if st.running: + holder = lease.get() + who = f"human ({holder.viewer_id})" if holder.holder == "human" else "agent" + print(f"Bot Desktop [{st.profile}]: running on DISPLAY {st.display} ({st.geometry}), pid {st.pid}") + print(f" control: {who} rfb socket: {st.socket}") + print(" View it: Hermes Desktop → Bots → this bot → Screen") + return 0 + print(f"Bot Desktop [{st.profile}]: installed, not running. Start: hermes computer-use screen start") + return 1 + + +def _screen_start(args) -> int: + from tools.bot_desktop import runtime + try: + st = runtime.start() + except RuntimeError as exc: + print(f"Bot Desktop: {exc}") + return 1 + print(f"Bot Desktop [{st.profile}]: running on DISPLAY {st.display} ({st.geometry})") + return 0 + + +def _screen_stop(args) -> int: + from tools.bot_desktop import runtime + print("Bot Desktop: stopped" if runtime.stop() else "Bot Desktop: was not running") + return 0 + + +def _screen_install(args) -> int: + from tools.bot_desktop import runtime + if not runtime.is_supported_host(): + print("Bot Desktop screens run on Linux gateway hosts only.") + return 1 + if not runtime.missing_binaries(): + print("Bot Desktop: packages already installed.") + return 0 + cmd = runtime.install_command() + if cmd is None: + print("Bot Desktop: no supported package manager (apt/dnf/pacman) found. Install TigerVNC (Xvnc) and " + "the Xfce core (xfwm4, xfce4-panel, xfdesktop, xfce4-settings) by hand.") + return 1 + print(f"Bot Desktop: installing → {cmd}") + if not bool(getattr(args, "yes", False)) and sys.stdin.isatty(): + answer = input("Proceed? [Y/n] ").strip().lower() + if answer not in ("", "y", "yes"): + return 1 + rc = subprocess.run(cmd, shell=True, stdin=None).returncode # windows-footgun: ok — Linux-only, apt/dnf/pacman + if rc != 0: + print(f"Bot Desktop: installer exited {rc}") + return rc + missing = runtime.missing_binaries() + if missing: + print("Bot Desktop: still missing " + ", ".join(missing)) + return 1 + print("Bot Desktop: ready. Start with `hermes computer-use screen start` or from Hermes Desktop.") + return 0 + + +SCREEN_ACTIONS = {"status": _screen_status, "start": _screen_start, "stop": _screen_stop, "install": _screen_install} + + +def build_screen_parser(computer_use_sub, add_json_flag) -> None: + screen = computer_use_sub.add_parser( + "screen", help="Bot Desktop: the headless screen this profile's computer_use drives (Linux)", + description="On a headless Linux gateway host Hermes gives each profile its own Xfce screen\n" + "(TigerVNC Xvnc on a private Unix socket). The agent's computer_use and headed\n" + "browser act on it; Hermes Desktop shows it live and lets a human take over for\n" + "logins, 2FA or CAPTCHAs, then hand control back.\n\n" + "`install` adds the system packages (apt/dnf/pacman); `start`/`stop` manage this\n" + "profile's screen; `status` shows display, control holder and socket.") + sub = screen.add_subparsers(dest="computer_use_screen_action") + st = sub.add_parser("status", help="Show whether this profile's screen is installed/running and who holds control") + add_json_flag(st, "Emit the status payload as JSON.") + sub.add_parser("start", help="Start this profile's screen") + sub.add_parser("stop", help="Stop this profile's screen (hands control back to the agent first)") + inst = sub.add_parser("install", help="Install TigerVNC + Xfce core via the host package manager") + inst.add_argument("-y", "--yes", action="store_true", help="Do not ask before running the package manager") + + def _cmd(args): + handler = SCREEN_ACTIONS.get(str(getattr(args, "computer_use_screen_action", None) or "")) + if handler is not None: + return handler(args) + screen.print_help() + screen.set_defaults(screen_func=_cmd) diff --git a/hermes_cli/web_routers/display.py b/hermes_cli/web_routers/display.py new file mode 100644 index 0000000000..a9fe07a5c3 --- /dev/null +++ b/hermes_cli/web_routers/display.py @@ -0,0 +1,144 @@ +"""``/api/display/ws`` — raw RFB over WebSocket for the Bot Desktop viewer. + +The Desktop renderer calls ``display.observe`` on its authenticated ``/api/ws`` connection, gets a +single-use 30 s ticket pinned to that profile's RFB socket, then opens this route with +``?display_ticket=``. No websockify, no new port: the bridge splices the profile's 0600 Unix socket +into the WebSocket as binary frames with backpressure both ways, and runs the client stream through +:class:`tools.bot_desktop.rfb_filter.RfbClientFilter` so keyboard, pointer and clipboard reach Xvnc +only from the viewer that currently holds the lease. noVNC's ``viewOnly`` is UX; this is the gate. + +A lease change closes the evicted viewer's socket with 4000 ``control-taken`` so its UI drops back to +Watch mode and reconnects. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Optional + +from fastapi import APIRouter, WebSocket, WebSocketDisconnect + +from hermes_cli.web_server_chat import _ws_request_is_allowed + +_log = logging.getLogger(__name__) +router = APIRouter() + +_READ_CHUNK = 64 * 1024 +_CLOSE_CONTROL_TAKEN = 4000 +_CLOSE_DESKTOP_GONE = 4001 +_CLOSE_BAD_TICKET = 4401 +_CLOSE_NOT_ALLOWED = 4403 +_CLOSE_PROTOCOL = 1003 + + +def _consume_display_ticket(ws: WebSocket) -> Optional[dict]: + from hermes_cli.dashboard_auth.ws_tickets import TicketInvalid, consume_ticket + ticket = ws.query_params.get("display_ticket", "") + if not ticket: + return None + try: + info = consume_ticket(ticket) + except TicketInvalid: + return None + if info.get("provider") != "bot-desktop" or not info.get("hermes_home"): + return None + return info + + +@router.websocket("/api/display/ws") +async def display_ws(ws: WebSocket) -> None: + if not _ws_request_is_allowed(ws): + await ws.close(code=_CLOSE_NOT_ALLOWED) + return + info = _consume_display_ticket(ws) + if info is None: + await ws.close(code=_CLOSE_BAD_TICKET, reason="display ticket missing, expired or used") + return + + from hermes_constants import hermes_home_key + from tools.bot_desktop import lease as _lease + from tools.bot_desktop.rfb_filter import RfbClientFilter + from pathlib import Path + + sock = Path(info["hermes_home"]) / "bot-desktop" / "rfb.sock" + profile_key = hermes_home_key(info["hermes_home"]) + viewer_id = str(info.get("viewer_id") or info.get("user_id") or "viewer") + if not sock.exists(): + await ws.close(code=_CLOSE_DESKTOP_GONE, reason="Bot Desktop is not running") + return + try: + reader, writer = await asyncio.open_unix_connection(str(sock)) + except OSError as exc: + _log.warning("display ws: cannot reach RFB socket %s: %s", sock, exc) + await ws.close(code=_CLOSE_DESKTOP_GONE, reason="Bot Desktop socket unreachable") + return + + await ws.accept() + loop = asyncio.get_running_loop() + evicted = asyncio.Event() + held = {"ever": _lease.viewer_may_send_input(viewer_id, profile_key=profile_key)} + + def _on_lease(key: str, lease) -> None: + # A viewer that held control during this connection and lost it to ANOTHER human is kicked so + # its UI repaints; a plain hand-back to the agent, pure watchers and the new holder stay connected. + if key != profile_key: + return + mine = lease.holder == _lease.HUMAN and lease.viewer_id == viewer_id + if mine: + held["ever"] = True + elif held["ever"] and lease.holder == _lease.HUMAN: + loop.call_soon_threadsafe(evicted.set) + unsubscribe = _lease.on_change(_on_lease) + + rfb_filter = RfbClientFilter(lambda: _lease.viewer_may_send_input(viewer_id, profile_key=profile_key)) + + async def rfb_to_ws() -> None: + while True: + chunk = await reader.read(_READ_CHUNK) + if not chunk: + return + await ws.send_bytes(chunk) # awaiting the send is the backpressure toward Xvnc + + async def ws_to_rfb() -> None: + while True: + message = await ws.receive() + if message.get("type") == "websocket.disconnect": + return + data = message.get("bytes") + if data is None: + await ws.close(code=_CLOSE_PROTOCOL, reason="RFB is binary") + return + try: + allowed = rfb_filter.feed(data) + except ValueError as exc: + await ws.close(code=_CLOSE_PROTOCOL, reason=str(exc)[:100]) + return + if allowed: + writer.write(allowed) + await writer.drain() # backpressure toward the browser + + async def watch_eviction() -> None: + await evicted.wait() + await ws.close(code=_CLOSE_CONTROL_TAKEN, reason="control-taken") + + tasks = [asyncio.create_task(rfb_to_ws()), asyncio.create_task(ws_to_rfb()), + asyncio.create_task(watch_eviction())] + try: + done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for t in pending: + t.cancel() + for t in done: + exc = t.exception() + if exc and not isinstance(exc, (WebSocketDisconnect, ConnectionError)): + _log.debug("display ws ended: %r", exc) + finally: + unsubscribe() + writer.close() + # Closing the viewer window hands control back; a stale holder never pins the agent out. + if _lease.viewer_may_send_input(viewer_id, profile_key=profile_key): + _lease.release(viewer_id, profile_key=profile_key) + try: + await ws.close() + except Exception: # already closed by the peer or by an eviction + pass diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index d95f8182a6..73faef43e6 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -932,6 +932,7 @@ from hermes_cli.web_routers import ( # noqa: E402 status as _status_routes, actions as _actions_routes, audio as _audio_routes, + display as _display_routes, sessions as _sessions_routes, profiles as _profiles_routes, memory_providers as _memory_providers_routes, @@ -955,6 +956,7 @@ app.include_router(_local_models_routes.router) app.include_router(_status_routes.router) app.include_router(_actions_routes.router) app.include_router(_audio_routes.router) +app.include_router(_display_routes.router) app.include_router(_actions_routes.status_router) app.include_router(_sessions_routes.list_router) app.include_router(_profiles_routes.sessions_router) diff --git a/package-lock.json b/package-lock.json index eaf154d450..a24c4496ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,7 @@ }, "apps/bootstrap-installer": { "name": "@hermes/bootstrap-installer", - "version": "0.0.1", + "version": "0.21.1", "dependencies": { "@nous-research/ui": "0.18.2", "@tailwindcss/typography": "0.5.20", @@ -85,6 +85,7 @@ "@lezer/highlight": "1.2.3", "@nanostores/react": "1.1.0", "@nous-research/ui": "0.18.2", + "@novnc/novnc": "1.7.0", "@streamdown/code": "1.1.1", "@streamdown/math": "1.0.2", "@tabler/icons-react": "3.44.0", @@ -3498,6 +3499,12 @@ } } }, + "node_modules/@novnc/novnc": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@novnc/novnc/-/novnc-1.7.0.tgz", + "integrity": "sha512-ucEJOx4T2avIRCleodk7YobZj5O2Ga2AeLfQ69A/yjG9HHba2+PDgwSkN3FttrmG+70ZGx21sElNFouK13RzyA==", + "license": "MPL-2.0" + }, "node_modules/@npmcli/agent": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.2.2.tgz", diff --git a/tests/hermes_cli/test_display_ws_ticket.py b/tests/hermes_cli/test_display_ws_ticket.py new file mode 100644 index 0000000000..fc072596b1 --- /dev/null +++ b/tests/hermes_cli/test_display_ws_ticket.py @@ -0,0 +1,27 @@ +"""The display bridge admits only a display ticket minted for THIS profile's socket: a gateway ticket, +an expired ticket, or one for another provider is refused before any socket is dialled.""" + +from __future__ import annotations + +from hermes_cli.dashboard_auth import ws_tickets +from hermes_cli.web_routers import display + + +class _Ws: + def __init__(self, **params): + self.query_params = params + + +def test_display_ticket_must_be_a_bot_desktop_ticket_pinned_to_a_profile_home(monkeypatch): + ws_tickets._reset_for_tests() + gateway_ticket = ws_tickets.mint_ticket(user_id="u", provider="google") + assert display._consume_display_ticket(_Ws(display_ticket=gateway_ticket)) is None + + unpinned = ws_tickets.mint_ticket(user_id="display:v", provider="bot-desktop") + assert display._consume_display_ticket(_Ws(display_ticket=unpinned)) is None + + good = ws_tickets.mint_ticket(user_id="display:v", provider="bot-desktop", + extra={"hermes_home": "/srv/hermes/bot-a", "viewer_id": "v"}) + info = display._consume_display_ticket(_Ws(display_ticket=good)) + assert info and info["hermes_home"] == "/srv/hermes/bot-a" and info["viewer_id"] == "v" + assert display._consume_display_ticket(_Ws(display_ticket=good)) is None, "single use" diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index f5c2c431fd..34811b799b 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -35,6 +35,24 @@ def _no_host_browser_use_cli(): yield +@pytest.fixture(autouse=True) +def _no_host_bot_desktop_autostart(): + """Keep the host's TigerVNC/Xfce install out of tests. + + ``computer_use`` auto-starts the profile's Bot Desktop on a headless Linux + host with the packages installed, so a developer box that has them would + launch a real Xvnc + Xfce session per test. Pin the binaries to "missing"; + tests that exercise the desktop path monkeypatch ``runtime`` themselves. + """ + try: + from tools.bot_desktop import runtime as bd_runtime + except Exception: + yield + return + with patch.object(bd_runtime, "missing_binaries", lambda: ["Xvnc"]): + yield + + @pytest.fixture(autouse=True) def _materialize_mcp_sdk_symbols(): """Materialize the lazily-imported MCP SDK before each tools test. diff --git a/tests/tools/test_bot_desktop_lease.py b/tests/tools/test_bot_desktop_lease.py new file mode 100644 index 0000000000..9266ad1ec2 --- /dev/null +++ b/tests/tools/test_bot_desktop_lease.py @@ -0,0 +1,67 @@ +"""Bot Desktop invariants: the byte-level RFB input gate follows the lease, and computer_use refuses +every action (capture included) while a human holds the screen.""" + +from __future__ import annotations + +import json + +import pytest + +from tools.bot_desktop import lease +from tools.bot_desktop.rfb_filter import RfbClientFilter + +_HANDSHAKE = b"RFB 003.008\n" + b"\x01" + b"\x00" +_KEY = b"\x04\x01\x00\x00\x00\x00\x00\x61" # KeyEvent 'a' down +# QEMU Extended KeyEvent (type 255, sub 0): what noVNC sends once Xvnc advertises the pseudo-encoding. +_QEMU_KEY = bytes([255, 0, 0, 1]) + (0x65).to_bytes(4, "big") + (0x12).to_bytes(4, "big") +_POINTER = b"\x05\x01\x00\x10\x00\x10" # PointerEvent, button 1 +_CUT = b"\x06\x00\x00\x00\x00\x00\x00\x02hi" # ClientCutText "hi" +_FBUR = b"\x03\x00" + b"\x00" * 8 # FramebufferUpdateRequest +_SETENC = b"\x02\x00\x00\x02" + b"\x00\x00\x00\x07" + b"\xff\xff\xff\x21" # SetEncodings x2 + + +@pytest.fixture(autouse=True) +def _fresh_lease(): + lease._reset_for_tests() + yield + lease._reset_for_tests() + + +def test_rfb_filter_forwards_input_only_from_the_lease_holder_across_arbitrary_chunking(): + f = RfbClientFilter(lambda: lease.viewer_may_send_input("v1")) + head = f.feed(_HANDSHAKE) + assert head[-1:] == b"\x01", "ClientInit is forced shared so a viewer never kicks the agent's watcher" + + # Agent holds: read-only messages pass, input is dropped, even when split byte by byte. + stream = _KEY + _FBUR + _POINTER + _SETENC + _CUT + _QEMU_KEY + out = b"".join(f.feed(stream[i:i + 1]) for i in range(len(stream))) + assert out == _FBUR + _SETENC + + lease.acquire("v1") + assert f.feed(_KEY + _POINTER + _QEMU_KEY) == _KEY + _POINTER + _QEMU_KEY + + lease.acquire("v2") # last writer wins: v1 is evicted from input on the very next message + assert f.feed(_KEY) == b"" + assert lease.release("v1").holder == lease.HUMAN, "a stale viewer's release must not yank control from v2" + assert lease.release("v2").holder == lease.AGENT + + +def test_computer_use_refuses_every_action_while_a_human_holds_the_screen(monkeypatch): + from tools.computer_use import tool + + calls = [] + monkeypatch.setattr(tool, "_get_backend", lambda session_id="": calls.append(session_id) or object()) + lease.acquire("human") + for action in ("capture", "click", "type", "list_windows"): + res = json.loads(tool.handle_computer_use({"action": action, "text": "pw"})) + assert res["code"] == "human_has_control", action + assert calls == [], "the driver is never touched while the human may be typing a credential" + + # Handoff round trip: the agent asks, the human takes over and hands back, the agent is unblocked. + asked = json.loads(tool.handle_computer_use({"action": "request_handoff", "reason": "log in"})) + assert asked["ok"] and asked["state"]["pending_handoff"] == "log in" + lease.acquire("human", reason="log in") + assert lease.get().pending_handoff is None + lease.release("human") + done = json.loads(tool.handle_computer_use({"action": "wait_for_human", "seconds": 1})) + assert done["ok"] and done["state"]["holder"] == lease.AGENT diff --git a/tools/bot_desktop/__init__.py b/tools/bot_desktop/__init__.py new file mode 100644 index 0000000000..2ffa400fd6 --- /dev/null +++ b/tools/bot_desktop/__init__.py @@ -0,0 +1,5 @@ +"""Bot Desktop: per-profile headless Xfce desktop over RFB, viewed from Hermes Desktop. + +``runtime`` owns the Xvnc/Xfce process and the published env; ``lease`` owns who may drive the +screen (agent vs. human); ``rfb_filter`` is the byte-level input gate the WebSocket bridge applies. +""" diff --git a/tools/bot_desktop/launcher.sh b/tools/bot_desktop/launcher.sh new file mode 100755 index 0000000000..60c3635982 --- /dev/null +++ b/tools/bot_desktop/launcher.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Hermes Bot Desktop — one headless Xfce desktop per Hermes profile, served over RFB. +# +# Spawned by tools/bot_desktop/runtime.py with HERMES_BD_* variables set. Runs TigerVNC's Xvnc +# (X server + RFB server in one process; damage-driven, resizable via SetDesktopSize) listening on a +# 0600 Unix socket only, then a minimal Xfce started component-wise under a private dbus session. +# +# Why not startxfce4 / xfce4-session: xfce4-session expects a logind session scope; outside one it +# spawns polkit agents that pop empty dialogs and light-locker/xfce4-screensaver lock the desktop for a +# user who has no password. Starting xfsettingsd -> xfwm4 -> xfdesktop -> xfce4-panel directly, with +# the screensaver/locker/power-manager autostarts masked, is the shape every headless-VNC recipe +# converges on (TigerVNC #1096/#581, OpenOnDemand's apptainer desktop, the Arch wiki). +# +# Why not the xfce4 metapackage: it drags in the screensaver, power manager and polkit agent this +# script exists to keep out. +set -euo pipefail + +: "${HERMES_BD_PROFILE:?}" # profile name (display name in the VNC title) +: "${HERMES_BD_DISPLAY_NUM:?}" # allocated by runtime.py +: "${HERMES_BD_SOCKET:?}" # RFB unix socket path +: "${HERMES_BD_XAUTH:?}" # Xauthority path +: "${HERMES_BD_ENV_FILE:?}" # where to publish DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS +: "${HERMES_BD_CONFIG_HOME:?}" # per-profile XDG_CONFIG_HOME (xfconf lives here) +GEOM="${HERMES_BD_GEOMETRY:-1440x900}" +DEPTH=24 + +export XDG_CONFIG_HOME="$HERMES_BD_CONFIG_HOME" +export XDG_CACHE_HOME="${HERMES_BD_CACHE_HOME:-$HERMES_BD_CONFIG_HOME/.cache}" +export XDG_DATA_HOME="${HERMES_BD_DATA_HOME:-$HERMES_BD_CONFIG_HOME/.local-share}" +export XDG_SESSION_TYPE=x11 XDG_CURRENT_DESKTOP=XFCE +export GDK_BACKEND=x11 QT_QPA_PLATFORM=xcb NO_AT_BRIDGE=1 GTK_A11Y=none +export LANG="${LANG:-C.UTF-8}" +# Inheriting a login session's bus/session manager yields "Another session manager is already +# running" / "Unable to contact settings server". +unset SESSION_MANAGER DBUS_SESSION_BUS_ADDRESS DISPLAY XAUTHORITY WAYLAND_DISPLAY + +mkdir -p "$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml" "$XDG_CONFIG_HOME/autostart" \ + "$XDG_CACHE_HOME" "$XDG_DATA_HOME" "$(dirname "$HERMES_BD_SOCKET")" + +export DISPLAY=":$HERMES_BD_DISPLAY_NUM" +export XAUTHORITY="$HERMES_BD_XAUTH" + +# Stale lock files from a crashed server block restart. +rm -f "$HERMES_BD_SOCKET" "/tmp/.X${HERMES_BD_DISPLAY_NUM}-lock" "/tmp/.X11-unix/X${HERMES_BD_DISPLAY_NUM}" +: > "$XAUTHORITY"; chmod 600 "$XAUTHORITY" +xauth -q -f "$XAUTHORITY" add "$DISPLAY" MIT-MAGIC-COOKIE-1 "$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" + +# ---- pre-seed xfconf BEFORE xfconfd starts (it caches; edits after start are overwritten) ---- +X="$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml" +[[ -e "$X/xfwm4.xml" ]] || cat > "$X/xfwm4.xml" <<'EOF' + + + + + + + + +EOF +[[ -e "$X/xfce4-screensaver.xml" ]] || cat > "$X/xfce4-screensaver.xml" <<'EOF' + + + + + +EOF +[[ -e "$X/xsettings.xml" ]] || cat > "$X/xsettings.xml" <<'EOF' + + + + + + + + + + + +EOF +[[ -e "$X/xfce4-desktop.xml" ]] || cat > "$X/xfce4-desktop.xml" <<'EOF' + + + + + + + + + + + + + + + + + + + + +EOF +# The vendor default panel layout suppresses the first-run "Welcome to the panel" dialog. +if [[ ! -e "$X/xfce4-panel.xml" ]]; then + for d in /etc/xdg/xfce4/panel/default.xml /usr/share/xfce4-panel/default.xml \ + /etc/xdg/xdg-xubuntu/xfce4/panel/default.xml; do + [[ -e "$d" ]] && { cp "$d" "$X/xfce4-panel.xml"; break; } + done +fi +# Mask system autostarts that want logind/polkit/keyring/at-spi. +for a in xfce4-screensaver light-locker xfce4-power-manager xfce-polkit \ + polkit-gnome-authentication-agent-1 lxpolkit xfce4-notifyd blueman at-spi-dbus-bus \ + gnome-keyring-pkcs11 gnome-keyring-secrets gnome-keyring-ssh xdg-user-dirs; do + [[ -e "$XDG_CONFIG_HOME/autostart/$a.desktop" ]] || \ + printf '[Desktop Entry]\nType=Application\nName=%s\nHidden=true\n' "$a" > "$XDG_CONFIG_HOME/autostart/$a.desktop" +done + +# ---- X server + RFB (TigerVNC Xvnc), Unix socket only ---- +# SecurityTypes None is safe ONLY because -rfbport -1 disables TCP and the 0600 socket is reachable +# solely by the gateway process, whose WebSocket bridge performs the real authentication. +Xvnc "$DISPLAY" -geometry "$GEOM" -depth "$DEPTH" -dpi 96 \ + -rfbport -1 -rfbunixpath "$HERMES_BD_SOCKET" -rfbunixmode 0600 \ + -SecurityTypes None -AlwaysShared -AcceptSetDesktopSize -FrameRate 30 \ + -desktop "hermes:$HERMES_BD_PROFILE" -auth "$XAUTHORITY" -nolisten tcp \ + -Log '*:stderr:30' & +XVNC_PID=$! +trap 'kill "$XVNC_PID" 2>/dev/null || true' EXIT +for _ in $(seq 1 100); do + xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 && break + kill -0 "$XVNC_PID" 2>/dev/null || { echo "Xvnc exited during startup" >&2; exit 1; } + sleep 0.1 +done +xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 || { echo "Xvnc did not become ready" >&2; exit 1; } + +setxkbmap -display "$DISPLAY" us 2>/dev/null || true # RFB keysyms + xdotool assume a known layout +xsetroot -display "$DISPLAY" -solid '#1c1f29' 2>/dev/null || true +xset -display "$DISPLAY" s off -dpms s noblank 2>/dev/null || true + +# ---- private session bus + Xfce components (no xfce4-session) ---- +# dbus-run-session scopes the bus to this subshell: no leaked dbus-daemons on restart. The env file +# is written from INSIDE the bus so DBUS_SESSION_BUS_ADDRESS is the real one; runtime.py and every +# cua-driver / browser spawn for this profile source it. +exec dbus-run-session -- bash -c ' + set -e + umask 077 + printf "DISPLAY=%s\nXAUTHORITY=%s\nDBUS_SESSION_BUS_ADDRESS=%s\nXDG_CONFIG_HOME=%s\nXDG_CACHE_HOME=%s\nXDG_DATA_HOME=%s\n" \ + "$DISPLAY" "$XAUTHORITY" "$DBUS_SESSION_BUS_ADDRESS" "$XDG_CONFIG_HOME" "$XDG_CACHE_HOME" "$XDG_DATA_HOME" \ + > "$HERMES_BD_ENV_FILE.tmp" && mv -f "$HERMES_BD_ENV_FILE.tmp" "$HERMES_BD_ENV_FILE" + xfsettingsd --sm-client-disable --daemon 2>/dev/null || true + xfwm4 --compositor=off --sm-client-disable & + for _ in $(seq 1 50); do xprop -root _NET_SUPPORTING_WM_CHECK >/dev/null 2>&1 && break; sleep 0.1; done + xfdesktop --sm-client-disable --disable-wm-check & + exec xfce4-panel --sm-client-disable --disable-wm-check +' diff --git a/tools/bot_desktop/lease.py b/tools/bot_desktop/lease.py new file mode 100644 index 0000000000..405c683936 --- /dev/null +++ b/tools/bot_desktop/lease.py @@ -0,0 +1,154 @@ +"""Who may drive a profile's Bot Desktop screen: the agent (default) or exactly one human viewer. + +The lease is the single truth shared by the RFB bridge (drops human input from non-holders), the +``computer_use`` tool (refuses to act while a human holds control — the person may be typing a +credential, so even screenshots are refused; fail closed rather than trusting the agent to pause +itself) and the Desktop UI (Watch / Take over / Hand back). + +Per-process, keyed by ``hermes_home_key()`` so multiplexed profiles never share a lease. Handoff +requests raised by the agent (``request_handoff``) are how it asks for hands and later learns the +human is done: ``wait_for_release`` blocks the tool call until the lease returns to the agent. +""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional + +from hermes_constants import hermes_home_key + +AGENT = "agent" +HUMAN = "human" + + +class HumanHasControl(RuntimeError): + """Raised by screen-driving tools while a human holds the lease.""" + + +@dataclass +class Lease: + holder: str = AGENT + viewer_id: Optional[str] = None + since: float = field(default_factory=time.time) + reason: str = "" + pending_handoff: Optional[str] = None # agent's reason for asking, until the human takes over + + def as_dict(self) -> Dict[str, object]: + return {"holder": self.holder, "viewer_id": self.viewer_id, "since": self.since, + "reason": self.reason, "pending_handoff": self.pending_handoff} + + +_lock = threading.Condition() +_leases: Dict[str, Lease] = {} +_listeners: List[Callable[[str, Lease], None]] = [] + + +def _key(profile_key: Optional[str]) -> str: + return profile_key or hermes_home_key() + + +def get(profile_key: Optional[str] = None) -> Lease: + with _lock: + return _leases.setdefault(_key(profile_key), Lease()) + + +def on_change(listener: Callable[[str, Lease], None]) -> Callable[[], None]: + """Subscribe to lease transitions (gateway broadcasts them to Desktop clients).""" + with _lock: + _listeners.append(listener) + + def _off() -> None: + with _lock: + if listener in _listeners: + _listeners.remove(listener) + return _off + + +def _notify(key: str, lease: Lease) -> None: + for cb in list(_listeners): + try: + cb(key, lease) + except Exception: # a broken subscriber must not wedge the handoff + pass + + +def acquire(viewer_id: str, *, profile_key: Optional[str] = None, reason: str = "") -> Lease: + """Human ``viewer_id`` takes control. Last writer wins: a second viewer evicts the first, and the + RFB bridge closes the evicted socket so its UI drops to view-only.""" + key = _key(profile_key) + with _lock: + lease = _leases.setdefault(key, Lease()) + lease.holder, lease.viewer_id, lease.since, lease.reason = HUMAN, viewer_id, time.time(), reason + lease.pending_handoff = None + _lock.notify_all() + _notify(key, lease) + return lease + + +def release(viewer_id: Optional[str] = None, *, profile_key: Optional[str] = None) -> Lease: + """Return control to the agent. With ``viewer_id`` only that holder may release (a stale viewer + closing its window must not yank control from the one who took over after it).""" + key = _key(profile_key) + with _lock: + lease = _leases.setdefault(key, Lease()) + if viewer_id is not None and lease.holder == HUMAN and lease.viewer_id != viewer_id: + return lease + lease.holder, lease.viewer_id, lease.since, lease.reason = AGENT, None, time.time(), "" + lease.pending_handoff = None # "hand back" answers an open request even if nobody formally took over + _lock.notify_all() + _notify(key, lease) + return lease + + +def request_handoff(reason: str, *, profile_key: Optional[str] = None) -> Lease: + """Agent asks a human to take over (login, 2FA, CAPTCHA, payment). Recorded so the UI can show + why and the bridge can page the user; control itself still flips only on ``acquire``.""" + key = _key(profile_key) + with _lock: + lease = _leases.setdefault(key, Lease()) + lease.pending_handoff = reason + _lock.notify_all() + _notify(key, lease) + return lease + + +def wait_for_release(*, timeout: float, profile_key: Optional[str] = None) -> bool: + """Block until the agent holds the lease (and no handoff is pending) or ``timeout`` elapses. + True when control is back with the agent.""" + key = _key(profile_key) + deadline = time.monotonic() + timeout + with _lock: + while True: + lease = _leases.setdefault(key, Lease()) + if lease.holder == AGENT and lease.pending_handoff is None: + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + _lock.wait(remaining) + + +def human_holds(profile_key: Optional[str] = None) -> bool: + return get(profile_key).holder == HUMAN + + +def viewer_may_send_input(viewer_id: str, *, profile_key: Optional[str] = None) -> bool: + lease = get(profile_key) + return lease.holder == HUMAN and lease.viewer_id == viewer_id + + +def assert_agent_may_act(profile_key: Optional[str] = None) -> None: + lease = get(profile_key) + if lease.holder == HUMAN: + raise HumanHasControl( + "A human has taken over this desktop (they may be entering a credential). Screen actions and " + "captures are refused until they hand control back; call computer_use action='wait_for_human' " + "to block until then.") + + +def _reset_for_tests() -> None: + with _lock: + _leases.clear() + _listeners.clear() diff --git a/tools/bot_desktop/rfb_filter.py b/tools/bot_desktop/rfb_filter.py new file mode 100644 index 0000000000..804ad9d3fd --- /dev/null +++ b/tools/bot_desktop/rfb_filter.py @@ -0,0 +1,94 @@ +"""Byte-level RFB client→server gate for the Bot Desktop WebSocket bridge. + +noVNC's ``viewOnly`` is a UI hint; anyone holding the socket could still inject input. The bridge +parses the client stream and forwards only non-input messages from viewers that do not hold the +lease. RFB messages do not align with WebSocket frames, so this is a stateful stream parser fed +arbitrary chunks (RFC 6143 §7.5 layouts; TigerVNC's EnableContinuousUpdates 150 and Fence 248 pass +through untouched since they carry no input; QEMU Extended KeyEvent 255 is keyboard input — noVNC +switches to it as soon as Xvnc advertises the pseudo-encoding — so it is gated like KeyEvent). + +Xvnc runs ``-SecurityTypes None``, so the handshake is fixed-size: 12-byte version, 1-byte security +choice, then ``ClientInit`` (1 byte). ``ServerInit`` is server→client and never crosses this filter. +""" + +from __future__ import annotations + +from typing import Callable + +_INPUT_TYPES = {4, 5, 6, 255} # KeyEvent, PointerEvent, ClientCutText, QEMU Extended KeyEvent + +# Fixed-length client messages: type -> total length including the type byte. +_FIXED = { + 0: 20, # SetPixelFormat + 3: 10, # FramebufferUpdateRequest + 4: 8, # KeyEvent + 5: 6, # PointerEvent + 150: 10, # EnableContinuousUpdates + 255: 12, # QEMU client message; sub-type 0 = Extended KeyEvent (the only one noVNC sends) +} +_SET_ENCODINGS = 2 +_CLIENT_CUT_TEXT = 6 +_FENCE = 248 + + +class RfbClientFilter: + """Feed client bytes with :meth:`feed`; get back the bytes allowed to reach Xvnc. + + ``allow_input`` is consulted per message so a lease flip mid-stream applies to the very next + key or pointer event. + """ + + def __init__(self, allow_input: Callable[[], bool]) -> None: + self._allow_input = allow_input + self._buf = bytearray() + self._handshake_left = 12 + 1 + 1 # version + security type + ClientInit(shared flag) + + def feed(self, chunk: bytes) -> bytes: + self._buf += chunk + out = bytearray() + if self._handshake_left: + take = min(self._handshake_left, len(self._buf)) + if take: + head = bytes(self._buf[:take]) + # ClientInit shared-flag: force shared so a human viewer never disconnects the agent's + # watcher or another observer (Xvnc also runs -AlwaysShared; belt and braces at zero cost). + if self._handshake_left - take == 0 and take >= 1: + head = head[:-1] + b"\x01" + out += head + del self._buf[:take] + self._handshake_left -= take + if self._handshake_left: + return bytes(out) + while self._buf: + length = self._message_length() + if length is None or len(self._buf) < length: + break + msg = bytes(self._buf[:length]) + del self._buf[:length] + if msg[0] in _INPUT_TYPES and not self._allow_input(): + continue + out += msg + return bytes(out) + + def _message_length(self) -> int | None: + t = self._buf[0] + if t in _FIXED: + return _FIXED[t] + if t == _SET_ENCODINGS: + if len(self._buf) < 4: + return None + n = int.from_bytes(self._buf[2:4], "big") + return 4 + 4 * n + if t == _CLIENT_CUT_TEXT: + if len(self._buf) < 8: + return None + n = int.from_bytes(self._buf[4:8], "big", signed=True) + # Extended clipboard (RFB 3.8 + TigerVNC): negative length, |n| bytes follow. + return 8 + abs(n) + if t == _FENCE: + if len(self._buf) < 9: + return None + return 9 + self._buf[8] + # Unknown client message: we cannot frame it, and forwarding blind would let an input message + # hide behind it. Drop the rest of the stream; the viewer reconnects. + raise ValueError(f"unknown RFB client message type {t}") diff --git a/tools/bot_desktop/runtime.py b/tools/bot_desktop/runtime.py new file mode 100644 index 0000000000..cead59dd06 --- /dev/null +++ b/tools/bot_desktop/runtime.py @@ -0,0 +1,295 @@ +"""Bot Desktop runtime: one headless Xfce desktop per Hermes profile, served over RFB on a private +Unix socket, viewed and driven from Hermes Desktop. + +Layout under ``/bot-desktop/``: ``display`` (allocated X display number), ``rfb.sock`` +(Xvnc RFB Unix socket, 0600), ``Xauthority``, ``env`` (DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS +published by the launcher once Xfce's bus exists), ``launcher.pid``, ``launcher.log``, ``xdg/`` +(per-profile XDG_CONFIG_HOME so two profiles never share xfconf). Everything is profile-scoped via +``get_hermes_home()`` so N profiles in one gateway get N desktops: one screen per bot on the shared +machine. + +The launcher is ``launcher.sh`` next to this module; :func:`desktop_env` is what cua-driver and headed +Chromium spawns merge in so the agent acts on this profile's screen and nowhere else. +""" + +from __future__ import annotations + +import logging +import os +import shutil +import signal +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Optional + +from hermes_constants import get_hermes_home + +logger = logging.getLogger(__name__) + +_LAUNCHER = Path(__file__).with_name("launcher.sh") + +# Display numbers below 10 collide with real seats and default Xvfb recipes (:99 is popular too); scan a +# private band and record the choice so restarts reuse it. +_DISPLAY_MIN, _DISPLAY_MAX = 20, 89 + +# Binaries the launcher execs; the package hint is per distro family. +REQUIRED_BINARIES = ("Xvnc", "xfwm4", "xfce4-panel", "xfdesktop", "xfsettingsd", "dbus-run-session", + "xauth", "xdpyinfo", "setxkbmap") + +PACKAGES = { + "apt": ["tigervnc-standalone-server", "xfce4-panel", "xfwm4", "xfdesktop4", "xfce4-settings", + "xfce4-terminal", "dbus-x11", "x11-xserver-utils", "x11-utils", "xauth", "fonts-dejavu-core"], + "dnf": ["tigervnc-server-minimal", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", + "xfce4-terminal", "dbus-x11", "xorg-x11-server-utils", "xorg-x11-utils", "xorg-x11-xauth", + "dejavu-sans-fonts"], + "pacman": ["tigervnc", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", "xfce4-terminal", + "xorg-xsetroot", "xorg-xset", "xorg-xdpyinfo", "xorg-xauth", "xorg-setxkbmap", "ttf-dejavu"], +} + + +def state_dir() -> Path: + return get_hermes_home() / "bot-desktop" + + +def is_supported_host() -> bool: + return sys.platform.startswith("linux") + + +def missing_binaries() -> list[str]: + return [b for b in REQUIRED_BINARIES if shutil.which(b) is None] + + +def package_manager() -> Optional[str]: + for pm in ("apt-get", "dnf", "pacman"): + if shutil.which(pm): + return "apt" if pm == "apt-get" else pm + return None + + +def install_command() -> Optional[str]: + pm = package_manager() + if pm is None: + return None + pkgs = " ".join(PACKAGES[pm]) + return { + "apt": f"sudo apt-get install -y --no-install-recommends {pkgs}", + "dnf": f"sudo dnf install -y {pkgs}", + "pacman": f"sudo pacman -S --needed --noconfirm {pkgs}", + }[pm] + + +@dataclass +class DesktopStatus: + profile: str + supported: bool + installed: bool + missing: list[str] + running: bool + pid: Optional[int] + display: Optional[str] + socket: Optional[str] + geometry: str + install_command: Optional[str] + + def as_dict(self) -> Dict[str, object]: + return dict(self.__dict__) + + +def _read(path: Path) -> Optional[str]: + try: + return path.read_text(encoding="utf-8").strip() or None + except OSError: + return None + + +def _pid_alive(pid: int) -> bool: + import psutil + return psutil.pid_exists(pid) + + +def _launcher_pid() -> Optional[int]: + raw = _read(state_dir() / "launcher.pid") + if not raw or not raw.isdigit(): + return None + pid = int(raw) + return pid if _pid_alive(pid) else None + + +def _display_in_use(num: int) -> bool: + return Path(f"/tmp/.X{num}-lock").exists() or Path(f"/tmp/.X11-unix/X{num}").exists() + + +def _allocate_display() -> int: + recorded = _read(state_dir() / "display") + if recorded and recorded.isdigit(): + return int(recorded) + for num in range(_DISPLAY_MIN, _DISPLAY_MAX + 1): + if not _display_in_use(num): + return num + raise RuntimeError("no free X display number in the Bot Desktop band") + + +def desktop_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]: + """``base_env`` (default ``os.environ``) with this profile's DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS + merged in when its desktop is running. Unchanged otherwise, so hosts with a real seat keep it. + Pure: never starts anything (it is called from env builders, status probes and tests).""" + env = dict(os.environ if base_env is None else base_env) + published = published_env() + if published: + env.update(published) + env.pop("WAYLAND_DISPLAY", None) # X11 desktop; a leaked Wayland socket flips GTK/Chromium backends + return env + + +def ensure_started_for_tool() -> None: + """Tool-boundary hook (``computer_use`` dispatch): with ``bot_desktop.auto_start`` (default on) a Linux + host that has NO display and the packages installed gets its screen started on first use, so a headless + gateway works the first time instead of answering "no DISPLAY is set". Failure is not an error here; + the tool's own "no display" diagnosis is the right message then.""" + if published_env() or not _should_auto_start(os.environ): + return + try: + start() + except Exception as exc: + logger.info("Bot Desktop auto-start skipped: %s", exc) + + +def _should_auto_start(env: Dict[str, str]) -> bool: + if not is_supported_host() or env.get("DISPLAY") or env.get("WAYLAND_DISPLAY"): + return False + if missing_binaries(): + return False + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly().get("bot_desktop") or {} + return bool(cfg.get("auto_start", True)) + + +def published_env() -> Dict[str, str]: + """Variables the launcher wrote once Xfce's private bus existed; empty when the desktop is down.""" + if _launcher_pid() is None: + return {} + raw = _read(state_dir() / "env") + if not raw: + return {} + out: Dict[str, str] = {} + for line in raw.splitlines(): + key, sep, value = line.partition("=") + if sep: + out[key.strip()] = value.strip() + return out + + +def rfb_socket_path() -> Optional[Path]: + sock = state_dir() / "rfb.sock" + return sock if _launcher_pid() is not None and sock.exists() else None + + +def geometry() -> str: + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly().get("bot_desktop") or {} + return str(cfg.get("geometry") or "1440x900") + + +def status(profile: Optional[str] = None) -> DesktopStatus: + missing: list[str] = missing_binaries() if is_supported_host() else list(REQUIRED_BINARIES) + pid = _launcher_pid() + env = published_env() + return DesktopStatus( + profile=profile or _profile_name(), + supported=is_supported_host(), + installed=not missing, + missing=missing, + running=pid is not None and bool(env.get("DISPLAY")), + pid=pid, + display=env.get("DISPLAY"), + socket=str(rfb_socket_path()) if rfb_socket_path() else None, + geometry=geometry(), + install_command=install_command() if missing else None, + ) + + +def _profile_name() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return get_active_profile_name() or "default" + except Exception: + return "default" + + +def start(*, wait_seconds: float = 15.0) -> DesktopStatus: + """Start this profile's desktop (idempotent). Blocks until the launcher publishes its env file or + ``wait_seconds`` pass; raises ``RuntimeError`` naming the blocker.""" + if not is_supported_host(): + raise RuntimeError("Bot Desktop runs on Linux gateway hosts only") + missing = missing_binaries() + if missing: + hint = install_command() or "install TigerVNC (Xvnc) and the Xfce core components" + raise RuntimeError(f"Bot Desktop needs {', '.join(missing)} on the gateway host. Install: {hint}") + if _launcher_pid() is not None and published_env().get("DISPLAY"): + return status() + + sd = state_dir() + sd.mkdir(parents=True, exist_ok=True) + os.chmod(sd, 0o700) + num = _allocate_display() + (sd / "display").write_text(str(num), encoding="utf-8") + env_file = sd / "env" + env_file.unlink(missing_ok=True) + + child_env = {k: v for k, v in os.environ.items() if k not in { + "DISPLAY", "XAUTHORITY", "WAYLAND_DISPLAY", "DBUS_SESSION_BUS_ADDRESS", "SESSION_MANAGER"}} + child_env.update({ + "HERMES_BD_PROFILE": _profile_name(), + "HERMES_BD_DISPLAY_NUM": str(num), + "HERMES_BD_SOCKET": str(sd / "rfb.sock"), + "HERMES_BD_XAUTH": str(sd / "Xauthority"), + "HERMES_BD_ENV_FILE": str(env_file), + "HERMES_BD_CONFIG_HOME": str(sd / "xdg"), + "HERMES_BD_GEOMETRY": geometry(), + }) + log = open(sd / "launcher.log", "ab") # noqa: SIM115 — handed to the child, closed by it + proc = subprocess.Popen( # windows-footgun: ok — Linux-only runtime (is_supported_host) + ["bash", str(_LAUNCHER)], env=child_env, stdin=subprocess.DEVNULL, stdout=log, stderr=log, + start_new_session=True, close_fds=True) + log.close() + (sd / "launcher.pid").write_text(str(proc.pid), encoding="utf-8") + + deadline = time.monotonic() + wait_seconds + while time.monotonic() < deadline: + if proc.poll() is not None: + tail = (sd / "launcher.log").read_bytes()[-2000:].decode("utf-8", "replace") + raise RuntimeError(f"Bot Desktop launcher exited with {proc.returncode}:\n{tail}") + if env_file.exists() and (sd / "rfb.sock").exists(): + logger.info("Bot Desktop for profile %s up on :%s", _profile_name(), num) + return status() + time.sleep(0.1) + raise RuntimeError(f"Bot Desktop did not publish its display within {wait_seconds:.0f}s (see {sd / 'launcher.log'})") + + +def stop() -> bool: + """Stop this profile's desktop; True when a running launcher was signalled.""" + pid = _launcher_pid() + sd = state_dir() + if pid is None: + (sd / "env").unlink(missing_ok=True) + return False + # The launcher runs in its own session; killing the group takes Xvnc, dbus and Xfce with it. + try: + os.killpg(pid, signal.SIGTERM) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start) + except ProcessLookupError: + pass + for _ in range(50): + if not _pid_alive(pid): + break + time.sleep(0.1) + else: + try: + os.killpg(pid, signal.SIGKILL) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start) + except ProcessLookupError: + pass + (sd / "launcher.pid").unlink(missing_ok=True) + (sd / "env").unlink(missing_ok=True) + return True diff --git a/tools/browser_tool.py b/tools/browser_tool.py index 104bc7694d..5a8c67b3ab 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -42,7 +42,9 @@ def _build_browser_env() -> dict: env = hermes_subprocess_env(inherit_credentials=False) env.update({k: os.environ[k] for k in _BROWSER_PASSTHROUGH_KEYS if k in os.environ}) - return env + # Headed Chromium opens on this profile's Bot Desktop when one is running (human can take it over). + from tools.bot_desktop.runtime import desktop_env as _bot_desktop_env + return _bot_desktop_env(env) try: diff --git a/tools/browser_tool_real_profile.py b/tools/browser_tool_real_profile.py index 103bc7a64c..dfbe6f0e4a 100644 --- a/tools/browser_tool_real_profile.py +++ b/tools/browser_tool_real_profile.py @@ -163,12 +163,13 @@ def _launch_real_profile_chrome(real_binary: str, copy_dir: str) -> Tuple[Option except OSError: pass chrome_argv = [real_binary, f"--user-data-dir={copy_dir}", *_REAL_PROFILE_CHROME_FLAGS] - _has_display = bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")) + browser_env = _bt._build_browser_env() # carries the Bot Desktop DISPLAY when one is running + _has_display = bool(browser_env.get("DISPLAY") or browser_env.get("WAYLAND_DISPLAY")) if not (_cloud._is_headed_mode() and (_has_display or not sys.platform.startswith("linux"))): chrome_argv.append("--headless=new") try: chrome_proc = subprocess.Popen(chrome_argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - stdin=subprocess.DEVNULL, start_new_session=True, env=_bt._build_browser_env()) + stdin=subprocess.DEVNULL, start_new_session=True, env=browser_env) except (subprocess.SubprocessError, OSError) as e: return None, f"{_RP}the launch failed: {e}" _bt._real_profile_chrome_procs.append(chrome_proc) diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py index 9be4b19cb7..812d755609 100644 --- a/tools/computer_use/cua_backend.py +++ b/tools/computer_use/cua_backend.py @@ -106,6 +106,10 @@ def cua_driver_child_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, the child has a Wayland display). Used by every spawn site (MCP, status, doctor, install) so CLI and gateway runtimes share one policy.""" env = dict(os.environ if base_env is None else base_env) + # A running Bot Desktop for this profile owns the agent's screen: DISPLAY/XAUTHORITY/DBUS point there so + # cua-driver never acts on a seat the human is sitting at (#90374 class) and headless hosts get a display. + from tools.bot_desktop.runtime import desktop_env as _bot_desktop_env + env = _bot_desktop_env(env) if _cua_telemetry_disabled(): env[_CUA_TELEMETRY_ENV_VAR] = "0" if sys.platform == "linux" and env.get("WAYLAND_DISPLAY") and bool(_computer_use_cfg().get("native_wayland", False)): diff --git a/tools/computer_use/handoff.py b/tools/computer_use/handoff.py new file mode 100644 index 0000000000..6d04f048de --- /dev/null +++ b/tools/computer_use/handoff.py @@ -0,0 +1,38 @@ +"""Human handoff actions for ``computer_use`` on a Bot Desktop: ``request_handoff`` asks the person to +take over the screen (login, 2FA, CAPTCHA, payment) and ``wait_for_human`` blocks until control is +back. Both are answered without touching cua-driver, so a human typing a credential is never +captured. + +The notification itself (Desktop pane badge, Telegram/Discord message) is emitted by the gateway's +lease listener; this module only records intent on the lease and waits. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict + +from tools.bot_desktop import lease as _lease + +HANDOFF_ACTIONS = frozenset({"request_handoff", "wait_for_human"}) +_DEFAULT_WAIT_SECONDS = 600.0 +_MAX_WAIT_SECONDS = 1800.0 + + +def handle_handoff(action: str, args: Dict[str, Any]) -> str: + if action == "request_handoff": + reason = str(args.get("reason") or "The agent needs you to complete a step on its screen.").strip() + _lease.request_handoff(reason) + return json.dumps({ + "ok": True, "action": action, "state": _lease.get().as_dict(), + "next": "The user has been asked to open this bot's Desktop pane and take over. Call " + "computer_use action='wait_for_human' to block until they hand control back, then " + "re-capture before continuing — the screen state is whatever they left."}) + timeout = min(_MAX_WAIT_SECONDS, max(1.0, float(args.get("seconds") or _DEFAULT_WAIT_SECONDS))) + released = _lease.wait_for_release(timeout=timeout) + state = _lease.get().as_dict() + if released: + return json.dumps({"ok": True, "action": action, "state": state, + "next": "Control is back with you. Take a fresh capture; do not assume prior state."}) + return json.dumps({"ok": False, "action": action, "code": "still_waiting", "state": state, + "error": f"No hand-back within {timeout:.0f}s. Call wait_for_human again or ask the user in chat."}) diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index a38943c142..e984bfe3f6 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -32,12 +32,16 @@ _PROPERTIES: Dict[str, Any] = { "list_apps", "list_windows", "focus_app", + "request_handoff", + "wait_for_human", ], "description": ( "Which action to perform. `capture` is free (no side effects). All other actions " "require approval unless auto-approved. Use `set_value` for select/popup elements and " "sliders — it selects the matching option directly without opening the native menu (no " - "focus steal)." + "focus steal). When a login, 2FA, CAPTCHA or payment step needs the human, call " + "`request_handoff` (with `reason`) so they can take over this screen from the Hermes " + "Desktop app, then `wait_for_human`; while they hold control every other action is refused." ), }, "mode": { @@ -141,13 +145,14 @@ _PROPERTIES: Dict[str, Any] = { ), }, "text": {"type": "string", "description": "Text to type (respects the current layout)."}, + "reason": {"type": "string", "description": "request_handoff: one sentence telling the human what to do on the screen (e.g. 'Sign in to LinkedIn and complete 2FA')."}, "keys": { "type": "string", "description": ( "Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', 'escape', 'tab'. Use '+' to combine." ), }, - "seconds": {"type": "number", "description": "Seconds to wait. Max 30."}, + "seconds": {"type": "number", "description": "wait: seconds to pause (max 30). wait_for_human: how long to block for the hand-back (default 600, max 1800)."}, "raise_window": { "type": "boolean", "description": ( diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index afb000913f..e082c0b4c0 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -248,6 +248,17 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: if not action: return json.dumps({"error": "missing `action`"}) session_id = str(kwargs.get("session_id") or "") # approval-state / daemon-mode isolation key + from tools.computer_use.handoff import HANDOFF_ACTIONS, handle_handoff + if action in HANDOFF_ACTIONS: + return handle_handoff(action, args) + # Bot Desktop lease: while a human drives the screen every action, capture included, is refused. + from tools.bot_desktop import lease as _bd_lease + from tools.bot_desktop.runtime import ensure_started_for_tool as _bd_ensure_started + try: + _bd_lease.assert_agent_may_act() + except _bd_lease.HumanHasControl as e: + return json.dumps({"ok": False, "action": action, "code": "human_has_control", "error": str(e)}) + _bd_ensure_started() # headless gateway: bring the profile's screen up before the backend probes DISPLAY if (err := _reject_unsafe(action, args)) is not None: return err scopes = ([action] if action in _ACTIONS and _ACTIONS[action].destructive else []) + ( diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py new file mode 100644 index 0000000000..9f9aede010 --- /dev/null +++ b/tui_gateway/methods_display.py @@ -0,0 +1,121 @@ +"""Bot Desktop JSON-RPC handlers: the Desktop app's door to a profile's headless screen. + +``display.status`` reports runtime + lease; ``display.start`` / ``display.stop`` manage the Xvnc/Xfce +process; ``display.observe`` mints a single-use ticket the renderer redeems on ``/api/display/ws`` +(``hermes_cli.web_routers.display``) to stream raw RFB; ``display.lease.acquire`` / ``release`` are +Take over / Hand back. Every handler is profile-scoped so a multiplexed gateway answers for the bot +the pane is looking at. Lease transitions fan out as the global ``display.lease`` event so every +connected client repaints (badge on the bot row, red border on the viewer, agent handoff prompt). + +Bodies are rebound onto server.py's globals (method_ctx.bind_module) and reference them bare. +""" + +import logging +import threading + +from .method_ctx import HandlerRegistry, bind_module + +logger = logging.getLogger(__name__) +_registry = HandlerRegistry() +method = _registry.method +_profile_scoped = _registry.profile_scoped + +_DISPLAY_ERR = 5300 +_lease_listener_installed = threading.Event() + + +def _display_snapshot() -> dict: + from hermes_constants import hermes_home_key + from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime + st = _bd_runtime.status() + return {**st.as_dict(), "lease": _bd_lease.get().as_dict(), "profile_key": hermes_home_key()} + + +def _install_lease_listener() -> None: + """Once per process: broadcast every lease change to all connected clients.""" + if _lease_listener_installed.is_set(): + return + _lease_listener_installed.set() + from tools.bot_desktop import lease as _bd_lease + + def _on_change(profile_key: str, lease) -> None: + _broadcast_global_event("display.lease", {"profile_key": profile_key, "lease": lease.as_dict()}) + _bd_lease.on_change(_on_change) + + +@method("display.status") +@_profile_scoped +def _(rid, params: dict) -> dict: + _install_lease_listener() + try: + return _ok(rid, _display_snapshot()) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.start") +@_profile_scoped +def _(rid, params: dict) -> dict: + _install_lease_listener() + from tools.bot_desktop import runtime as _bd_runtime + try: + _bd_runtime.start() + return _ok(rid, _display_snapshot()) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.stop") +@_profile_scoped +def _(rid, params: dict) -> dict: + from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime + try: + _bd_lease.release() + stopped = _bd_runtime.stop() + return _ok(rid, {**_display_snapshot(), "stopped": stopped}) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.observe") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Mint a single-use, 30 s ticket for ``/api/display/ws``. The ticket carries the profile home so + the bridge dials THIS profile's RFB socket, and the viewer id so the lease can name the holder.""" + from hermes_constants import get_hermes_home + from hermes_cli.dashboard_auth.ws_tickets import mint_ticket + from tools.bot_desktop import runtime as _bd_runtime + try: + if _bd_runtime.rfb_socket_path() is None: + return _err(rid, _DISPLAY_ERR, "this profile's Bot Desktop is not running; call display.start first") + viewer_id = str(params.get("viewer_id") or "").strip() or f"viewer-{rid}" + ticket = mint_ticket(user_id=f"display:{viewer_id}", provider="bot-desktop", + extra={"hermes_home": str(get_hermes_home()), "viewer_id": viewer_id}) + return _ok(rid, {"ticket": ticket, "path": "/api/display/ws", "viewer_id": viewer_id, + **_display_snapshot()}) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.lease.acquire") +@_profile_scoped +def _(rid, params: dict) -> dict: + from tools.bot_desktop import lease as _bd_lease + viewer_id = str(params.get("viewer_id") or "").strip() + if not viewer_id: + return _err(rid, _DISPLAY_ERR, "viewer_id required") + lease = _bd_lease.acquire(viewer_id, reason=str(params.get("reason") or "")) + return _ok(rid, {"lease": lease.as_dict()}) + + +@method("display.lease.release") +@_profile_scoped +def _(rid, params: dict) -> dict: + from tools.bot_desktop import lease as _bd_lease + viewer_id = str(params.get("viewer_id") or "").strip() or None + lease = _bd_lease.release(viewer_id) + return _ok(rid, {"lease": lease.as_dict()}) + + +def register(server) -> None: + bind_module(globals(), server, skip=("_",)) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index cbfe41572d..bb99b563fa 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -3230,7 +3230,7 @@ from . import ( # noqa: E402 methods_projects as _methods_projects, methods_session_foreign as _methods_session_foreign, methods_session_control as _methods_session_control, methods_subagents as _methods_subagents, methods_vault as _methods_vault, methods_free_tier as _methods_free_tier, - methods_connectors as _methods_connectors) + methods_connectors as _methods_connectors, methods_display as _methods_display) for _m in ( _session_transports, _session_reaper, _session_lifecycle, _session_workdir, _compute_host_bridge, _model_switch, @@ -3240,6 +3240,7 @@ for _m in ( _methods_browser_control, _methods_session, _methods_prompt, _methods_config, _methods_config_set, _methods_complete, _methods_tools, _methods_profiles, _methods_images, _methods_bot_relay, _prompt_turn, _billing_view, _methods_projects, _methods_session_foreign, - _methods_session_control, _methods_subagents, _methods_vault, _methods_free_tier, _methods_connectors): + _methods_session_control, _methods_subagents, _methods_vault, _methods_free_tier, _methods_connectors, + _methods_display): _m.register(sys.modules[__name__]) del _m diff --git a/website/docs/user-guide/features/bot-screen.md b/website/docs/user-guide/features/bot-screen.md new file mode 100644 index 0000000000..6079927774 --- /dev/null +++ b/website/docs/user-guide/features/bot-screen.md @@ -0,0 +1,121 @@ +--- +title: Bot Screen +sidebar_position: 17 +--- + +# Bot Screen + +On a headless Linux gateway host (a server, a cloud VM, Hermes Cloud) each bot +gets its **own desktop**: an Xfce screen the bot's `computer_use` and headed +browser act on, streamed live into Hermes Desktop. Watch what the bot does, +**take over** when it hits a login, 2FA prompt, CAPTCHA or payment step, then +**hand control back** and let it continue with the session you just signed in +to. The bot keeps working after you close the app or turn off your laptop; the +screen lives on the gateway host, not on your machine. + +Every Hermes profile ("bot") has its own screen, its own browser profile and +its own cookies. Screens are work surfaces, not security boundaries: the bots +share the host's user account, files and network (the same model as other +hosted-agent products). + +## Requirements + +- The gateway host runs Linux. macOS and Windows hosts already have a real + display; the pane is not offered there. +- TigerVNC's `Xvnc` and the Xfce core components are installed on the host. + Hermes Desktop and `hermes computer-use screen status` print the exact + package-manager line when they are missing; `hermes computer-use screen + install` runs it for you: + + | Distro | Packages | + |---|---| + | Debian / Ubuntu | `tigervnc-standalone-server xfce4-panel xfwm4 xfdesktop4 xfce4-settings xfce4-terminal dbus-x11 x11-xserver-utils x11-utils xauth fonts-dejavu-core` | + | Fedora | `tigervnc-server-minimal xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal dbus-x11 xorg-x11-server-utils xorg-x11-utils xorg-x11-xauth dejavu-sans-fonts` | + | Arch | `tigervnc xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal xorg-xsetroot xorg-xset xorg-xdpyinfo xorg-xauth xorg-setxkbmap ttf-dejavu` | + + Deliberately **not** the `xfce4` metapackage: it pulls in the screensaver, + power manager and polkit agent that lock or prompt a headless desktop. +- [Computer Use](./computer-use.md) enabled for the bot (cua-driver installed). + +## Using it + +1. In Hermes Desktop open **Bots**, right-click a bot, choose **Open Screen**. + The first time, click **Start screen** (or leave `bot_desktop.auto_start` on, + the default: the screen starts on the bot's first `computer_use` call when the + host has no display). A headed browser opens on the screen once it is running. +2. The pane streams the bot's desktop. The chip in the header says who is in + control: **Bot is in control** by default. +3. Click **Take over**. The border turns red, your keyboard and mouse now drive + the bot's screen. Sign in, solve the CAPTCHA, approve the payment. +4. Click **Hand back**. The bot regains control and re-captures the screen + before continuing. Closing the pane also hands control back. + +While you hold control the bot's `computer_use` calls (captures included) are +refused with `human_has_control`; the bot never sees what you type. + +The bot can ask for you: when it recognises a login or verification step it +calls `computer_use` with `action: "request_handoff"` and a reason, the pane +shows **Bot needs you**, and the bot blocks in `action: "wait_for_human"` until +you hand back. Your Telegram/Discord chat with the bot gets the same request. + +Two viewers on one screen: the most recent **Take over** wins; the previous +controller drops back to watching. + +## Browser sessions that survive the handoff + +The bot's headed Chromium (`browser.headed: true` or a real-profile session) +opens on the bot's screen and keeps one persistent profile per bot. What you +sign in to during a takeover is what the bot uses afterwards, and in every later +session for that bot, until the site itself expires the login. + +## CLI + +```bash +hermes computer-use screen status # installed? running? who holds control? +hermes computer-use screen start # start this profile's screen +hermes computer-use screen stop # stop it (hands control back first) +hermes computer-use screen install [-y] # apt/dnf/pacman the packages +hermes -p research computer-use screen start # another bot's screen +``` + +## Configuration + +```yaml +bot_desktop: + geometry: "1440x900" # screen size; the viewer scales to fit the pane + auto_start: true # start on the first computer_use call when the host has no display +``` + +State lives under `/bot-desktop/` per profile (RFB Unix socket, +Xauthority, launcher log, per-profile xfconf). + +## How it works + +- **TigerVNC `Xvnc`** is the X server and the RFB server in one process, per + profile, listening only on a `0600` Unix socket. No TCP port, no VNC + password: the gateway is the only process that can reach it. +- **Xfce** starts component-wise (`xfsettingsd`, `xfwm4 --compositor=off`, + `xfdesktop`, `xfce4-panel`) under a private D-Bus session, without + `xfce4-session`, so nothing tries to lock the screen or reach `logind`. +- **Hermes Desktop** bundles noVNC. It asks the gateway for a single-use ticket + (`display.observe`) over its normal authenticated connection and opens a + sibling WebSocket to `/api/display/ws`; the gateway splices the RFB stream + through. Nothing new is exposed; the pane works over local, SSH, URL+token + and Hermes Cloud connections alike. +- **Control lease.** The gateway drops keyboard, pointer and clipboard messages + from any viewer that does not hold the lease, at the RFB byte level; noVNC's + view-only flag is only the UI hint. The same lease gates `computer_use`. +- **Display binding.** The launcher publishes `DISPLAY`, `XAUTHORITY` and the + D-Bus address; every cua-driver and headed-browser spawn for that profile + inherits them, so the bot never acts on a display a human is sitting at. + +## Troubleshooting + +- **"Screen packages missing"** — run the printed install line on the gateway + host (not on the machine running Hermes Desktop). +- **Screen starts then stops** — read `/bot-desktop/launcher.log`. +- **Typing produces wrong characters** — the screen uses a US keymap so RFB + keysyms and cua-driver agree; change it with `setxkbmap` on that `DISPLAY` + if you need another layout. +- **Bot says `human_has_control` after you left** — click **Hand back** in the + pane, or `hermes computer-use screen stop` / `start`. diff --git a/website/docs/user-guide/features/computer-use.md b/website/docs/user-guide/features/computer-use.md index 1148e2949a..3bb53d2d20 100644 --- a/website/docs/user-guide/features/computer-use.md +++ b/website/docs/user-guide/features/computer-use.md @@ -410,10 +410,11 @@ of screenshot context, not ~600K. [windows-ssh](https://cua.ai/docs/how-to-guides/driver/windows-ssh) has the recipe. - **Linux** requires a reachable display server. Headless servers - need Xvfb (`Xvfb :99 -screen 0 1920x1080x24`) before - `computer_use` can capture or inject events. Pure Wayland sessions - need an XWayland bridge for screen capture (cua-driver's Wayland - inject path handles input independently). + get one from [Bot Screen](./bot-screen.md): a per-profile Xfce + desktop over TigerVNC that Hermes starts on first use and streams + into Hermes Desktop, where you can take over for logins and 2FA. + Pure Wayland sessions need an XWayland bridge for screen capture + (cua-driver's Wayland inject path handles input independently). For cross-platform GUI automation without the desktop overhead (and without TCC / Session 0 / X11 setup), the `browser` toolset uses a diff --git a/website/sidebars.ts b/website/sidebars.ts index 55d1ed8016..621205071a 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -121,6 +121,7 @@ const sidebars: SidebarsConfig = { 'user-guide/features/browser', 'user-guide/features/credential-vault', 'user-guide/features/computer-use', + 'user-guide/features/bot-screen', 'user-guide/features/vision', 'user-guide/features/image-generation', 'user-guide/features/spotify',