diff --git a/packages/app/src/runtime/server/runtime.tsx b/packages/app/src/runtime/server/runtime.tsx index 296cd7889d6..98b86cb2a2a 100644 --- a/packages/app/src/runtime/server/runtime.tsx +++ b/packages/app/src/runtime/server/runtime.tsx @@ -198,6 +198,8 @@ function createServerController( }, connection: sdk.connection, directory: "", + // The timeline shows tool output as text only, so screenshots in tool results never need their bytes here. + omitToolFileData: true, onError(error) { showToast({ variant: "error", diff --git a/packages/client/src/solid/data.ts b/packages/client/src/solid/data.ts index 61c14869f00..a77b410e575 100644 --- a/packages/client/src/solid/data.ts +++ b/packages/client/src/solid/data.ts @@ -34,6 +34,7 @@ import type { SessionInboxCompaction, ShellInfo, SkillInfo, + ToolContent, VcsInfo, OpenCodeEvent, OpenCodeClient, @@ -78,6 +79,11 @@ export type CreateDataInput = { } /** Receives failed event-driven reads. Explicit reads still reject to their caller. */ readonly onError?: (error: unknown) => void + /** + * Drops the payload of inline `data:` files in tool results before they enter the store. For clients that never + * render tool-result files: a session full of screenshots otherwise keeps hundreds of MB of base64 alive. + */ + readonly omitToolFileData?: boolean } const messagePageLimit = 20 @@ -161,6 +167,27 @@ function optimisticFiles(files: SessionPromptInput["files"]) { return items?.length ? items : undefined } +// Keeps the `data:;base64,` header so the item still says what it was. +function omitFileData(content: [ToolContent, ...ToolContent[]]): [ToolContent, ...ToolContent[]] { + const omit = (item: ToolContent): ToolContent => + item.type === "file" && item.uri.startsWith("data:") + ? { ...item, uri: item.uri.slice(0, item.uri.indexOf(",") + 1) } + : item + return [omit(content[0]), ...content.slice(1).map(omit)] +} + +function omitMessageFileData(message: SessionMessageInfo): SessionMessageInfo { + if (message.type !== "assistant") return message + return { + ...message, + content: message.content.map((part) => + part.type === "tool" && "content" in part.state && part.state.content + ? { ...part, state: { ...part.state, content: omitFileData(part.state.content) } } + : part, + ), + } +} + function formRequestOptions(sessionID: string, ref?: LocationRef) { if (sessionID !== "global" || !ref) return undefined return { @@ -279,6 +306,7 @@ export function createData(config: CreateDataInput) { Object.values(store.session.info).toSorted((a, b) => b.time.updated - a.time.updated), ) const messageIndex = new Map>() + const stored = (message: SessionMessageInfo) => (config.omitToolFileData ? omitMessageFileData(message) : message) const sync = createSync() let activeUpdates: Map | undefined const pendingUpdates = new Map>() @@ -1007,7 +1035,7 @@ export function createData(config: CreateDataInput) { status: "completed", input: tool.state.input, metadata: event.data.metadata, - content: [...event.data.content], + content: config.omitToolFileData ? omitFileData(event.data.content) : [...event.data.content], } tool.executed = event.data.executed || tool.executed === true if (event.data.native) tool.native = { ...tool.native, ...event.data.native } @@ -1022,7 +1050,8 @@ export function createData(config: CreateDataInput) { error: event.data.error, input: typeof tool.state.input === "string" ? {} : tool.state.input, metadata: event.data.metadata, - content: event.data.content, + content: + config.omitToolFileData && event.data.content ? omitFileData(event.data.content) : event.data.content, } tool.executed = event.data.executed || tool.executed === true if (event.data.native) tool.native = { ...tool.native, ...event.data.native } @@ -1710,7 +1739,7 @@ export function createData(config: CreateDataInput) { limit: config.initialMessageLimit?.() ?? messagePageLimit, order: "desc", }) - const fetched = response.data.toReversed() + const fetched = response.data.toReversed().map(stored) // Same protection as the pending sync: a re-fetch racing an // admission must not wipe its local transcript row. const ids = new Set(fetched.map((item) => item.id)) @@ -1778,7 +1807,7 @@ export function createData(config: CreateDataInput) { { signal }, ) if (signal?.aborted) return - fetched.push(...response.data) + fetched.push(...response.data.map(stored)) next = response.cursor.next ?? undefined if (!options?.all) break } while (next) diff --git a/packages/client/test/solid-data.test.ts b/packages/client/test/solid-data.test.ts index 0eac53d0a8f..7a679bb05b6 100644 --- a/packages/client/test/solid-data.test.ts +++ b/packages/client/test/solid-data.test.ts @@ -66,12 +66,19 @@ test("counts every subagent page in a root session's cost", async () => { const url = new URL((input instanceof Request ? input : new Request(input, init)).url) requests.push(`${url.pathname}?${url.searchParams}`) - if (url.pathname === "/api/session/ses_root") return Response.json({ data: { ...session(0), id: "ses_root", cost: 0.5 } }) + if (url.pathname === "/api/session/ses_root") + return Response.json({ data: { ...session(0), id: "ses_root", cost: 0.5 } }) if (url.searchParams.get("cursor") === "page_2") - return Response.json({ data: Array.from({ length: 50 }, (_, index) => child(200 + index)), cursor: { next: "page_3" } }) + return Response.json({ + data: Array.from({ length: 50 }, (_, index) => child(200 + index)), + cursor: { next: "page_3" }, + }) - return Response.json({ data: Array.from({ length: 200 }, (_, index) => child(index)), cursor: { next: "page_2" } }) + return Response.json({ + data: Array.from({ length: 200 }, (_, index) => child(index)), + cursor: { next: "page_2" }, + }) }, }) const setup = createRoot((dispose) => ({ @@ -232,6 +239,97 @@ test("reconciles a stale running tool when execution settles", async () => { } }) +test("omits inline tool-result file payloads from read and live tool results when asked", async () => { + const listeners = new Set[0]>() + const screenshot = `data:image/png;base64,${"A".repeat(4096)}` + const tool = (id: string, state: object) => ({ + type: "tool", + id, + name: "execute", + time: { created: 1, ran: 1 }, + state, + }) + const api = OpenCode.make({ + baseUrl: "http://opencode.local", + fetch: async () => + Response.json({ + data: [ + { + id: "msg_assistant", + type: "assistant", + agent: "build", + model: { providerID: "provider", id: "model" }, + time: { created: 1 }, + content: [ + tool("call_read", { + status: "completed", + input: {}, + content: [ + { type: "text", text: "Took a screenshot" }, + { type: "file", uri: screenshot, mime: "image/png", name: "page.png" }, + { type: "file", uri: "file:///project/notes.md", mime: "text/markdown" }, + ], + }), + tool("call_live", { status: "running", input: {}, metadata: {} }), + ], + }, + ], + cursor: {}, + }), + }) + const setup = createRoot((dispose) => ({ + data: createData({ + api: () => api, + directory: "/project", + omitToolFileData: true, + event: { + on: () => () => {}, + listen(handler) { + listeners.add(handler) + return () => listeners.delete(handler) + }, + }, + }), + dispose, + })) + try { + await setup.data.session.message.sync("ses_refresh") + const success: OpenCodeEvent = { + id: "evt_success", + created: 2, + type: "session.tool.success", + durable: { aggregateID: "ses_refresh", seq: 1, version: 2 }, + data: { + sessionID: "ses_refresh", + assistantMessageID: "msg_assistant", + id: "call_live", + executed: true, + content: [{ type: "file", uri: screenshot, mime: "image/png" }], + }, + } + listeners.forEach((listener) => listener({ name: success.type, details: success })) + + expect(setup.data.session.message.get("ses_refresh", "msg_assistant")).toMatchObject({ + content: [ + { + state: { + content: [ + { type: "text", text: "Took a screenshot" }, + { type: "file", uri: "data:image/png;base64,", mime: "image/png", name: "page.png" }, + { type: "file", uri: "file:///project/notes.md", mime: "text/markdown" }, + ], + }, + }, + { + state: { status: "completed", content: [{ type: "file", uri: "data:image/png;base64,", mime: "image/png" }] }, + }, + ], + }) + } finally { + setup.dispose() + } +}) + test("revalidates after an event overtakes an active session read", async () => { let release!: () => void const gate = new Promise((resolve) => (release = resolve))