diff --git a/packages/client/src/solid/data.ts b/packages/client/src/solid/data.ts index a1c8805bc26..8b6d247dc96 100644 --- a/packages/client/src/solid/data.ts +++ b/packages/client/src/solid/data.ts @@ -84,9 +84,10 @@ export type CreateDataInput = { readonly onError?: (error: unknown) => void /** * Drops the payload of inline `data:` files in tool results before they enter the store. For clients that never - * render tool-result files: a session full of screenshots otherwise keeps hundreds of MB of base64 alive. + * render tool-result files: a session full of screenshots otherwise keeps hundreds of MB of base64 alive. A function + * is read for each message and event, so a client can follow a setting that changes while it runs. */ - readonly omitToolFileData?: boolean + readonly omitToolFileData?: boolean | (() => boolean) } const messagePageLimit = 20 @@ -182,8 +183,23 @@ function omitFileData(content: [ToolContent, ...ToolContent[]]): [ToolContent, . return [omit(content[0]), ...content.slice(1).map(omit)] } +const hasFileData = (part: Extract["content"][number]) => + part.type === "tool" && + "content" in part.state && + !!part.state.content?.some((item) => item.type === "file" && item.uri.startsWith("data:")) + +// A tool file whose `data:` payload `omitFileData` emptied. +const isDroppedFile = (item: ToolContent) => item.type === "file" && item.uri === `data:${item.mime};base64,` + +const hasDroppedFileData = (message: SessionMessageInfo) => + message.type === "assistant" && + message.content.some( + (part) => part.type === "tool" && "content" in part.state && !!part.state.content?.some(isDroppedFile), + ) + +// Returns the same message when there is nothing to drop, so the store keeps its objects. function omitMessageFileData(message: SessionMessageInfo): SessionMessageInfo { - if (message.type !== "assistant") return message + if (message.type !== "assistant" || !message.content.some(hasFileData)) return message return { ...message, content: message.content.map((part) => @@ -268,16 +284,35 @@ export function createData(config: CreateDataInput) { function refresh(load: () => Promise) { if (disposed || (config.connection && config.connection.status() !== "connected")) return - void load().catch((error) => { - if (disposed || (config.connection && config.connection.status() !== "connected")) return - if (isSessionNotFoundError(error)) { - markDeleted(error.sessionID) - return - } - if (isMessageNotFoundError(error)) return - if (config.onError) return config.onError(error) - console.error("Failed to refresh client data", error) - }) + void load().catch(refreshFailed) + } + + // Copies back only the emptied tool files: live events may have moved the message on since the server read it. + function restoreMessageFiles(sessionID: string, messageID: string) { + return api() + .session.message.get({ sessionID, messageID }) + .then((item) => { + if (item.type !== "assistant" || omitFiles()) return + message.editAssistant(sessionID, item.id, (assistant) => { + assistant.content.forEach((part) => { + if (part.type !== "tool" || !("content" in part.state) || !part.state.content?.some(isDroppedFile)) return + const fetched = item.content.findLast((other) => other.type === "tool" && other.id === part.id) + if (fetched?.type === "tool" && "content" in fetched.state && fetched.state.content) + part.state.content = fetched.state.content + }) + }) + }) + } + + function refreshFailed(error: unknown) { + if (disposed || (config.connection && config.connection.status() !== "connected")) return + if (isSessionNotFoundError(error)) { + markDeleted(error.sessionID) + return + } + if (isMessageNotFoundError(error)) return + if (config.onError) return config.onError(error) + console.error("Failed to refresh client data", error) } // Runs `load` once a burst of same-key events goes quiet, so N events cost one refetch. @@ -319,7 +354,9 @@ export function createData(config: CreateDataInput) { Object.values(store.session.info).toSorted((a, b) => b.time.updated - a.time.updated), ) const messageIndex = new Map>() - const stored = (message: SessionMessageInfo) => (config.omitToolFileData ? omitMessageFileData(message) : message) + const omitFiles = () => + typeof config.omitToolFileData === "function" ? config.omitToolFileData() : config.omitToolFileData === true + const stored = (message: SessionMessageInfo) => (omitFiles() ? omitMessageFileData(message) : message) const sync = createSync() let activeUpdates: Map | undefined const pendingUpdates = new Map>() @@ -1103,7 +1140,7 @@ export function createData(config: CreateDataInput) { status: "completed", input: tool.state.input, metadata: event.data.metadata, - content: config.omitToolFileData ? omitFileData(event.data.content) : [...event.data.content], + content: omitFiles() ? omitFileData(event.data.content) : [...event.data.content], } tool.executed = event.data.executed || tool.executed === true if (event.data.native) tool.native = { ...tool.native, ...event.data.native } @@ -1119,7 +1156,7 @@ export function createData(config: CreateDataInput) { input: typeof tool.state.input === "string" ? {} : tool.state.input, metadata: event.data.metadata, content: - config.omitToolFileData && event.data.content ? omitFileData(event.data.content) : event.data.content, + omitFiles() && event.data.content ? omitFileData(event.data.content) : event.data.content, } tool.executed = event.data.executed || tool.executed === true if (event.data.native) tool.native = { ...tool.native, ...event.data.native } @@ -1823,6 +1860,23 @@ export function createData(config: CreateDataInput) { const position = messageIndex.get(sessionID)?.get(messageID) return position === undefined ? undefined : messages?.[position] }, + /** Re-reads every loaded message whose tool file data was dropped, for a client that starts showing tool files. */ + restoreFiles() { + const sessions = Object.entries(store.session.message).flatMap(([sessionID, messages]) => { + const ids = (messages ?? []).filter(hasDroppedFileData).map((item) => item.id) + return ids.length === 0 ? [] : [{ sessionID, ids }] + }) + // Each session's messages are re-read one at a time, so many screenshots don't fan out at once. + return Promise.all( + sessions.map((session) => + session.ids.reduce( + (previous, messageID) => + previous.then(() => restoreMessageFiles(session.sessionID, messageID).catch(refreshFailed)), + Promise.resolve(), + ), + ), + ) + }, sync(sessionID: string) { return sync.run(`session.message:${sessionID}`, async () => { const started = deletionEpoch diff --git a/packages/client/test/solid-data.test.ts b/packages/client/test/solid-data.test.ts index 69dea99a245..b040e634d59 100644 --- a/packages/client/test/solid-data.test.ts +++ b/packages/client/test/solid-data.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test" import { getEventListeners } from "node:events" -import { createRoot } from "solid-js" +import { createRoot, createSignal } from "solid-js" import { createData, type CreateDataInput } from "../src/solid" import { OpenCode, type ModelInfo, type OpenCodeEvent, type Project, type SessionInfo } from "../src/promise" @@ -330,6 +330,241 @@ test("omits inline tool-result file payloads from read and live tool results whe } }) +test("reads a function omitToolFileData for each live tool result", async () => { + const listeners = new Set[0]>() + const screenshot = `data:image/png;base64,${"A".repeat(4096)}` + const running = (id: string) => ({ + type: "tool", + id, + name: "execute", + time: { created: 1, ran: 1 }, + state: { status: "running", input: {}, metadata: {} }, + }) + const api = OpenCode.make({ + baseUrl: "http://opencode.local", + fetch: async () => + Response.json({ + data: [ + { + id: "msg_assistant", + type: "assistant", + agent: "build", + model: { providerID: "provider", id: "model" }, + time: { created: 1 }, + content: [running("call_off"), running("call_on")], + }, + ], + cursor: {}, + }), + }) + const [omit, setOmit] = createSignal(true) + const setup = createRoot((dispose) => ({ + data: createData({ + api: () => api, + directory: "/project", + omitToolFileData: omit, + event: { + on: () => () => {}, + listen(handler) { + listeners.add(handler) + return () => listeners.delete(handler) + }, + }, + }), + dispose, + })) + const succeed = (id: string, seq: number) => { + const event: OpenCodeEvent = { + id: `evt_${id}`, + created: 2, + type: "session.tool.success", + durable: { aggregateID: "ses_toggle", seq, version: 2 }, + data: { + sessionID: "ses_toggle", + assistantMessageID: "msg_assistant", + id, + executed: true, + content: [{ type: "file", uri: screenshot, mime: "image/png" }], + }, + } + listeners.forEach((listener) => listener({ name: event.type, details: event })) + } + try { + await setup.data.session.message.sync("ses_toggle") + succeed("call_off", 1) + setOmit(false) + succeed("call_on", 2) + + expect(setup.data.session.message.get("ses_toggle", "msg_assistant")).toMatchObject({ + content: [ + { state: { content: [{ type: "file", uri: "data:image/png;base64,", mime: "image/png" }] } }, + { state: { content: [{ type: "file", uri: screenshot, mime: "image/png" }] } }, + ], + }) + } finally { + setup.dispose() + } +}) + +test("restores dropped tool files from the server once they are shown again", async () => { + const listeners = new Set[0]>() + const screenshot = `data:image/png;base64,${"A".repeat(4096)}` + const assistant = (state: object) => ({ + id: "msg_assistant", + type: "assistant", + agent: "build", + model: { providerID: "provider", id: "model" }, + time: { created: 1 }, + content: [{ type: "tool", id: "call_shot", name: "execute", time: { created: 1, ran: 1 }, state }], + }) + const requests: string[] = [] + const api = OpenCode.make({ + baseUrl: "http://opencode.local", + fetch: async (input) => { + const url = new URL(input instanceof Request ? input.url : String(input)) + requests.push(url.pathname) + if (url.pathname.endsWith("/message/msg_assistant")) + return Response.json({ + data: assistant({ + status: "completed", + input: {}, + content: [{ type: "file", uri: screenshot, mime: "image/png" }], + }), + }) + return Response.json({ data: [assistant({ status: "running", input: {}, metadata: {} })], cursor: {} }) + }, + }) + const [omit, setOmit] = createSignal(true) + const setup = createRoot((dispose) => ({ + data: createData({ + api: () => api, + directory: "/project", + omitToolFileData: omit, + event: { + on: () => () => {}, + listen(handler) { + listeners.add(handler) + return () => listeners.delete(handler) + }, + }, + }), + dispose, + })) + try { + await setup.data.session.message.sync("ses_restore") + const success: OpenCodeEvent = { + id: "evt_shot", + created: 2, + type: "session.tool.success", + durable: { aggregateID: "ses_restore", seq: 1, version: 2 }, + data: { + sessionID: "ses_restore", + assistantMessageID: "msg_assistant", + id: "call_shot", + executed: true, + content: [{ type: "file", uri: screenshot, mime: "image/png" }], + }, + } + listeners.forEach((listener) => listener({ name: success.type, details: success })) + expect(setup.data.session.message.get("ses_restore", "msg_assistant")).toMatchObject({ + content: [{ state: { content: [{ uri: "data:image/png;base64," }] } }], + }) + + setOmit(false) + await setup.data.session.message.restoreFiles() + + expect(requests.filter((path) => path.endsWith("/message/msg_assistant"))).toHaveLength(1) + expect(setup.data.session.message.get("ses_restore", "msg_assistant")).toMatchObject({ + content: [{ state: { content: [{ type: "file", uri: screenshot, mime: "image/png" }] } }], + }) + // Nothing is left to restore, so a second call reads nothing. + await setup.data.session.message.restoreFiles() + expect(requests.filter((path) => path.endsWith("/message/msg_assistant"))).toHaveLength(1) + } finally { + setup.dispose() + } +}) + +test("restoring dropped tool files keeps live progress made while the server read was in flight", async () => { + const listeners = new Set[0]>() + const screenshot = `data:image/png;base64,${"A".repeat(4096)}` + const tool = (id: string, state: object) => ({ type: "tool", id, name: "execute", time: { created: 1, ran: 1 }, state }) + const running = { status: "running", input: {}, metadata: {} } + const assistant = (content: object[]) => ({ + id: "msg_assistant", + type: "assistant", + agent: "build", + model: { providerID: "provider", id: "model" }, + time: { created: 1 }, + content, + }) + const read = Promise.withResolvers() + const api = OpenCode.make({ + baseUrl: "http://opencode.local", + fetch: async (input) => { + const url = new URL(input instanceof Request ? input.url : String(input)) + if (url.pathname.endsWith("/message/msg_assistant")) return read.promise + return Response.json({ data: [assistant([tool("call_shot", running), tool("call_work", running)])], cursor: {} }) + }, + }) + const [omit, setOmit] = createSignal(true) + const setup = createRoot((dispose) => ({ + data: createData({ + api: () => api, + directory: "/project", + omitToolFileData: omit, + event: { + on: () => () => {}, + listen(handler) { + listeners.add(handler) + return () => listeners.delete(handler) + }, + }, + }), + dispose, + })) + const succeed = ( + id: string, + seq: number, + content: Extract["data"]["content"], + ) => { + const event: OpenCodeEvent = { + id: `evt_${id}`, + created: 2, + type: "session.tool.success", + durable: { aggregateID: "ses_inflight", seq, version: 2 }, + data: { sessionID: "ses_inflight", assistantMessageID: "msg_assistant", id, executed: true, content }, + } + listeners.forEach((listener) => listener({ name: event.type, details: event })) + } + try { + await setup.data.session.message.sync("ses_inflight") + succeed("call_shot", 1, [{ type: "file", uri: screenshot, mime: "image/png" }]) + setOmit(false) + const restoring = setup.data.session.message.restoreFiles() + // The second tool finishes while the read is in flight; the server's snapshot predates it. + succeed("call_work", 2, [{ type: "text", text: "done" }]) + read.resolve( + Response.json({ + data: assistant([ + tool("call_shot", { status: "completed", input: {}, content: [{ type: "file", uri: screenshot, mime: "image/png" }] }), + tool("call_work", running), + ]), + }), + ) + await restoring + + expect(setup.data.session.message.get("ses_inflight", "msg_assistant")).toMatchObject({ + content: [ + { id: "call_shot", state: { status: "completed", content: [{ type: "file", uri: screenshot }] } }, + { id: "call_work", state: { status: "completed", content: [{ type: "text", text: "done" }] } }, + ], + }) + } finally { + setup.dispose() + } +}) + test("refreshes a loaded session's execution start when it starts running", async () => { const listeners = new Set[0]>() const api = OpenCode.make({ diff --git a/packages/tui/src/context/data.tsx b/packages/tui/src/context/data.tsx index 8c7668f9a77..f9387658c2e 100644 --- a/packages/tui/src/context/data.tsx +++ b/packages/tui/src/context/data.tsx @@ -1,9 +1,11 @@ import { createData } from "@opencode/client/solid" import { withMediaRefs, type LocationRef } from "@opencode/client" import type { Plugin } from "@opencode/plugin/tui" +import { createEffect, on } from "solid-js" import { createStore } from "solid-js/store" import { createSimpleContext } from "./helper" import { useClient } from "./client" +import { useConfig } from "../config" export { locationKey } from "@opencode/client/solid" export type { FormWithLocation } from "@opencode/client/solid" @@ -12,13 +14,27 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({ name: "Data", init: (props: { directory: string }) => { const client = useClient() + const config = useConfig() const data = createData({ // Only the transcript store reads stored media as `media:` refs; plugins and other callers keep inline bytes. api: () => withMediaRefs(client.api), event: client.event, connection: client.connection, directory: props.directory, + // Live tool events carry screenshots as inline base64, which open tabs would otherwise hold for the whole run. + // Read per event, so toggling previews applies to new results without a restart. + omitToolFileData: () => !config.data.session?.image_preview, }) + // Turning previews on brings back the screenshots dropped while they were off. + createEffect( + on( + () => config.data.session?.image_preview === true, + (preview) => { + if (preview) void data.session.message.restoreFiles() + }, + { defer: true }, + ), + ) data satisfies Plugin.Context["data"] const [generatingTitles, setGeneratingTitles] = createStore>({}) return { diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index b8b4262ba88..b5b85321717 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -2728,7 +2728,8 @@ function inlineToolImages(part: SessionMessageAssistantTool) { return toolDisplayContent(part.state).flatMap((content) => content.type === "file" && content.mime.startsWith("image/") && - (content.uri.startsWith("data:image/") || mediaHashFromUri(content.uri)) + // An emptied `data:` URI is a payload the store dropped while previews were off. + ((content.uri.startsWith("data:image/") && !content.uri.endsWith(",")) || mediaHashFromUri(content.uri)) ? [{ uri: content.uri }] : [], )