fix(tui): drop tool-result image bytes from the transcript store unless image previews are on (#63577)

Co-authored-by: Aiden Cline <aidenpcline@gmail.com>
This commit is contained in:
Adam
2026-11-12 15:17:44 -06:00
committed by GitHub
Unverified
parent fd6a1167b4
commit f179e6e5f0
4 changed files with 324 additions and 18 deletions
+70 -16
View File
@@ -84,9 +84,10 @@ export type CreateDataInput = {
readonly onError?: (error: unknown) => void
/**
* Drops the payload of inline `data:` files in tool results before they enter the store. For clients that never
* render tool-result files: a session full of screenshots otherwise keeps hundreds of MB of base64 alive.
* render tool-result files: a session full of screenshots otherwise keeps hundreds of MB of base64 alive. A function
* is read for each message and event, so a client can follow a setting that changes while it runs.
*/
readonly omitToolFileData?: boolean
readonly omitToolFileData?: boolean | (() => boolean)
}
const messagePageLimit = 20
@@ -182,8 +183,23 @@ function omitFileData(content: [ToolContent, ...ToolContent[]]): [ToolContent, .
return [omit(content[0]), ...content.slice(1).map(omit)]
}
const hasFileData = (part: Extract<SessionMessageInfo, { type: "assistant" }>["content"][number]) =>
part.type === "tool" &&
"content" in part.state &&
!!part.state.content?.some((item) => item.type === "file" && item.uri.startsWith("data:"))
// A tool file whose `data:` payload `omitFileData` emptied.
const isDroppedFile = (item: ToolContent) => item.type === "file" && item.uri === `data:${item.mime};base64,`
const hasDroppedFileData = (message: SessionMessageInfo) =>
message.type === "assistant" &&
message.content.some(
(part) => part.type === "tool" && "content" in part.state && !!part.state.content?.some(isDroppedFile),
)
// Returns the same message when there is nothing to drop, so the store keeps its objects.
function omitMessageFileData(message: SessionMessageInfo): SessionMessageInfo {
if (message.type !== "assistant") return message
if (message.type !== "assistant" || !message.content.some(hasFileData)) return message
return {
...message,
content: message.content.map((part) =>
@@ -268,16 +284,35 @@ export function createData(config: CreateDataInput) {
function refresh(load: () => Promise<unknown>) {
if (disposed || (config.connection && config.connection.status() !== "connected")) return
void load().catch((error) => {
if (disposed || (config.connection && config.connection.status() !== "connected")) return
if (isSessionNotFoundError(error)) {
markDeleted(error.sessionID)
return
}
if (isMessageNotFoundError(error)) return
if (config.onError) return config.onError(error)
console.error("Failed to refresh client data", error)
})
void load().catch(refreshFailed)
}
// Copies back only the emptied tool files: live events may have moved the message on since the server read it.
function restoreMessageFiles(sessionID: string, messageID: string) {
return api()
.session.message.get({ sessionID, messageID })
.then((item) => {
if (item.type !== "assistant" || omitFiles()) return
message.editAssistant(sessionID, item.id, (assistant) => {
assistant.content.forEach((part) => {
if (part.type !== "tool" || !("content" in part.state) || !part.state.content?.some(isDroppedFile)) return
const fetched = item.content.findLast((other) => other.type === "tool" && other.id === part.id)
if (fetched?.type === "tool" && "content" in fetched.state && fetched.state.content)
part.state.content = fetched.state.content
})
})
})
}
function refreshFailed(error: unknown) {
if (disposed || (config.connection && config.connection.status() !== "connected")) return
if (isSessionNotFoundError(error)) {
markDeleted(error.sessionID)
return
}
if (isMessageNotFoundError(error)) return
if (config.onError) return config.onError(error)
console.error("Failed to refresh client data", error)
}
// Runs `load` once a burst of same-key events goes quiet, so N events cost one refetch.
@@ -319,7 +354,9 @@ export function createData(config: CreateDataInput) {
Object.values(store.session.info).toSorted((a, b) => b.time.updated - a.time.updated),
)
const messageIndex = new Map<string, Map<string, number>>()
const stored = (message: SessionMessageInfo) => (config.omitToolFileData ? omitMessageFileData(message) : message)
const omitFiles = () =>
typeof config.omitToolFileData === "function" ? config.omitToolFileData() : config.omitToolFileData === true
const stored = (message: SessionMessageInfo) => (omitFiles() ? omitMessageFileData(message) : message)
const sync = createSync()
let activeUpdates: Map<string, DataSessionStatus | undefined> | undefined
const pendingUpdates = new Map<string, Map<string, SessionInboxInfo | SessionInbox.Delivery | undefined>>()
@@ -1103,7 +1140,7 @@ export function createData(config: CreateDataInput) {
status: "completed",
input: tool.state.input,
metadata: event.data.metadata,
content: config.omitToolFileData ? omitFileData(event.data.content) : [...event.data.content],
content: omitFiles() ? omitFileData(event.data.content) : [...event.data.content],
}
tool.executed = event.data.executed || tool.executed === true
if (event.data.native) tool.native = { ...tool.native, ...event.data.native }
@@ -1119,7 +1156,7 @@ export function createData(config: CreateDataInput) {
input: typeof tool.state.input === "string" ? {} : tool.state.input,
metadata: event.data.metadata,
content:
config.omitToolFileData && event.data.content ? omitFileData(event.data.content) : event.data.content,
omitFiles() && event.data.content ? omitFileData(event.data.content) : event.data.content,
}
tool.executed = event.data.executed || tool.executed === true
if (event.data.native) tool.native = { ...tool.native, ...event.data.native }
@@ -1823,6 +1860,23 @@ export function createData(config: CreateDataInput) {
const position = messageIndex.get(sessionID)?.get(messageID)
return position === undefined ? undefined : messages?.[position]
},
/** Re-reads every loaded message whose tool file data was dropped, for a client that starts showing tool files. */
restoreFiles() {
const sessions = Object.entries(store.session.message).flatMap(([sessionID, messages]) => {
const ids = (messages ?? []).filter(hasDroppedFileData).map((item) => item.id)
return ids.length === 0 ? [] : [{ sessionID, ids }]
})
// Each session's messages are re-read one at a time, so many screenshots don't fan out at once.
return Promise.all(
sessions.map((session) =>
session.ids.reduce(
(previous, messageID) =>
previous.then(() => restoreMessageFiles(session.sessionID, messageID).catch(refreshFailed)),
Promise.resolve(),
),
),
)
},
sync(sessionID: string) {
return sync.run(`session.message:${sessionID}`, async () => {
const started = deletionEpoch
+236 -1
View File
@@ -1,6 +1,6 @@
import { expect, test } from "bun:test"
import { getEventListeners } from "node:events"
import { createRoot } from "solid-js"
import { createRoot, createSignal } from "solid-js"
import { createData, type CreateDataInput } from "../src/solid"
import { OpenCode, type ModelInfo, type OpenCodeEvent, type Project, type SessionInfo } from "../src/promise"
@@ -330,6 +330,241 @@ test("omits inline tool-result file payloads from read and live tool results whe
}
})
test("reads a function omitToolFileData for each live tool result", async () => {
const listeners = new Set<Parameters<CreateDataInput["event"]["listen"]>[0]>()
const screenshot = `data:image/png;base64,${"A".repeat(4096)}`
const running = (id: string) => ({
type: "tool",
id,
name: "execute",
time: { created: 1, ran: 1 },
state: { status: "running", input: {}, metadata: {} },
})
const api = OpenCode.make({
baseUrl: "http://opencode.local",
fetch: async () =>
Response.json({
data: [
{
id: "msg_assistant",
type: "assistant",
agent: "build",
model: { providerID: "provider", id: "model" },
time: { created: 1 },
content: [running("call_off"), running("call_on")],
},
],
cursor: {},
}),
})
const [omit, setOmit] = createSignal(true)
const setup = createRoot((dispose) => ({
data: createData({
api: () => api,
directory: "/project",
omitToolFileData: omit,
event: {
on: () => () => {},
listen(handler) {
listeners.add(handler)
return () => listeners.delete(handler)
},
},
}),
dispose,
}))
const succeed = (id: string, seq: number) => {
const event: OpenCodeEvent = {
id: `evt_${id}`,
created: 2,
type: "session.tool.success",
durable: { aggregateID: "ses_toggle", seq, version: 2 },
data: {
sessionID: "ses_toggle",
assistantMessageID: "msg_assistant",
id,
executed: true,
content: [{ type: "file", uri: screenshot, mime: "image/png" }],
},
}
listeners.forEach((listener) => listener({ name: event.type, details: event }))
}
try {
await setup.data.session.message.sync("ses_toggle")
succeed("call_off", 1)
setOmit(false)
succeed("call_on", 2)
expect(setup.data.session.message.get("ses_toggle", "msg_assistant")).toMatchObject({
content: [
{ state: { content: [{ type: "file", uri: "data:image/png;base64,", mime: "image/png" }] } },
{ state: { content: [{ type: "file", uri: screenshot, mime: "image/png" }] } },
],
})
} finally {
setup.dispose()
}
})
test("restores dropped tool files from the server once they are shown again", async () => {
const listeners = new Set<Parameters<CreateDataInput["event"]["listen"]>[0]>()
const screenshot = `data:image/png;base64,${"A".repeat(4096)}`
const assistant = (state: object) => ({
id: "msg_assistant",
type: "assistant",
agent: "build",
model: { providerID: "provider", id: "model" },
time: { created: 1 },
content: [{ type: "tool", id: "call_shot", name: "execute", time: { created: 1, ran: 1 }, state }],
})
const requests: string[] = []
const api = OpenCode.make({
baseUrl: "http://opencode.local",
fetch: async (input) => {
const url = new URL(input instanceof Request ? input.url : String(input))
requests.push(url.pathname)
if (url.pathname.endsWith("/message/msg_assistant"))
return Response.json({
data: assistant({
status: "completed",
input: {},
content: [{ type: "file", uri: screenshot, mime: "image/png" }],
}),
})
return Response.json({ data: [assistant({ status: "running", input: {}, metadata: {} })], cursor: {} })
},
})
const [omit, setOmit] = createSignal(true)
const setup = createRoot((dispose) => ({
data: createData({
api: () => api,
directory: "/project",
omitToolFileData: omit,
event: {
on: () => () => {},
listen(handler) {
listeners.add(handler)
return () => listeners.delete(handler)
},
},
}),
dispose,
}))
try {
await setup.data.session.message.sync("ses_restore")
const success: OpenCodeEvent = {
id: "evt_shot",
created: 2,
type: "session.tool.success",
durable: { aggregateID: "ses_restore", seq: 1, version: 2 },
data: {
sessionID: "ses_restore",
assistantMessageID: "msg_assistant",
id: "call_shot",
executed: true,
content: [{ type: "file", uri: screenshot, mime: "image/png" }],
},
}
listeners.forEach((listener) => listener({ name: success.type, details: success }))
expect(setup.data.session.message.get("ses_restore", "msg_assistant")).toMatchObject({
content: [{ state: { content: [{ uri: "data:image/png;base64," }] } }],
})
setOmit(false)
await setup.data.session.message.restoreFiles()
expect(requests.filter((path) => path.endsWith("/message/msg_assistant"))).toHaveLength(1)
expect(setup.data.session.message.get("ses_restore", "msg_assistant")).toMatchObject({
content: [{ state: { content: [{ type: "file", uri: screenshot, mime: "image/png" }] } }],
})
// Nothing is left to restore, so a second call reads nothing.
await setup.data.session.message.restoreFiles()
expect(requests.filter((path) => path.endsWith("/message/msg_assistant"))).toHaveLength(1)
} finally {
setup.dispose()
}
})
test("restoring dropped tool files keeps live progress made while the server read was in flight", async () => {
const listeners = new Set<Parameters<CreateDataInput["event"]["listen"]>[0]>()
const screenshot = `data:image/png;base64,${"A".repeat(4096)}`
const tool = (id: string, state: object) => ({ type: "tool", id, name: "execute", time: { created: 1, ran: 1 }, state })
const running = { status: "running", input: {}, metadata: {} }
const assistant = (content: object[]) => ({
id: "msg_assistant",
type: "assistant",
agent: "build",
model: { providerID: "provider", id: "model" },
time: { created: 1 },
content,
})
const read = Promise.withResolvers<Response>()
const api = OpenCode.make({
baseUrl: "http://opencode.local",
fetch: async (input) => {
const url = new URL(input instanceof Request ? input.url : String(input))
if (url.pathname.endsWith("/message/msg_assistant")) return read.promise
return Response.json({ data: [assistant([tool("call_shot", running), tool("call_work", running)])], cursor: {} })
},
})
const [omit, setOmit] = createSignal(true)
const setup = createRoot((dispose) => ({
data: createData({
api: () => api,
directory: "/project",
omitToolFileData: omit,
event: {
on: () => () => {},
listen(handler) {
listeners.add(handler)
return () => listeners.delete(handler)
},
},
}),
dispose,
}))
const succeed = (
id: string,
seq: number,
content: Extract<OpenCodeEvent, { type: "session.tool.success" }>["data"]["content"],
) => {
const event: OpenCodeEvent = {
id: `evt_${id}`,
created: 2,
type: "session.tool.success",
durable: { aggregateID: "ses_inflight", seq, version: 2 },
data: { sessionID: "ses_inflight", assistantMessageID: "msg_assistant", id, executed: true, content },
}
listeners.forEach((listener) => listener({ name: event.type, details: event }))
}
try {
await setup.data.session.message.sync("ses_inflight")
succeed("call_shot", 1, [{ type: "file", uri: screenshot, mime: "image/png" }])
setOmit(false)
const restoring = setup.data.session.message.restoreFiles()
// The second tool finishes while the read is in flight; the server's snapshot predates it.
succeed("call_work", 2, [{ type: "text", text: "done" }])
read.resolve(
Response.json({
data: assistant([
tool("call_shot", { status: "completed", input: {}, content: [{ type: "file", uri: screenshot, mime: "image/png" }] }),
tool("call_work", running),
]),
}),
)
await restoring
expect(setup.data.session.message.get("ses_inflight", "msg_assistant")).toMatchObject({
content: [
{ id: "call_shot", state: { status: "completed", content: [{ type: "file", uri: screenshot }] } },
{ id: "call_work", state: { status: "completed", content: [{ type: "text", text: "done" }] } },
],
})
} finally {
setup.dispose()
}
})
test("refreshes a loaded session's execution start when it starts running", async () => {
const listeners = new Set<Parameters<CreateDataInput["event"]["listen"]>[0]>()
const api = OpenCode.make({
+16
View File
@@ -1,9 +1,11 @@
import { createData } from "@opencode/client/solid"
import { withMediaRefs, type LocationRef } from "@opencode/client"
import type { Plugin } from "@opencode/plugin/tui"
import { createEffect, on } from "solid-js"
import { createStore } from "solid-js/store"
import { createSimpleContext } from "./helper"
import { useClient } from "./client"
import { useConfig } from "../config"
export { locationKey } from "@opencode/client/solid"
export type { FormWithLocation } from "@opencode/client/solid"
@@ -12,13 +14,27 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({
name: "Data",
init: (props: { directory: string }) => {
const client = useClient()
const config = useConfig()
const data = createData({
// Only the transcript store reads stored media as `media:<hash>` refs; plugins and other callers keep inline bytes.
api: () => withMediaRefs(client.api),
event: client.event,
connection: client.connection,
directory: props.directory,
// Live tool events carry screenshots as inline base64, which open tabs would otherwise hold for the whole run.
// Read per event, so toggling previews applies to new results without a restart.
omitToolFileData: () => !config.data.session?.image_preview,
})
// Turning previews on brings back the screenshots dropped while they were off.
createEffect(
on(
() => config.data.session?.image_preview === true,
(preview) => {
if (preview) void data.session.message.restoreFiles()
},
{ defer: true },
),
)
data satisfies Plugin.Context["data"]
const [generatingTitles, setGeneratingTitles] = createStore<Record<string, boolean | undefined>>({})
return {
+2 -1
View File
@@ -2728,7 +2728,8 @@ function inlineToolImages(part: SessionMessageAssistantTool) {
return toolDisplayContent(part.state).flatMap((content) =>
content.type === "file" &&
content.mime.startsWith("image/") &&
(content.uri.startsWith("data:image/") || mediaHashFromUri(content.uri))
// An emptied `data:` URI is a payload the store dropped while previews were off.
((content.uri.startsWith("data:image/") && !content.uri.endsWith(",")) || mediaHashFromUri(content.uri))
? [{ uri: content.uri }]
: [],
)