From 84aed13ac8a054f880eb1f42c413369724153d09 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Thu, 29 Oct 2026 13:04:23 -0700 Subject: [PATCH] feat(tui): load stored images by reference, like desktop (#60116) --- packages/app/src/runtime/server/api.ts | 50 +-- packages/app/src/runtime/server/media.ts | 11 +- packages/client/src/promise/index.ts | 1 + packages/client/src/promise/media.ts | 57 +++ packages/client/test/media.test.ts | 51 +++ .../src/component/dialog-image-preview.tsx | 16 +- packages/tui/src/component/media-source.ts | 43 +++ .../tui/src/component/prompt/draft-stash.ts | 31 +- packages/tui/src/component/prompt/index.tsx | 3 +- packages/tui/src/context/client.tsx | 9 +- packages/tui/src/context/data.tsx | 5 +- packages/tui/src/prompt/attachment.ts | 19 +- packages/tui/src/prompt/codec.ts | 36 +- packages/tui/src/prompt/restore.ts | 36 ++ .../tui/src/routes/session/dialog-fork.tsx | 22 +- .../tui/src/routes/session/dialog-message.tsx | 29 +- .../src/routes/session/dialog-timeline.tsx | 11 +- packages/tui/src/routes/session/index.tsx | 195 +++++----- packages/tui/src/util/media.ts | 43 +++ .../test/fixture/plugin/client-reads/tui.tsx | 9 + packages/tui/test/prompt/attachment.test.ts | 13 + packages/tui/test/prompt/codec.test.ts | 77 +++- packages/tui/test/session-media-refs.test.tsx | 359 ++++++++++++++++++ packages/tui/test/util/media.test.ts | 54 +++ 24 files changed, 969 insertions(+), 211 deletions(-) create mode 100644 packages/client/src/promise/media.ts create mode 100644 packages/client/test/media.test.ts create mode 100644 packages/tui/src/component/media-source.ts create mode 100644 packages/tui/src/prompt/restore.ts create mode 100644 packages/tui/src/util/media.ts create mode 100644 packages/tui/test/fixture/plugin/client-reads/tui.tsx create mode 100644 packages/tui/test/session-media-refs.test.tsx create mode 100644 packages/tui/test/util/media.test.ts diff --git a/packages/app/src/runtime/server/api.ts b/packages/app/src/runtime/server/api.ts index 5da5f219b34..249607d33c0 100644 --- a/packages/app/src/runtime/server/api.ts +++ b/packages/app/src/runtime/server/api.ts @@ -1,10 +1,4 @@ -import { - OpenCode, - type MessageListInput, - type OpenCodeClient, - type SessionInboxListInput, - type SessionMessageGetInput, -} from "@opencode/client/promise" +import { OpenCode, withMediaRefs, type MediaRequestOptions } from "@opencode/client/promise" import type { ServerConnection } from "@/runtime/server/registry" import { decode64 } from "@/runtime/persistence/base64" @@ -25,12 +19,8 @@ export function authFromToken(token: string | null) { } } -export const MEDIA_URI_CAPABILITY = "experimental-media-uri" - -export const CAPABILITIES_HEADER = "X-OpenCode-Capabilities" - /** `mediaRefs: false` reads stored media as inline bytes, for callers such as export that need self-contained messages. */ -export type ServerRequestOptions = OpenCode.RequestOptions & { readonly mediaRefs?: false } +export type ServerRequestOptions = MediaRequestOptions export function createApiForServer(input: { server: ServerConnection.HttpBase; fetch?: typeof globalThis.fetch }) { return withMediaRefs( @@ -48,42 +38,6 @@ export function createApiForServer(input: { server: ServerConnection.HttpBase; f ) } -/** - * Only the message and inbox reads can return stored media as `media:` refs, so only they carry the capability. - * A custom header makes a cross-origin request preflighted, which every other request must keep avoiding. - */ -function withMediaRefs(api: OpenCodeClient) { - return { - ...api, - message: { - ...api.message, - list: (input: MessageListInput, options?: ServerRequestOptions) => - api.message.list(input, mediaRefOptions(options)), - }, - session: { - ...api.session, - message: { - ...api.session.message, - get: (input: SessionMessageGetInput, options?: ServerRequestOptions) => - api.session.message.get(input, mediaRefOptions(options)), - }, - inbox: { - ...api.session.inbox, - list: (input: SessionInboxListInput, options?: ServerRequestOptions) => - api.session.inbox.list(input, mediaRefOptions(options)), - }, - }, - } -} - -function mediaRefOptions(options: ServerRequestOptions | undefined): OpenCode.RequestOptions | undefined { - if (options?.mediaRefs === false) return options - const headers = new Headers(options?.headers) - headers.set(CAPABILITIES_HEADER, MEDIA_URI_CAPABILITY) - - return { ...options, headers } -} - /** * The server's ranged read of one file, for media elements. They cannot send the Authorization header, and a credential * never goes in a DOM URL, so a server that needs a password has none: its media downloads through the API instead. diff --git a/packages/app/src/runtime/server/media.ts b/packages/app/src/runtime/server/media.ts index 7dddc1de67f..e654003fa3f 100644 --- a/packages/app/src/runtime/server/media.ts +++ b/packages/app/src/runtime/server/media.ts @@ -1,13 +1,6 @@ -import type { OpenCodeClient } from "@opencode/client/promise" +import { mediaHashFromUri, type OpenCodeClient } from "@opencode/client/promise" -const MEDIA_REF = /^media:([0-9a-f]{64})$/ - -/** The blob hash in a `media:` ref, the only form the server sends stored media in. */ -export function mediaHashFromUri(uri: string | undefined): string | undefined { - if (!uri) return undefined - - return MEDIA_REF.exec(uri)?.[1] -} +export { mediaHashFromUri } export type MediaCache = { peek(uri: string, mime?: string): string | undefined diff --git a/packages/client/src/promise/index.ts b/packages/client/src/promise/index.ts index 04d10d1eef5..cc41c53b277 100644 --- a/packages/client/src/promise/index.ts +++ b/packages/client/src/promise/index.ts @@ -2,6 +2,7 @@ import type { OpenCode } from "./client.js" export * from "./generated/index.js" export { OpenCode } from "./client.js" +export { mediaHashFromUri, withMediaRefs, type MediaRequestOptions } from "./media.js" export type { AgentApi, CommandApi, diff --git a/packages/client/src/promise/media.ts b/packages/client/src/promise/media.ts new file mode 100644 index 00000000000..bbdf557ca09 --- /dev/null +++ b/packages/client/src/promise/media.ts @@ -0,0 +1,57 @@ +import type { make, RequestOptions } from "./client.js" +import type { MessageListInput, SessionInboxListInput, SessionMessageGetInput } from "./generated/types.js" + +const MEDIA_URI_CAPABILITY = "experimental-media-uri" + +const CAPABILITIES_HEADER = "X-OpenCode-Capabilities" + +const MEDIA_REF = /^media:([0-9a-f]{64})$/ + +/** `mediaRefs: false` reads stored media as inline bytes, for callers such as export that need self-contained messages. */ +export type MediaRequestOptions = RequestOptions & { readonly mediaRefs?: false } + +/** The blob hash in a `media:` ref, the only form the server sends stored media in. */ +export function mediaHashFromUri(uri: string | undefined): string | undefined { + if (!uri) return undefined + + return MEDIA_REF.exec(uri)?.[1] +} + +/** + * Opts a client into the experimental `experimental-media-uri` capability: while `experimental.media_blob` is on, + * stored media comes back as `media:` refs, whose bytes `experimental.media.get` reads. + * + * Only the message and inbox reads can return refs, so only they carry the capability. + * A custom header makes a cross-origin request preflighted, which every other request must keep avoiding. + */ +export function withMediaRefs(api: ReturnType) { + return { + ...api, + message: { + ...api.message, + list: (input: MessageListInput, options?: MediaRequestOptions) => + api.message.list(input, mediaRefOptions(options)), + }, + session: { + ...api.session, + message: { + ...api.session.message, + get: (input: SessionMessageGetInput, options?: MediaRequestOptions) => + api.session.message.get(input, mediaRefOptions(options)), + }, + inbox: { + ...api.session.inbox, + list: (input: SessionInboxListInput, options?: MediaRequestOptions) => + api.session.inbox.list(input, mediaRefOptions(options)), + }, + }, + } +} + +function mediaRefOptions(options: MediaRequestOptions | undefined): RequestOptions | undefined { + if (options?.mediaRefs === false) return options + const headers = new Headers(options?.headers) + headers.set(CAPABILITIES_HEADER, MEDIA_URI_CAPABILITY) + + return { ...options, headers } +} diff --git a/packages/client/test/media.test.ts b/packages/client/test/media.test.ts new file mode 100644 index 00000000000..4d97fa11647 --- /dev/null +++ b/packages/client/test/media.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test" +import { mediaHashFromUri, OpenCode, withMediaRefs } from "../src/promise/index" + +test("withMediaRefs asks for media refs on message and inbox reads only", async () => { + const requests: Array<{ path: string; headers: Headers }> = [] + const api = withMediaRefs( + OpenCode.make({ + baseUrl: "http://localhost:3000", + headers: { Authorization: "Basic secret" }, + // SAFETY: the API client only calls fetch with a URL and init, which this stub accepts. + fetch: (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ + path: new URL(input instanceof Request ? input.url : input).pathname, + headers: new Headers(init?.headers), + }) + return Response.json({ data: [], cursor: {} }) + }) as typeof fetch, + }), + ) + + await api.message.list({ sessionID: "ses_1" }) + await api.message.list({ sessionID: "ses_1" }, { mediaRefs: false }) + await api.session.message.get({ sessionID: "ses_1", messageID: "msg_1" }) + await api.session.inbox.list({ sessionID: "ses_1" }, { headers: { "X-Trace": "1" } }) + await api.session.list() + + expect( + requests.map((request) => [ + request.path, + request.headers.get("x-opencode-capabilities"), + request.headers.get("authorization"), + ]), + ).toEqual([ + ["/api/session/ses_1/message", "experimental-media-uri", "Basic secret"], + ["/api/session/ses_1/message", null, "Basic secret"], + ["/api/session/ses_1/message/msg_1", "experimental-media-uri", "Basic secret"], + ["/api/session/ses_1/inbox", "experimental-media-uri", "Basic secret"], + ["/api/session", null, "Basic secret"], + ]) + expect(requests[3]?.headers.get("x-trace")).toBe("1") +}) + +test.each([ + [`media:${"a".repeat(64)}`, "a".repeat(64)], + [`media:${"A".repeat(64)}`, undefined], + [`media:${"a".repeat(63)}`, undefined], + ["data:image/png;base64,AAA", undefined], + [undefined, undefined], +])("mediaHashFromUri(%p)", (uri, hash) => { + expect(mediaHashFromUri(uri)).toBe(hash) +}) diff --git a/packages/tui/src/component/dialog-image-preview.tsx b/packages/tui/src/component/dialog-image-preview.tsx index f2bdd1d9e1b..7fff59b322e 100644 --- a/packages/tui/src/component/dialog-image-preview.tsx +++ b/packages/tui/src/component/dialog-image-preview.tsx @@ -3,6 +3,7 @@ import { createMemo, createSignal } from "solid-js" import { Keymap } from "../context/keymap" import { useTheme } from "../context/theme" import { DialogHeader, DialogLayout, useDialog } from "../ui/dialog" +import { createMediaSource } from "./media-source" type ImagePreviewItem = Readonly<{ uri: string @@ -14,8 +15,8 @@ export function DialogImagePreview(props: { images: readonly ImagePreviewItem[]; const dimensions = useTerminalDimensions() const theme = useTheme().surface("dialog") const [index, setIndex] = createSignal(Math.max(0, Math.min(props.images.length - 1, props.initial))) - const [failed, setFailed] = createSignal(false) const current = createMemo(() => props.images[index()]) + const media = createMediaSource(() => current().uri) const imageHeight = createMemo(() => Math.max(3, dimensions().height - 8)) dialog.setSize("xlarge") @@ -23,7 +24,6 @@ export function DialogImagePreview(props: { images: readonly ImagePreviewItem[]; function move(direction: number) { if (props.images.length < 2) return - setFailed(false) setIndex((value) => (value + direction + props.images.length) % props.images.length) } @@ -51,8 +51,8 @@ export function DialogImagePreview(props: { images: readonly ImagePreviewItem[]; }, get children() { return ( - - {failed() ? "No preview" : (current().mention?.text ?? `Image ${index() + 1}`)} + + {media.failed() ? "No preview" : (current().mention?.text ?? `Image ${index() + 1}`)} ) }, @@ -62,8 +62,8 @@ export function DialogImagePreview(props: { images: readonly ImagePreviewItem[]; move(-1)}> {props.images.length > 1 ? "← previous" : ""} - - {failed() ? "No preview" : (current().mention?.text ?? `Image ${index() + 1}`)} + + {media.failed() ? "No preview" : (current().mention?.text ?? `Image ${index() + 1}`)} move(1)}> {props.images.length > 1 ? "next →" : ""} @@ -73,12 +73,12 @@ export function DialogImagePreview(props: { images: readonly ImagePreviewItem[]; > setFailed(true)} + onError={media.onError} /> ) diff --git a/packages/tui/src/component/media-source.ts b/packages/tui/src/component/media-source.ts new file mode 100644 index 00000000000..ec2366c27c4 --- /dev/null +++ b/packages/tui/src/component/media-source.ts @@ -0,0 +1,43 @@ +import { mediaHashFromUri } from "@opencode/client" +import { createEffect, createSignal, on, onCleanup, useContext, type Accessor } from "solid-js" +import { ClientContext } from "../context/client" + +/** + * An `` source for a URI that may be a `media:` ref. A ref reads its bytes from the server once something + * shows it, so images nobody looks at fetch nothing; other URIs pass through. `failed` covers a failed read and, through + * `onError`, a failed decode, until the URI changes. + */ +export function createMediaSource(uri: Accessor) { + // Optional: image dialogs also render outside a connected client, where only plain URIs can load. + const media = useContext(ClientContext)?.media + const [bytes, setBytes] = createSignal(media?.peek(uri())) + const [failed, setFailed] = createSignal(false) + + createEffect( + on(uri, (current) => { + setFailed(false) + const cached = media?.peek(current) + setBytes(cached) + if (cached || !mediaHashFromUri(current)) return + if (!media) return setFailed(true) + let active = true + onCleanup(() => { + active = false + }) + media.read(current).then( + (result) => { + if (active) setBytes(result) + }, + () => { + if (active) setFailed(true) + }, + ) + }), + ) + + return { + source: (): string | Uint8Array | undefined => (mediaHashFromUri(uri()) ? bytes() : uri()), + failed, + onError: () => setFailed(true), + } +} diff --git a/packages/tui/src/component/prompt/draft-stash.ts b/packages/tui/src/component/prompt/draft-stash.ts index eaec92e8b0e..3c1160c81cb 100644 --- a/packages/tui/src/component/prompt/draft-stash.ts +++ b/packages/tui/src/component/prompt/draft-stash.ts @@ -1,4 +1,6 @@ -import type { PromptInfo } from "../../prompt/history" +import type { WidthMethod } from "@opentui/core" +import { appendPrompt, type PromptInfo } from "../../prompt/history" +import type { PromptRef } from "./index" // Holds one in-progress draft per tab across Prompt remounts. A draft is // consumed on take: restoring it moves it out of the stash, so a stale copy @@ -16,3 +18,30 @@ export function takeDraft(sessionID: string | undefined) { export function saveDraft(sessionID: string | undefined, entry: DraftEntry) { byTab.set(sessionID, entry) } + +const composers = new Map() + +/** Marks a tab's mounted composer as the one {@link appendToSessionPrompt} writes to; returns its unregister. */ +export function registerComposer(sessionID: string | undefined, composer: PromptRef) { + composers.set(sessionID, composer) + return () => { + if (composers.get(sessionID) === composer) composers.delete(sessionID) + } +} + +/** + * Appends a prompt that comes back after an await (an undo or revert) to a session's composer if one is mounted, else + * to its stashed draft, after anything already typed there. A session view remounts with its tab, so the composer it + * started with may be gone by then. Returns the composer it wrote to. + */ +export function appendToSessionPrompt(sessionID: string, prompt: PromptInfo, widthMethod: WidthMethod) { + const composer = composers.get(sessionID) + if (!composer) { + const draft = takeDraft(sessionID)?.prompt + saveDraft(sessionID, { prompt: draft ? appendPrompt(draft, prompt, widthMethod) : prompt, cursor: Infinity }) + return + } + composer.setMode("normal") + composer.set(appendPrompt(composer.current, prompt, widthMethod)) + return composer +} diff --git a/packages/tui/src/component/prompt/index.tsx b/packages/tui/src/component/prompt/index.tsx index 8f5ae83aea6..d10d2b67262 100644 --- a/packages/tui/src/component/prompt/index.tsx +++ b/packages/tui/src/component/prompt/index.tsx @@ -31,7 +31,7 @@ import { expandPromptInputPastedText, realignPromptInputMentions } from "../../p import { parseSlashHead } from "../../prompt/parse" import { createStore, produce, unwrap } from "solid-js/store" import { emptyPrompt, usePromptHistory, type PromptInfo, type PromptPartRef } from "../../prompt/history" -import { saveDraft, takeDraft } from "./draft-stash" +import { registerComposer, saveDraft, takeDraft } from "./draft-stash" import { Skill } from "@opencode/schema/skill" import { computePromptTraits } from "../../prompt/traits" import { expandPastedTextPlaceholders, expandTrackedPastedText } from "../../prompt/part" @@ -717,6 +717,7 @@ export function Prompt(props: PromptProps) { const stashSessionID = props.sessionID onMount(() => { + onCleanup(registerComposer(stashSessionID, ref)) const saved = takeDraft(stashSessionID) if (store.prompt.text) return if (saved && saved.prompt.text) { diff --git a/packages/tui/src/context/client.tsx b/packages/tui/src/context/client.tsx index 6f30f963f09..22d72bd8ff7 100644 --- a/packages/tui/src/context/client.tsx +++ b/packages/tui/src/context/client.tsx @@ -3,6 +3,7 @@ import { createClientConnection, createPersistentPtyClient } from "@opencode/cli import { createGlobalEmitter } from "@solid-primitives/event-bus" import { onCleanup } from "solid-js" import { createSimpleContext } from "./helper" +import { createMediaCache } from "../util/media" import { useLog } from "./log" type ManagedService = { @@ -12,7 +13,11 @@ type ManagedService = { type ClientEventMap = { [Type in OpenCodeEvent["type"]]: Extract } -export const { use: useClient, provider: ClientProvider } = createSimpleContext({ +export const { + use: useClient, + provider: ClientProvider, + context: ClientContext, +} = createSimpleContext({ name: "Client", init: (props: { api: OpenCodeClient; url?: string; headers?: Record; service?: ManagedService }) => { const log = useLog({ component: "client" }) @@ -22,6 +27,7 @@ export const { use: useClient, provider: ClientProvider } = createSimpleContext( let url = props.url let headers = props.headers let persistentPty = url ? createPersistentPtyClient(api, { url }) : undefined + const media = createMediaCache(() => api) const connection = createClientConnection(api, { reconnect: service @@ -63,6 +69,7 @@ export const { use: useClient, provider: ClientProvider } = createSimpleContext( listen: events.listen, }, connection, + media, restart: service?.restart, } }, diff --git a/packages/tui/src/context/data.tsx b/packages/tui/src/context/data.tsx index d773547c140..8c7668f9a77 100644 --- a/packages/tui/src/context/data.tsx +++ b/packages/tui/src/context/data.tsx @@ -1,5 +1,5 @@ import { createData } from "@opencode/client/solid" -import type { LocationRef } from "@opencode/client" +import { withMediaRefs, type LocationRef } from "@opencode/client" import type { Plugin } from "@opencode/plugin/tui" import { createStore } from "solid-js/store" import { createSimpleContext } from "./helper" @@ -13,7 +13,8 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({ init: (props: { directory: string }) => { const client = useClient() const data = createData({ - api: () => client.api, + // Only the transcript store reads stored media as `media:` refs; plugins and other callers keep inline bytes. + api: () => withMediaRefs(client.api), event: client.event, connection: client.connection, directory: props.directory, diff --git a/packages/tui/src/prompt/attachment.ts b/packages/tui/src/prompt/attachment.ts index d3fc469ace2..75f9eda3e82 100644 --- a/packages/tui/src/prompt/attachment.ts +++ b/packages/tui/src/prompt/attachment.ts @@ -2,14 +2,15 @@ import type { PromptInput } from "@opencode/schema" type PromptFile = PromptInput.FileAttachment type PromptFileIdentity = Pick -type ProjectedFile = Readonly<{ - data: string - mime: string - source: { type: string } - name?: string - description?: string - mention?: { text: string } -}> +type ProjectedFile = Readonly< + { + mime: string + source: { type: string } + name?: string + description?: string + mention?: { text: string } + } & ({ data: string } | { uri: string }) +> function attachmentKind(uri: string) { if (uri.startsWith("data:image/")) return "Image" @@ -67,7 +68,7 @@ export function deduplicateVisibleImages(files: readonl file.mime.startsWith("image/") && file.source.type === "inline" && file.mention?.text ? { metadata: JSON.stringify([file.mime, file.name ?? null, file.description ?? null, file.mention.text]), - payload: file.data, + payload: "uri" in file ? file.uri : file.data, } : undefined, ) diff --git a/packages/tui/src/prompt/codec.ts b/packages/tui/src/prompt/codec.ts index d93130672e8..3f5811bdfaa 100644 --- a/packages/tui/src/prompt/codec.ts +++ b/packages/tui/src/prompt/codec.ts @@ -1,18 +1,26 @@ +import { mediaHashFromUri, type PromptAttachment } from "@opencode/client" import type { Prompt, PromptInput } from "@opencode/schema" import { Skill } from "@opencode/schema/skill" import type { Types } from "effect" export type EditablePromptInput = Types.DeepMutable -type ProjectedPrompt = Pick & { +export type ProjectedPrompt = Pick & { + readonly files?: ReadonlyArray readonly skills?: ReadonlyArray<{ readonly id: string; readonly mention?: PromptInput.SkillAttachment["mention"] }> } +/** A projected prompt as composer input. Files the server kept as `media:` refs stay refs. */ export function projectedPromptInput(input: ProjectedPrompt): EditablePromptInput { return { text: input.text, files: input.files?.map((file) => ({ - uri: file.source.type === "uri" ? file.source.uri : `data:${file.mime};base64,${file.data}`, + uri: + file.source.type === "uri" + ? file.source.uri + : "uri" in file + ? file.uri + : `data:${file.mime};base64,${file.data}`, name: file.name, description: file.description, mention: file.mention ? { ...file.mention } : undefined, @@ -27,3 +35,27 @@ export function projectedPromptInput(input: ProjectedPrompt): EditablePromptInpu })), } } + +/** + * {@link projectedPromptInput} with every `media:` file read back into a `data:` URI, for putting a sent prompt + * back in the composer. The draft then keeps its bytes after the server releases the blob (an inbox cancel, or a + * committed revert), and the composer previews and resends it like any pasted image. A read that resolves `undefined` + * leaves its file out. + */ +export async function restoredPromptInput( + input: ProjectedPrompt, + read: (uri: string) => Promise, +): Promise { + const output = projectedPromptInput(input) + if (!output.files?.some((file) => mediaHashFromUri(file.uri))) return output + const files = await Promise.all( + output.files.map(async (file, index) => { + if (!mediaHashFromUri(file.uri)) return [file] + const bytes = await read(file.uri) + if (!bytes) return [] + const mime = input.files?.[index]?.mime ?? "application/octet-stream" + return [{ ...file, uri: `data:${mime};base64,${Buffer.from(bytes).toString("base64")}` }] + }), + ) + return { ...output, files: files.flat() } +} diff --git a/packages/tui/src/prompt/restore.ts b/packages/tui/src/prompt/restore.ts new file mode 100644 index 00000000000..90b8aec6da6 --- /dev/null +++ b/packages/tui/src/prompt/restore.ts @@ -0,0 +1,36 @@ +import { useClient } from "../context/client" +import { useToast } from "../ui/toast" +import { isMissingMedia } from "../util/media" +import { restoredPromptInput, type ProjectedPrompt } from "./codec" +import type { PromptInfo } from "./history" + +/** + * Reads a sent prompt back for the composer, with its images as bytes. Reading before an undo or revert means a failed + * read changes nothing, and the draft owns its bytes before the server releases the blob: an inbox + * cancel releases it at once, a revert once it's committed on the next submit. An image the server no longer has is + * left out with a warning, so it can't block the rewind; `allowMissing: false` rejects instead, as a fork without its + * prompt is not what was asked for. Any other failed read rejects, since that image may still exist. + */ +export function usePromptRestore() { + const client = useClient() + const toast = useToast() + return async (input: ProjectedPrompt, options?: { allowMissing?: false }): Promise => { + let missing = 0 + const prompt = await restoredPromptInput(input, (uri) => + client.media.read(uri).catch((error) => { + if (options?.allowMissing === false || !isMissingMedia(error)) throw error + missing++ + return undefined + }), + ) + if (missing) + toast.show({ + message: + missing === 1 + ? "An image in this message is no longer stored, so it can't be restored" + : `${missing} images in this message are no longer stored, so they can't be restored`, + variant: "warning", + }) + return { ...prompt, pasted: [] } + } +} diff --git a/packages/tui/src/routes/session/dialog-fork.tsx b/packages/tui/src/routes/session/dialog-fork.tsx index 2ba1a27fe14..d2e2b3f9561 100644 --- a/packages/tui/src/routes/session/dialog-fork.tsx +++ b/packages/tui/src/routes/session/dialog-fork.tsx @@ -8,7 +8,7 @@ import { DIALOG_LIST_INSET, DialogHeader, DialogLayout, useDialog } from "../../ import { useToast } from "../../ui/toast" import { errorMessage } from "../../util/error" import { Locale } from "../../util/locale" -import { projectedPromptInput } from "../../prompt/codec" +import { usePromptRestore } from "../../prompt/restore" export function DialogFork(props: { sessionID: string; messageID?: string; onMove?: (messageID?: string) => void }) { const data = useData() @@ -16,10 +16,20 @@ export function DialogFork(props: { sessionID: string; messageID?: string; onMov const client = useClient() const route = useRoute() const toast = useToast() + const restorePrompt = usePromptRestore() const [pending, setPending] = createSignal(!!props.messageID) const fork = async (messageID?: string) => { setPending(true) + const message = messageID ? data.session.message.get(props.sessionID, messageID) : undefined + const prompt = + message?.type === "user" + ? await restorePrompt(message, { allowMissing: false }).catch((error) => { + toast.show({ message: errorMessage(error), variant: "error", duration: 5000 }) + return null + }) + : undefined + if (prompt === null) return dialog.clear() const result = await client.api.session .fork({ sessionID: props.sessionID, @@ -30,18 +40,10 @@ export function DialogFork(props: { sessionID: string; messageID?: string; onMov return undefined }) if (!result) return dialog.clear() - const message = messageID ? data.session.message.get(props.sessionID, messageID) : undefined - const prompt = message?.type === "user" ? projectedPromptInput(message) : undefined route.navigate({ sessionID: result.id, type: "session", - prompt: prompt - ? { - ...prompt, - agents: prompt.agents ?? [], - pasted: [], - } - : undefined, + prompt: prompt && { ...prompt, agents: prompt.agents ?? [] }, }) dialog.clear() toast.show({ message: "Forked session", variant: "success", duration: 4000 }) diff --git a/packages/tui/src/routes/session/dialog-message.tsx b/packages/tui/src/routes/session/dialog-message.tsx index bf706ddb566..5b2b67419d9 100644 --- a/packages/tui/src/routes/session/dialog-message.tsx +++ b/packages/tui/src/routes/session/dialog-message.tsx @@ -1,4 +1,5 @@ import { createMemo } from "solid-js" +import { useRenderer } from "@opentui/solid" import { useData } from "../../context/data" import { DialogSelect } from "../../ui/dialog-select" import { useClipboard } from "../../context/clipboard" @@ -6,19 +7,17 @@ import { useToast } from "../../ui/toast" import { useClient } from "../../context/client" import { errorMessage } from "../../util/error" import { DialogFork } from "./dialog-fork" -import type { PromptInfo } from "../../prompt/history" -import { projectedPromptInput } from "../../prompt/codec" +import { usePromptRestore } from "../../prompt/restore" import { useNewDesigns } from "../../config" +import { appendToSessionPrompt } from "../../component/prompt/draft-stash" -export function DialogMessage(props: { - messageID: string - sessionID: string - setPrompt?: (prompt: PromptInfo) => void -}) { +export function DialogMessage(props: { messageID: string; sessionID: string }) { const data = useData() const clipboard = useClipboard() const toast = useToast() const client = useClient() + const renderer = useRenderer() + const restorePrompt = usePromptRestore() const newDesigns = useNewDesigns() const message = createMemo(() => data.session.message.get(props.sessionID, props.messageID)) @@ -39,15 +38,13 @@ export function DialogMessage(props: { description: "undo messages and file changes", onSelect: (dialog) => { const value = message() - if (value?.type === "user") { - props.setPrompt?.({ - ...projectedPromptInput(value), - pasted: [], - }) - } - void client.api.session.revert - .stage({ sessionID: props.sessionID, messageID: props.messageID }) - .catch((error) => toast.show({ message: errorMessage(error), variant: "error", duration: 5000 })) + const { sessionID, messageID } = props + // The composer changes only once the revert is staged, so a failed revert leaves the draft alone. + void (async () => { + const restored = value?.type === "user" ? await restorePrompt(value) : undefined + await client.api.session.revert.stage({ sessionID, messageID }) + if (restored) appendToSessionPrompt(sessionID, restored, renderer.widthMethod) + })().catch((error) => toast.show({ message: errorMessage(error), variant: "error", duration: 5000 })) dialog.clear() }, }, diff --git a/packages/tui/src/routes/session/dialog-timeline.tsx b/packages/tui/src/routes/session/dialog-timeline.tsx index fdf6454cf4e..3b51ca919f4 100644 --- a/packages/tui/src/routes/session/dialog-timeline.tsx +++ b/packages/tui/src/routes/session/dialog-timeline.tsx @@ -4,13 +4,8 @@ import { DialogSelect, type DialogSelectOption } from "../../ui/dialog-select" import { Locale } from "../../util/locale" import { DialogMessage } from "./dialog-message" import { useDialog } from "../../ui/dialog" -import type { PromptInfo } from "../../prompt/history" -export function DialogTimeline(props: { - sessionID: string - onMove: (messageID: string) => void - setPrompt?: (prompt: PromptInfo) => void -}) { +export function DialogTimeline(props: { sessionID: string; onMove: (messageID: string) => void }) { const data = useData() const dialog = useDialog() @@ -28,9 +23,7 @@ export function DialogTimeline(props: { value: message.id, footer: Locale.time(message.time.created), onSelect: (dialog) => { - dialog.replace(() => ( - - )) + dialog.replace(() => ) }, }) } diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index 82be1e432d7..318e9d5aecd 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -33,15 +33,17 @@ import { type Renderable, } from "@opentui/core" import { Prompt, type PromptRef } from "../../component/prompt" -import type { - SessionMessageInfo, - SessionMessageAssistant, - SessionMessageAssistantReasoning, - SessionMessageAssistantText, - SessionMessageAssistantTool, - SessionMessageUser, - SessionInfo, - ModelInfo, +import { + mediaHashFromUri, + type PromptAttachment, + type SessionMessageInfo, + type SessionMessageAssistant, + type SessionMessageAssistantReasoning, + type SessionMessageAssistantText, + type SessionMessageAssistantTool, + type SessionMessageUser, + type SessionInfo, + type ModelInfo, } from "@opencode/client" import { useLocal } from "../../context/local" import { Locale } from "../../util/locale" @@ -66,6 +68,7 @@ import { DialogSelect } from "../../ui/dialog-select" import { DialogSessionRename } from "../../component/dialog-session-rename" import { TodoItem } from "../../component/todo-item" import { DialogImagePreview } from "../../component/dialog-image-preview" +import { createMediaSource } from "../../component/media-source" import { statusLabel } from "../../component/dialog-workspace-file-changes" import { DialogMessage } from "./dialog-message" import { DialogFork } from "./dialog-fork" @@ -80,8 +83,9 @@ import { errorMessage } from "../../util/error" import { useToast } from "../../ui/toast" import stripAnsi from "strip-ansi" import { usePromptRef } from "../../context/prompt" -import { projectedPromptInput } from "../../prompt/codec" -import { appendPrompt } from "../../prompt/history" +import { usePromptRestore } from "../../prompt/restore" +import { appendToSessionPrompt } from "../../component/prompt/draft-stash" +import type { ProjectedPrompt } from "../../prompt/codec" import { deduplicateVisibleImages } from "../../prompt/attachment" import { useEpilogue } from "../../context/epilogue" import { normalizePath } from "../../util/path" @@ -260,6 +264,7 @@ export function Session(props: { const scrollAcceleration = createMemo(() => getScrollAcceleration(config)) const toast = useToast() + const restorePrompt = usePromptRestore() const client = useClient() const autoApproved = new Set() createEffect(() => { @@ -642,32 +647,39 @@ export function Session(props: { // Restore the prompt when the free allowance or an OpenCode Go limit stops it, and offer OpenCode Go on the first free // allowance hit in a day. onCleanup( - data.on("session.step.failed", (event) => { + data.on("session.step.failed", async (event) => { const type = event.data.error.type if (event.data.sessionID !== route.sessionID) return if (type !== "provider.free-limit" && type !== "provider.go-limit") return if (handledLimits.has(event.data.assistantMessageID)) return handledLimits.add(event.data.assistantMessageID) - const current = prompt() - const wasEmpty = Boolean( - current && - !current.current.text.trim() && - !current.current.files?.length && - !current.current.agents?.length && - !current.current.skills?.length && - !current.current.pasted.length, - ) - if (!wasEmpty || !current) return + const empty = (ref: PromptRef | undefined): ref is PromptRef => + !!ref && + !ref.current.text.trim() && + !ref.current.files?.length && + !ref.current.agents?.length && + !ref.current.skills?.length && + !ref.current.pasted.length + if (!empty(prompt())) return const list = messages() const end = list.findIndex((item) => item.id === event.data.assistantMessageID) - for (let i = end - 1; i >= 0; i--) { + let recalled: SessionMessageUser | undefined + for (let i = end - 1; i >= 0 && !recalled; i--) { const item = list[i] if (!item || item.type === "shell" || item.type === "idle") break - if (item.type === "user" && !data.session.input.has(route.sessionID, item.id)) { - current.set({ ...projectedPromptInput(item), pasted: [] }) - break - } + if (item.type === "user" && !data.session.input.has(route.sessionID, item.id)) recalled = item } + const restored = recalled + ? await restorePrompt(recalled).catch((error) => { + toast.show({ message: errorMessage(error), variant: "error", duration: 5000 }) + return null + }) + : undefined + // Media refs load before the prompt comes back; typing or a shell command meanwhile keeps it and skips the offer. + if (restored === null || sessionDisposed || route.sessionID !== event.data.sessionID) return + const current = prompt() + if (!empty(current) || current.mode !== "normal") return + if (restored) current.set(restored) if (type === "provider.go-limit" || dialog.stack.length > 0) return if (goUpsell.dontShow || goUpsellMemory.dontShow) return const now = Date.now() @@ -747,25 +759,33 @@ export function Session(props: { }), ) const runPendingAction = createSingleFlight() - const mutatePending = async (action: PendingAction, inboxID: string, failureLabel?: string) => { - const result = await runPendingAction(inboxID, async () => { - const request = - action === "steer" - ? client.api.session.inbox.update({ sessionID: route.sessionID, inboxID, delivery: "steer" }) - : action === "queue" - ? client.api.session.inbox.update({ sessionID: route.sessionID, inboxID, delivery: "queue" }) - : client.api.session.inbox.cancel({ sessionID: route.sessionID, inboxID }) - const error = await request.then( - () => undefined, - (error) => error, - ) - if (!error) return true - const label = failureLabel ?? (action === "cancel" ? "delete" : action) - toast.show({ title: `Failed to ${label} pending prompt`, message: errorMessage(error), variant: "error" }) - return false - }) - return result ?? false + // Uses the captured sessionID: after an await, route.sessionID may already name another tab, or none. + const requestPending = async (action: PendingAction, inboxID: string, failureLabel?: string) => { + const request = + action === "steer" + ? client.api.session.inbox.update({ sessionID, inboxID, delivery: "steer" }) + : action === "queue" + ? client.api.session.inbox.update({ sessionID, inboxID, delivery: "queue" }) + : client.api.session.inbox.cancel({ sessionID, inboxID }) + const error = await request.then( + () => undefined, + (error) => error, + ) + if (!error) return true + const label = failureLabel ?? (action === "cancel" ? "delete" : action) + toast.show({ title: `Failed to ${label} pending prompt`, message: errorMessage(error), variant: "error" }) + return false } + const mutatePending = async (action: PendingAction, inboxID: string) => + (await runPendingAction(inboxID, () => requestPending(action, inboxID))) ?? false + // The cancel releases blobs nothing else references, so media refs load into the draft first. Resolves with the + // composer it wrote to, if any, once the prompt is back; undefined if the cancel failed or was already running. + const undoPending = (inboxID: string, message: ProjectedPrompt, failureLabel?: string) => + runPendingAction(inboxID, async () => { + const restored = await restorePrompt(message) + if (!(await requestPending("cancel", inboxID, failureLabel))) return + return { composer: appendToSessionPrompt(sessionID, restored, renderer.widthMethod) } + }) const openQueuedPrompts = () => dialog.replace(() => ( { - if (!undone) return - target.setMode("normal") - target.set( - appendPrompt( - target.current, - { ...projectedPromptInput(queued.payload), pasted: [] }, - renderer.widthMethod, - ), + void undoPending(queued.id, queued.payload, "undo") + .then((undone) => { + if (!undone || sessionDisposed) return + dialog.clear() + undone.composer?.focus() + }) + .catch((error) => + toast.show({ + title: "Failed to undo pending prompt", + message: errorMessage(error), + variant: "error", + }), ) - dialog.clear() - target.focus() - }) }, }, ]} @@ -1053,13 +1073,7 @@ export function Session(props: { group: "Session", slash: { name: "timeline" }, run: () => { - dialog.replace(() => ( - promptRef.current?.set(value)} - /> - )) + dialog.replace(() => ) }, }, { @@ -1117,21 +1131,16 @@ export function Session(props: { dialog.clear() return } - const sessionID = route.sessionID - const target = prompt() - void (async () => { - if (pendingDeliveries().has(message.id)) { - if (!(await mutatePending("cancel", message.id))) return - } else { - await client.api.session.interrupt({ sessionID }) - await client.api.experimental.session.wait({ sessionID }) - await client.api.session.revert.stage({ sessionID, messageID: message.id }) - } - target?.set({ - ...projectedPromptInput(message), - pasted: [], - }) - })().catch((error) => toast.show({ message: errorMessage(error), variant: "error", duration: 5000 })) + const undo = pendingDeliveries().has(message.id) + ? undoPending(message.id, message) + : (async () => { + const restored = await restorePrompt(message) + await client.api.session.interrupt({ sessionID }) + await client.api.experimental.session.wait({ sessionID }) + await client.api.session.revert.stage({ sessionID, messageID: message.id }) + appendToSessionPrompt(sessionID, restored, renderer.widthMethod) + })() + void undo.catch((error) => toast.show({ message: errorMessage(error), variant: "error", duration: 5000 })) dialog.clear() }, }, @@ -2354,11 +2363,14 @@ function UserMessage(props: { message: SessionMessageUser }) { const newDesigns = useNewDesigns() const data = useData() const local = useLocal() - const files = createMemo(() => deduplicateVisibleImages(props.message.files ?? [])) + // Opted-in reads leave stored files as `media:` refs, a PromptAttachment variant the generated message type omits. + const files = createMemo(() => deduplicateVisibleImages(props.message.files ?? [])) const skills = createMemo(() => props.message.skills ?? []) const images = createMemo(() => files().flatMap((file) => - file.mime.startsWith("image/") ? [{ uri: `data:${file.mime};base64,${file.data}` }] : [], + file.mime.startsWith("image/") + ? [{ uri: "uri" in file ? file.uri : `data:${file.mime};base64,${file.data}` }] + : [], ), ) const themes = useThemes() @@ -2369,7 +2381,6 @@ function UserMessage(props: { message: SessionMessageUser }) { const delivery = createMemo(() => ctx.pendingDelivery(props.message.id)) const dialog = useDialog() const renderer = useRenderer() - const promptRef = usePromptRef() const updatePendingSteer = async (action: "queue" | "cancel") => { if (await ctx.mutatePending(action, props.message.id)) dialog.clear() @@ -2411,13 +2422,7 @@ function UserMessage(props: { message: SessionMessageUser }) { } // The dialog outlives this row, whose props go stale when a resync drops the message. const messageID = props.message.id - dialog.replace(() => ( - promptRef.current?.set(value)} - /> - )) + dialog.replace(() => ) }} paddingTop={1} paddingBottom={1} @@ -2675,7 +2680,7 @@ function SessionImages(props: { images: readonly { uri: string }[]; paddingLeft? > {(image, index) => { - const [failed, setFailed] = createSignal(false) + const media = createMediaSource(() => image.uri) return ( ) }} > - No preview}> + No preview}> setFailed(true)} + onError={media.onError} /> @@ -2718,7 +2723,9 @@ function SessionImages(props: { images: readonly { uri: string }[]; paddingLeft? function inlineToolImages(part: SessionMessageAssistantTool) { return toolDisplayContent(part.state).flatMap((content) => - content.type === "file" && content.mime.startsWith("image/") && content.uri.startsWith("data:image/") + content.type === "file" && + content.mime.startsWith("image/") && + (content.uri.startsWith("data:image/") || mediaHashFromUri(content.uri)) ? [{ uri: content.uri }] : [], ) diff --git a/packages/tui/src/util/media.ts b/packages/tui/src/util/media.ts new file mode 100644 index 00000000000..0d811a1a03d --- /dev/null +++ b/packages/tui/src/util/media.ts @@ -0,0 +1,43 @@ +import { mediaHashFromUri, type OpenCodeClient } from "@opencode/client" + +export type MediaCache = ReturnType + +/** + * Bytes for `media:` refs, read from the server the first time something shows or restores them. A hash names + * its bytes, so entries stay valid across reconnects. + */ +export function createMediaCache( + api: () => { experimental: { media: Pick } }, +) { + const bytes = new Map() + const inflight = new Map>() + + return { + peek: (uri: string) => { + const hash = mediaHashFromUri(uri) + return hash ? bytes.get(hash) : undefined + }, + read: (uri: string) => { + const hash = mediaHashFromUri(uri) + if (!hash) return Promise.reject(new Error(`Not a media ref: ${uri}`)) + const cached = bytes.get(hash) + if (cached) return Promise.resolve(cached) + const pending = + inflight.get(hash) ?? + api() + .experimental.media.get({ hash }) + .then((result) => { + bytes.set(hash, result) + return result + }) + .finally(() => inflight.delete(hash)) + inflight.set(hash, pending) + return pending + }, + } +} + +/** The server answered that the blob is gone (404), as opposed to a failure where it may still exist. */ +export function isMissingMedia(error: unknown) { + return error instanceof Error && error.name === "FileNotFoundError" +} diff --git a/packages/tui/test/fixture/plugin/client-reads/tui.tsx b/packages/tui/test/fixture/plugin/client-reads/tui.tsx new file mode 100644 index 00000000000..e5c5d519b98 --- /dev/null +++ b/packages/tui/test/fixture/plugin/client-reads/tui.tsx @@ -0,0 +1,9 @@ +import { Plugin } from "@opencode/plugin/tui" + +// Reads messages through the plugin client, whose files must stay inline bytes. +export default Plugin.define({ + id: "fixture.client-reads", + setup(context) { + void context.client.message.list({ sessionID: "ses_plugin_probe" }) + }, +}) diff --git a/packages/tui/test/prompt/attachment.test.ts b/packages/tui/test/prompt/attachment.test.ts index 643c4a95c83..ce04d421238 100644 --- a/packages/tui/test/prompt/attachment.test.ts +++ b/packages/tui/test/prompt/attachment.test.ts @@ -122,4 +122,17 @@ describe("prompt attachments", () => { ] expect(deduplicateVisibleImages([file, ...distinct])).toEqual([file, ...distinct]) }) + + test("deduplicates visible image cards the server sent as media refs by their ref", () => { + const file = { + uri: `media:${"a".repeat(64)}`, + mime: "image/png", + source: { type: "inline" }, + name: "clipboard", + mention: { text: "[Image 1]" }, + } + const other = { ...file, uri: `media:${"b".repeat(64)}` } + + expect(deduplicateVisibleImages([file, { ...file }, other])).toEqual([file, other]) + }) }) diff --git a/packages/tui/test/prompt/codec.test.ts b/packages/tui/test/prompt/codec.test.ts index 42db78d6892..20832b65cb8 100644 --- a/packages/tui/test/prompt/codec.test.ts +++ b/packages/tui/test/prompt/codec.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test" import type { Prompt } from "@opencode/schema" -import { projectedPromptInput } from "../../src/prompt/codec" +import { projectedPromptInput, restoredPromptInput } from "../../src/prompt/codec" describe("prompt codec", () => { test("converts projected URI and inline attachments without mutation", () => { @@ -59,3 +59,78 @@ describe("prompt codec", () => { }) }) }) + +describe("restored prompt input", () => { + const hash = "c".repeat(64) + const message = { + text: "Compare [Image 1] with [Image 2]", + files: [ + { + uri: `media:${hash}`, + mime: "image/png", + source: { type: "inline" }, + name: "stored.png", + mention: { start: 8, end: 17, text: "[Image 1]" }, + }, + { + data: "YWJj", + mime: "image/jpeg", + source: { type: "inline" }, + name: "inline.jpg", + mention: { start: 23, end: 32, text: "[Image 2]" }, + }, + ], + } satisfies Parameters[0] + + test("keeps media refs as refs in the synchronous projection", () => { + expect(projectedPromptInput(message).files?.map((file) => file.uri)).toEqual([ + `media:${hash}`, + "data:image/jpeg;base64,YWJj", + ]) + }) + + test("reads media refs back into data URIs and leaves inline files alone", async () => { + const reads: string[] = [] + const output = await restoredPromptInput(message, async (uri) => { + reads.push(uri) + return new Uint8Array([137, 80, 78, 71]) + }) + + expect(reads).toEqual([`media:${hash}`]) + expect(output).toEqual({ + ...projectedPromptInput(message), + files: [ + { + uri: `data:image/png;base64,${Buffer.from([137, 80, 78, 71]).toString("base64")}`, + name: "stored.png", + description: undefined, + mention: { start: 8, end: 17, text: "[Image 1]" }, + }, + { + uri: "data:image/jpeg;base64,YWJj", + name: "inline.jpg", + description: undefined, + mention: { start: 23, end: 32, text: "[Image 2]" }, + }, + ], + }) + }) + + test("reads nothing for a prompt without media refs", async () => { + const inline = { ...message, files: [message.files[1]] } + const output = await restoredPromptInput(inline, () => Promise.reject(new Error("unexpected read"))) + + expect(output).toEqual(projectedPromptInput(inline)) + }) + + test("rejects when a media ref cannot be read, so callers keep the server copy", async () => { + await expect(restoredPromptInput(message, () => Promise.reject(new Error("offline")))).rejects.toThrow("offline") + }) + + test("leaves out a file whose read resolves undefined", async () => { + const output = await restoredPromptInput(message, async () => undefined) + + expect(output.text).toBe(message.text) + expect(output.files?.map((file) => file.uri)).toEqual(["data:image/jpeg;base64,YWJj"]) + }) +}) diff --git a/packages/tui/test/session-media-refs.test.tsx b/packages/tui/test/session-media-refs.test.tsx new file mode 100644 index 00000000000..3efba502a40 --- /dev/null +++ b/packages/tui/test/session-media-refs.test.tsx @@ -0,0 +1,359 @@ +import { expect, test } from "bun:test" +import { createAppFixture } from "./fixture/app" +import { directory, json } from "./fixture/tui-client" +import { takeDraft } from "../src/component/prompt/draft-stash" +import type { PromptInfo } from "../src/prompt/history" +import { copyPlugin } from "./fixture/fixture" + +const hash = "e".repeat(64) +const png = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + "base64", +) +const pngURI = `data:image/png;base64,${png.toString("base64")}` +const session = { + id: "ses_media_refs", + title: "Media refs", + projectID: "proj_test", + location: { directory }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 0, updated: 0 }, +} +const messages = [ + { + type: "assistant", + id: "msg_reply", + agent: "build", + model: { providerID: "fixture", id: "fixture" }, + finish: "stop", + time: { created: 2, completed: 3 }, + content: [{ type: "text", text: "Looked at it" }], + }, + { + type: "user", + id: "msg_shot", + text: "What is in [Image 1]", + files: [ + { + uri: `media:${hash}`, + mime: "image/png", + source: { type: "inline" }, + name: "shot.png", + mention: { start: 11, end: 20, text: "[Image 1]" }, + }, + ], + time: { created: 1 }, + }, +] +// What forking from msg_shot opens: its own transcript, with a different prompt to undo. +const forked = { ...session, id: "ses_forked", title: "Forked" } +const forkedMessages = [ + { ...messages[0], id: "msg_forked_reply" }, + { ...messages[1], id: "msg_forked_ask", text: "Forked ask [Image 1]" }, +] +const queuedShot = { + id: "msg_queued", + sessionID: session.id, + type: "user", + delivery: "queue", + payload: messages[1], + time: { created: 4 }, +} + +// The requests a rewind makes, logged with their status. +const MEDIA = `GET /api/experimental/media/${hash}` +const REVERT = `POST /api/session/${session.id}/revert/stage` +const CANCEL = `DELETE /api/session/${session.id}/inbox/msg_queued` +const FORK = `POST /api/session/${session.id}/fork` +const FORKED_REVERT = `POST /api/session/${forked.id}/revert/stage` + +type Respond = () => Response | Promise +const image: Respond = () => new Response(png, { headers: { "content-type": "image/png" } }) +// What the server answers once a blob is swept, and a failure where the blob may still exist. +const gone: Respond = () => + json({ _tag: "FileNotFoundError", path: `/api/experimental/media/${hash}`, message: "gone" }, { status: 404 }) +const unavailable: Respond = () => new Response("unavailable", { status: 503 }) + +/** Opens the session against a fake server. `calls` logs media reads and rewinds as ` `. */ +async function openSession(server: { inbox?: unknown[]; plugins?: string[]; respond?: Record }) { + // Drafts are stashed per tab for the whole process, so an earlier test or test file would prefill a composer. A home + // draft would also turn the slash command typed there by `leaveAndReturn` into more text. + for (const id of [undefined, session.id, forked.id]) takeDraft(id) + const calls: string[] = [] + const capabilities = new Map() + const logged: Partial> = { + [MEDIA]: image, + [REVERT]: () => json({ data: session }), + [CANCEL]: () => new Response(null, { status: 204 }), + [FORK]: () => json({ data: forked }), + [FORKED_REVERT]: () => json({ data: forked }), + ...server.respond, + } + const setup = await createAppFixture({ + width: 100, + height: 30, + config: { animations: false, tabs: { enabled: false }, plugins: server.plugins }, + args: { sessionID: session.id }, + fetch: async (url, request) => { + const key = `${request.method} ${url.pathname}` + const [, id, rest] = /^\/api\/(?:experimental\/)?session\/([^/]+)(.*)$/.exec(url.pathname) ?? [] + const info = [session, forked].find((item) => item.id === id) + if (rest === "/message") { + capabilities.set(url.pathname, request.headers.get("x-opencode-capabilities")) + return json({ data: id === session.id ? messages : id === forked.id ? forkedMessages : [], cursor: {} }) + } + if (url.pathname === "/api/session") return json({ data: [session], cursor: {} }) + if (info && (rest === "" || rest === "/interrupt")) return json({ data: info }) + if (info && rest === "/inbox") return json({ data: id === session.id ? (server.inbox ?? []) : [] }) + if (info && rest === "/permission") return json({ data: [] }) + if (info && rest === "/wait") return new Response(null, { status: 204 }) + const respond = logged[key] + if (!respond) return undefined + // Logged on arrival, so a test can act while a read is still waiting; the status is added once it answers. + calls.push(key) + const response = await respond() + calls[calls.lastIndexOf(key)] = `${key} ${response.status}` + return response + }, + }) + await setup.waitForFrame((frame) => frame.includes("Looked at it")) + return Object.assign(setup, { calls, capabilities }) +} + +type App = Awaited> + +async function undoLast(app: App) { + await app.mockInput.typeText("/undo") + app.mockInput.pressEnter() +} + +async function undoQueued(app: App) { + app.mockInput.pressKey("x", { ctrl: true }) + app.mockInput.pressKey("q") + await app.waitForFrame((frame) => frame.includes("Queued prompts")) + app.mockInput.pressKey("u", { ctrl: true }) + await app.waitForFrame(() => true) +} + +// Picks one of a message's actions from /timeline. +async function messageAction(app: App, action: string) { + await app.mockInput.typeText("/timeline") + app.mockInput.pressEnter() + await app.waitForFrame((frame) => frame.includes("Jump to message") || frame.includes("Timeline")) + app.mockInput.pressEnter() + await app.waitForFrame((frame) => frame.includes(action)) + await app.mockInput.typeText(action) + app.mockInput.pressEnter() +} + +// Waits for `done`, then gives anything that shouldn't follow, such as a rewind after a failed read or a composer +// change, time to show before reading the frame. +async function settled(app: App, done: (frame: string) => boolean) { + await app.waitForFrame(done) + await Bun.sleep(50) + return app.waitForFrame(() => true) +} + +// How many times the frame shows the prompt: once in the transcript, twice once it is also back in the composer. +const shown = (frame: string) => frame.split("What is in [Image 1]").length - 1 + +test("reads transcripts with media refs and loads a ref's bytes before /undo rewinds its message", async () => { + await using app = await openSession({}) + expect([...app.capabilities.values()]).toEqual(["experimental-media-uri"]) + // Image previews are off by default, so showing the transcript reads no media. + expect(app.calls).toEqual([]) + + await undoLast(app) + await app.waitForFrame((frame) => shown(frame) === 2) + expect(app.calls).toEqual([`${MEDIA} 200`, `${REVERT} 200`]) +}, 20000) + +// Starts an undo, leaves the session while its media read waits, then lets the read finish. +async function undoAfterLeaving(input: { inbox: unknown[]; undo: (app: App) => Promise }) { + const media = Promise.withResolvers() + await using app = await openSession({ inbox: input.inbox, respond: { [MEDIA]: () => media.promise.then(image) } }) + await input.undo(app) + await app.waitForFrame(() => app.calls.length > 0) + await app.mockInput.typeText("/new") + app.mockInput.pressEnter() + await app.waitForFrame((frame) => !frame.includes("Looked at it")) + media.resolve() + // The rewind's request is logged on arrival, but the draft is written only once its response is back. + let draft: PromptInfo | undefined + await app.waitForFrame(() => (draft ??= takeDraft(session.id)?.prompt) !== undefined) + return { calls: app.calls, draft } +} + +test("an /undo that finishes after its session closed keeps the prompt in that session's draft", async () => { + const { calls, draft } = await undoAfterLeaving({ inbox: [], undo: undoLast }) + + expect(calls).toEqual([`${MEDIA} 200`, `${REVERT} 200`]) + expect(draft?.text).toBe("What is in [Image 1]") + expect(draft?.files?.map((file) => file.uri)).toEqual([pngURI]) +}, 20000) + +test("undoing a queued prompt after its session closed cancels it in that session and keeps it as a draft", async () => { + const { calls, draft } = await undoAfterLeaving({ + inbox: [queuedShot], + undo: async (app) => { + await undoQueued(app) + app.mockInput.pressEscape() + }, + }) + + expect(calls).toEqual([`${MEDIA} 200`, `${CANCEL} 204`]) + expect(draft?.text).toBe("What is in [Image 1]") + expect(draft?.files?.map((file) => file.uri)).toEqual([pngURI]) +}, 20000) + +// Undoes a prompt whose image read fails, until `done` shows the outcome. The composer is read from the draft it leaves +// when the app closes. +async function undoWithFailedRead(media: Respond, queued: boolean, done: (frame: string, app: App) => boolean) { + const result = await (async () => { + await using app = await openSession({ inbox: queued ? [queuedShot] : [], respond: { [MEDIA]: media } }) + await (queued ? undoQueued(app) : undoLast(app)) + const frame = await settled(app, (frame) => app.calls.length > 0 && done(frame, app)) + return { calls: app.calls, frame } + })() + return { ...result, draft: takeDraft(session.id)?.prompt } +} + +for (const queued of [false, true]) { + const name = queued ? "undoing a queued prompt" : "/undo" + + test(`${name} goes ahead without an image the server no longer has`, async () => { + // The warning shows before the rewind; the prompt reaches the composer once it's done. + const { calls, frame, draft } = await undoWithFailedRead( + gone, + queued, + (frame, app) => + frame.includes("no longer stored") && !!app.renderer.currentFocusedEditor?.plainText.includes("What is in"), + ) + + expect(calls).toEqual([`${MEDIA} 404`, queued ? `${CANCEL} 204` : `${REVERT} 200`]) + expect(draft?.text).toBe("What is in [Image 1]") + expect(draft?.files).toEqual([]) + expect(frame).toContain("no longer stored") + }, 20000) + + test(`${name} keeps the message when an image read fails for another reason`, async () => { + const { calls, draft } = await undoWithFailedRead(unavailable, queued, (frame) => /unavailable|503/i.test(frame)) + + expect(calls).toEqual([`${MEDIA} 503`]) + expect(draft).toBeUndefined() + }, 20000) +} + +test("a revert the server refuses leaves the composer as it was", async () => { + await using app = await openSession({ + respond: { [REVERT]: () => json({ name: "BusyError", data: {} }, { status: 409 }) }, + }) + await messageAction(app, "Revert") + const frame = await settled(app, () => app.calls.includes(`${REVERT} 409`)) + + expect(app.calls).toEqual([`${MEDIA} 200`, `${REVERT} 409`]) + expect(shown(frame)).toBe(1) +}, 20000) + +test("a revert goes ahead without an image the server no longer has", async () => { + await using app = await openSession({ respond: { [MEDIA]: gone } }) + await messageAction(app, "Revert") + const frame = await settled(app, (frame) => shown(frame) === 2) + + expect(app.calls).toEqual([`${MEDIA} 404`, `${REVERT} 200`]) + expect(shown(frame)).toBe(2) + expect(frame).toContain("no longer stored") +}, 20000) + +test("a revert whose image read fails for another reason does not revert", async () => { + await using app = await openSession({ respond: { [MEDIA]: unavailable } }) + await messageAction(app, "Revert") + const frame = await settled(app, () => app.calls.includes(`${MEDIA} 503`)) + + expect(app.calls).toEqual([`${MEDIA} 503`]) + expect(shown(frame)).toBe(1) +}, 20000) + +test("forking from a prompt whose image is gone does not fork", async () => { + await using app = await openSession({ respond: { [MEDIA]: gone } }) + await messageAction(app, "Fork") + await settled(app, () => app.calls.includes(`${MEDIA} 404`)) + + expect(app.calls).toEqual([`${MEDIA} 404`]) +}, 20000) + +test("plugins read messages with inline bytes while the transcript reads media refs", async () => { + await using plugin = await copyPlugin("client-reads") + await using app = await openSession({ plugins: [plugin.directory] }) + await app.waitForFrame(() => app.capabilities.size === 2) + + expect(Object.fromEntries(app.capabilities)).toEqual({ + [`/api/session/${session.id}/message`]: "experimental-media-uri", + "/api/session/ses_plugin_probe/message": null, + }) +}, 20000) + +async function leaveAndReturn(app: App) { + // A leader key, since text in the composer would turn a slash command into more text. + app.mockInput.pressKey("x", { ctrl: true }) + app.mockInput.pressKey("n") + await app.waitForFrame((frame) => !frame.includes("Looked at it")) + await app.mockInput.typeText("/sessions") + app.mockInput.pressEnter() + await app.waitForFrame((frame) => frame.includes("Sessions") && frame.includes("Media refs")) + app.mockInput.pressEnter() + await app.waitForFrame((frame) => frame.includes("Looked at it")) +} + +for (const queued of [false, true]) { + const name = queued ? "a queued-prompt undo" : "an /undo" + + test(`${name} that finishes after leaving and returning joins what was typed since`, async () => { + const media = Promise.withResolvers() + const result = await (async () => { + await using app = await openSession({ + inbox: queued ? [queuedShot] : [], + respond: { [MEDIA]: () => media.promise.then(image) }, + }) + if (queued) { + await undoQueued(app) + app.mockInput.pressEscape() + } else await undoLast(app) + await app.waitForFrame(() => app.calls.length > 0) + await leaveAndReturn(app) + await app.mockInput.typeText("typed note") + media.resolve() + // The rewind finishes by appending the prompt to the live composer, after what was typed. + await app.waitForFrame(() => !!app.renderer.currentFocusedEditor?.plainText.includes("What is in")) + await leaveAndReturn(app) + return { calls: app.calls, frame: await app.waitForFrame((frame) => frame.includes("typed note")) } + })() + const draft = takeDraft(session.id)?.prompt + + expect(result.calls).toEqual([`${MEDIA} 200`, queued ? `${CANCEL} 204` : `${REVERT} 200`]) + expect(result.frame).toContain("typed note") + expect(draft?.text).toBe("typed note\n\nWhat is in [Image 1]") + expect(draft?.files?.map((file) => file.uri)).toEqual([pngURI]) + }, 30000) +} + +test("an undo in a session opened with a seeded prompt, such as a fork, lands in its composer", async () => { + const result = await (async () => { + await using app = await openSession({}) + await messageAction(app, "Fork") + // The fork opens with the forked-from prompt already in its composer. + await app.waitForFrame((frame) => frame.includes("Forked ask") && frame.includes("What is in [Image 1]")) + app.mockInput.pressKey("x", { ctrl: true }) + app.mockInput.pressKey("u") + // Once in the transcript and once more in the composer, after the seeded prompt. + const frame = await settled(app, (frame) => frame.split("Forked ask").length - 1 === 2) + return { calls: app.calls, frame } + })() + const draft = takeDraft(forked.id)?.prompt + + expect(result.calls).toEqual([`${MEDIA} 200`, `${FORK} 200`, `${FORKED_REVERT} 200`]) + expect(result.frame).toContain("What is in [Image 1]") + expect(draft?.text).toBe("What is in [Image 1]\n\nForked ask [Image 1]") + expect(draft?.files?.map((file) => file.uri)).toEqual([pngURI, pngURI]) +}, 30000) diff --git a/packages/tui/test/util/media.test.ts b/packages/tui/test/util/media.test.ts new file mode 100644 index 00000000000..83d786b3120 --- /dev/null +++ b/packages/tui/test/util/media.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test" +import { createMediaCache } from "../../src/util/media" + +const hash = "d".repeat(64) + +function setup(get: (input: { hash: string }) => Promise) { + const calls: string[] = [] + const cache = createMediaCache(() => ({ + experimental: { + media: { + get: (input: { hash: string }) => { + calls.push(input.hash) + return get(input) + }, + }, + }, + })) + return { cache, calls } +} + +describe("media cache", () => { + test("reads each hash once, sharing a request in flight", async () => { + const bytes = new Uint8Array([1, 2, 3]) + const { cache, calls } = setup(async () => bytes) + + expect(cache.peek(`media:${hash}`)).toBeUndefined() + const [first, second] = await Promise.all([cache.read(`media:${hash}`), cache.read(`media:${hash}`)]) + + expect(first).toBe(bytes) + expect(second).toBe(bytes) + expect(await cache.read(`media:${hash}`)).toBe(bytes) + expect(cache.peek(`media:${hash}`)).toBe(bytes) + expect(calls).toEqual([hash]) + }) + + test("retries a hash whose read failed", async () => { + const results = [Promise.reject(new Error("offline")), Promise.resolve(new Uint8Array([4]))] + const { cache, calls } = setup(() => results.shift()!) + + await expect(cache.read(`media:${hash}`)).rejects.toThrow("offline") + expect(cache.peek(`media:${hash}`)).toBeUndefined() + expect(await cache.read(`media:${hash}`)).toEqual(new Uint8Array([4])) + expect(calls).toEqual([hash, hash]) + }) + + test("never requests anything for a URI that is not a media ref", async () => { + const { cache, calls } = setup(async () => new Uint8Array()) + + expect(cache.peek("data:image/png;base64,AAA")).toBeUndefined() + await expect(cache.read("data:image/png;base64,AAA")).rejects.toThrow("Not a media ref") + await expect(cache.read(`media:${hash.toUpperCase()}`)).rejects.toThrow("Not a media ref") + expect(calls).toEqual([]) + }) +})