From a5f7f8d3b50b01244ca397a43799d6375142043c Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Mon, 10 Aug 2026 23:39:38 -0400 Subject: [PATCH] fix(tui): deduplicate repeated image attachments (#41651) --- .../core/src/session/runner/to-llm-message.ts | 21 ++- .../core/test/session-runner-message.test.ts | 97 ++++++++++++++ packages/tui/src/component/prompt/index.tsx | 33 +++-- packages/tui/src/prompt/attachment.ts | 96 ++++++++++++++ packages/tui/src/routes/session/index.tsx | 3 +- packages/tui/test/prompt/attachment.test.ts | 125 ++++++++++++++++++ packages/tui/test/prompt/history.test.ts | 17 +++ 7 files changed, 378 insertions(+), 14 deletions(-) create mode 100644 packages/tui/src/prompt/attachment.ts create mode 100644 packages/tui/test/prompt/attachment.test.ts diff --git a/packages/core/src/session/runner/to-llm-message.ts b/packages/core/src/session/runner/to-llm-message.ts index 325fe271928..90900c38e34 100644 --- a/packages/core/src/session/runner/to-llm-message.ts +++ b/packages/core/src/session/runner/to-llm-message.ts @@ -59,6 +59,25 @@ const attachmentContent = (file: FileAttachment): ContentPart[] => { return [] } +const userAttachmentContent = (files: readonly FileAttachment[]) => { + const eligible = files.filter( + (file) => imageMimes.has(file.mime) && file.source.type === "inline" && file.mention?.text, + ) + if (eligible.length < 2) return files.flatMap(attachmentContent) + + const seen = new Map() + return files.flatMap((file) => { + if (!imageMimes.has(file.mime) || file.source.type !== "inline" || !file.mention?.text) + return attachmentContent(file) + const metadata = JSON.stringify([file.mime, file.name ?? null, file.description ?? null, file.mention.text]) + const matches = seen.get(metadata) + if (matches?.includes(file.data)) return [] + if (matches) matches.push(file.data) + if (!matches) seen.set(metadata, [file.data]) + return attachmentContent(file) + }) +} + const decodeToolInput = Schema.decodeUnknownOption(Schema.UnknownFromJsonString) const providerMetadata = ( @@ -186,7 +205,7 @@ function toLLMMessage(message: SessionMessage.Info, model: Model.Ref, providerMe const content = [ ...(message.skills ?? []).map((skill) => Message.text(skill.text)), ...(message.text === "" ? [] : [Message.text(message.text)]), - ...(message.files ?? []).flatMap(attachmentContent), + ...userAttachmentContent(message.files ?? []), ] if (content.length === 0) return [] return [ diff --git a/packages/core/test/session-runner-message.test.ts b/packages/core/test/session-runner-message.test.ts index ce4a9f64a31..93c93c8114b 100644 --- a/packages/core/test/session-runner-message.test.ts +++ b/packages/core/test/session-runner-message.test.ts @@ -373,6 +373,103 @@ Recent work ]) }) + test("deduplicates provider media while preserving durable attachment references", () => { + const data = Base64.make("AAECAw==") + const messages = toLLMMessages( + [ + SessionMessage.User.make({ + id: id("user-duplicate-image"), + type: "user", + text: "[Image 1] [Image 1] [Image 2]", + files: [ + FileAttachment.make({ + data, + mime: "image/png", + source: { type: "inline" }, + name: "image.png", + mention: { start: 0, end: 9, text: "[Image 1]" }, + }), + FileAttachment.make({ + data, + mime: "image/png", + source: { type: "inline" }, + name: "image.png", + mention: { start: 10, end: 19, text: "[Image 1]" }, + }), + FileAttachment.make({ + data, + mime: "image/png", + source: { type: "inline" }, + name: "image.png", + description: "alternate use", + mention: { start: 20, end: 29, text: "[Image 2]" }, + }), + ], + time: { created }, + }), + ], + model, + ) + + expect(messages[0]?.content).toEqual([ + { type: "text", text: "[Image 1] [Image 1] [Image 2]" }, + { type: "media", mediaType: "image/png", data, filename: "image.png" }, + { + type: "media", + mediaType: "image/png", + data, + filename: "image.png", + metadata: { description: "alternate use" }, + }, + ]) + }) + + test("preserves provider media with distinct labels or URI sources", () => { + const data = Base64.make("AAECAw==") + const messages = toLLMMessages( + [ + SessionMessage.User.make({ + id: id("user-distinct-images"), + type: "user", + text: "[Image 1] [Image 2]", + files: [ + FileAttachment.make({ + data, + mime: "image/png", + source: { type: "inline" }, + name: "image.png", + mention: { start: 0, end: 9, text: "[Image 1]" }, + }), + FileAttachment.make({ + data, + mime: "image/png", + source: { type: "inline" }, + name: "image.png", + mention: { start: 10, end: 19, text: "[Image 2]" }, + }), + FileAttachment.make({ + data, + mime: "image/png", + source: { type: "uri", uri: "file:///project/image.png" }, + name: "image.png", + mention: { start: 0, end: 9, text: "[Image 1]" }, + }), + FileAttachment.make({ + data, + mime: "image/png", + source: { type: "inline" }, + name: "image.png", + }), + ], + time: { created }, + }), + ], + model, + ) + + expect(messages[0]?.content.filter((part) => part.type === "media")).toHaveLength(4) + }) + test("replays durable tool media into canonical tool messages without structured base64", () => { const messages = toLLMMessages( [ diff --git a/packages/tui/src/component/prompt/index.tsx b/packages/tui/src/component/prompt/index.tsx index 4303432db14..0252cfc91b5 100644 --- a/packages/tui/src/component/prompt/index.tsx +++ b/packages/tui/src/component/prompt/index.tsx @@ -60,6 +60,11 @@ import { Keymap, type KeymapCommand } from "../../context/keymap" import { abbreviateHome } from "../../runtime" import { PluginSlot } from "../../plugin/render" import type { SessionPending } from "@opencode-ai/schema/session-pending" +import { + deduplicatePromptImages, + preserveMentionlessPromptAttachments, + promptAttachmentLabel, +} from "../../prompt/attachment" import { DialogImagePreview } from "../dialog-image-preview" export type PromptProps = { @@ -331,7 +336,7 @@ export function Prompt(props: PromptProps) { } const imageAttachments = createMemo(() => - (store.prompt.files ?? []).filter((file) => typeof file.uri === "string" && file.uri.startsWith("data:image/")), + (deduplicatePromptImages(store.prompt.files) ?? []).filter((file) => file.uri.startsWith("data:image/")), ) const imagePreviewHeight = createMemo(() => Math.max(4, Math.min(8, Math.floor(dimensions().height / 4)))) const imagePreviewWidth = createMemo(() => imagePreviewHeight() * 2) @@ -736,6 +741,7 @@ export function Prompt(props: PromptProps) { setStore( produce((draft) => { const newMap = new Map() + const fileExtmarks = new Map[number]>() const files: NonNullable = [] const agents: NonNullable = [] const skills: NonNullable = [] @@ -749,9 +755,8 @@ export function Prompt(props: PromptProps) { if (!part?.mention) continue part.mention.start = extmark.start part.mention.end = extmark.end - const index = files.length files.push(part) - newMap.set(extmark.id, { type: "file", index }) + fileExtmarks.set(extmark.id, part) continue } if (ref.type === "agent") { @@ -783,8 +788,19 @@ export function Prompt(props: PromptProps) { newMap.set(extmark.id, { type: "pasted", index }) } + const nextFiles = preserveMentionlessPromptAttachments(draft.prompt.files, files) + const fileIndices = new Map(nextFiles.map((file, index) => [file, index])) + for (const [extmark, file] of fileExtmarks) { + const index = fileIndices.get(file) + if (index !== undefined) newMap.set(extmark, { type: "file", index }) + } + draft.extmarkToPart = newMap - draft.prompt.files = files + if ( + nextFiles.length !== draft.prompt.files?.length || + nextFiles.some((file, index) => file !== draft.prompt.files?.[index]) + ) + draft.prompt.files = nextFiles draft.prompt.agents = agents draft.prompt.skills = skills draft.prompt.pasted = pasted @@ -1138,7 +1154,6 @@ export function Prompt(props: PromptProps) { // Capture mode before it gets reset const currentMode = store.mode - if (store.mode === "shell") { move.startSubmit() void client.api.session.shell({ @@ -1376,13 +1391,7 @@ export function Prompt(props: PromptProps) { function pasteAttachment(file: { filename?: string; uri: string }) { const currentOffset = input.cursorOffset const extmarkStart = currentOffset - const pdf = file.uri.startsWith("data:application/pdf;") - const count = pdf - ? (store.prompt.files?.filter( - (attachment) => typeof attachment.uri === "string" && attachment.uri.startsWith("data:application/pdf;"), - ).length ?? 0) - : imageAttachments().length - const virtualText = pdf ? `[PDF ${count + 1}]` : `[Image ${count + 1}]` + const virtualText = promptAttachmentLabel(store.prompt.files, { uri: file.uri, name: file.filename }) const extmarkEnd = extmarkStart + virtualText.length const textToInsert = virtualText + " " diff --git a/packages/tui/src/prompt/attachment.ts b/packages/tui/src/prompt/attachment.ts new file mode 100644 index 00000000000..4ebc6c31544 --- /dev/null +++ b/packages/tui/src/prompt/attachment.ts @@ -0,0 +1,96 @@ +import type { PromptInput } from "@opencode-ai/schema" + +type PromptFile = PromptInput.FileAttachment +type PromptFileIdentity = Pick +type ProjectedFile = Readonly<{ + data: string + mime: string + source: { type: string } + name?: string + description?: string + mention?: { text: string } +}> + +function attachmentKind(uri: string) { + if (uri.startsWith("data:image/")) return "Image" + if (uri.startsWith("data:application/pdf;")) return "PDF" + return undefined +} + +function attachmentMetadata(file: PromptFileIdentity) { + return JSON.stringify([file.name ?? null, file.description ?? null]) +} + +function deduplicateByIdentity( + items: readonly T[], + identity: (item: T) => { metadata: string; payload: string } | undefined, +) { + const seen = new Map() + return items.filter((item) => { + const key = identity(item) + if (!key) return true + const matches = seen.get(key.metadata) + if (matches?.includes(key.payload)) return false + if (matches) matches.push(key.payload) + if (!matches) seen.set(key.metadata, [key.payload]) + return true + }) +} + +export function deduplicatePromptImages(files: readonly PromptFile[] | undefined) { + if (!files || files.length < 2) return files + return deduplicateByIdentity(files, (file) => + file.uri.startsWith("data:image/") && file.mention?.text + ? { + metadata: JSON.stringify([attachmentMetadata(file), file.mention.text]), + payload: file.uri, + } + : undefined, + ) +} + +export function preserveMentionlessPromptAttachments( + files: readonly PromptFile[] | undefined, + mentioned: PromptFile[], +) { + if (!files) return mentioned + const tracked = mentioned.values() + return files.flatMap((file) => { + if (!file.mention?.text) return [file] + const next = tracked.next() + return next.done ? [] : [next.value] + }) +} + +export function deduplicateVisibleImages(files: readonly T[]) { + return deduplicateByIdentity(files, (file) => + file.mime.startsWith("image/") && file.source.type === "inline" && file.mention?.text + ? { + metadata: JSON.stringify([file.mime, file.name ?? null, file.description ?? null, file.mention.text]), + payload: file.data, + } + : undefined, + ) +} + +export function promptAttachmentLabel(files: readonly PromptFile[] | undefined, file: PromptFileIdentity) { + const kind = attachmentKind(file.uri) + if (!kind) throw new Error(`Unsupported inline attachment: ${file.uri}`) + const metadata = attachmentMetadata(file) + const existing = + kind === "Image" + ? files?.find( + (candidate) => + candidate.uri === file.uri && attachmentMetadata(candidate) === metadata && candidate.mention?.text, + )?.mention?.text + : undefined + if (existing) return existing + + const pattern = new RegExp(`^\\[${kind} (\\d+)\\]$`) + const count = + files?.reduce((highest, candidate) => { + const match = candidate.mention?.text.match(pattern) + return match ? Math.max(highest, Number(match[1])) : highest + }, 0) ?? 0 + return `[${kind} ${count + 1}]` +} diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index 9362c472a20..dd3f7366560 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -70,6 +70,7 @@ import stripAnsi from "strip-ansi" import { usePromptRef } from "../../context/prompt" import { sessionTabsFitVertically, SESSION_SIDEBAR_WIDTH } from "../../ui/layout" import { projectedPromptInput } from "../../prompt/codec" +import { deduplicateVisibleImages } from "../../prompt/attachment" import { useEpilogue } from "../../context/epilogue" import { normalizePath } from "../../util/path" import { PermissionPrompt } from "./permission" @@ -1899,7 +1900,7 @@ function UserMessage(props: { message: SessionMessageUser }) { const ctx = use() const data = useData() const local = useLocal() - const files = createMemo(() => props.message.files ?? []) + const files = createMemo(() => deduplicateVisibleImages(props.message.files ?? [])) const skills = createMemo(() => props.message.skills ?? []) const images = createMemo(() => files().flatMap((file) => diff --git a/packages/tui/test/prompt/attachment.test.ts b/packages/tui/test/prompt/attachment.test.ts new file mode 100644 index 00000000000..643c4a95c83 --- /dev/null +++ b/packages/tui/test/prompt/attachment.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, test } from "bun:test" +import { + deduplicatePromptImages, + deduplicateVisibleImages, + preserveMentionlessPromptAttachments, + promptAttachmentLabel, +} from "../../src/prompt/attachment" + +describe("prompt attachments", () => { + test("deduplicates identical inline images while preserving other attachments", () => { + const files = [ + { + uri: "data:image/png;base64,AAA", + name: "first.png", + mention: { start: 0, end: 9, text: "[Image 1]" }, + }, + { uri: "file:///same", name: "first.txt" }, + { uri: "data:application/pdf;base64,CCC", name: "first.pdf" }, + { + uri: "data:image/png;base64,BBB", + name: "second.png", + mention: { start: 10, end: 19, text: "[Image 2]" }, + }, + { + uri: "data:image/png;base64,AAA", + name: "first.png", + mention: { start: 20, end: 29, text: "[Image 1]" }, + }, + { + uri: "data:image/png;base64,AAA", + name: "first.png", + description: "alternate use", + mention: { start: 30, end: 39, text: "[Image 1]" }, + }, + { uri: "file:///same", name: "second.txt" }, + { uri: "data:application/pdf;base64,CCC", name: "first.pdf" }, + ] + + expect(deduplicatePromptImages(files)).toEqual([ + files[0], + files[1], + files[2], + files[3], + files[5], + files[6], + files[7], + ]) + expect(files).toHaveLength(8) + }) + + test("reuses labels for identical image data", () => { + const first = "data:image/png;base64,AAA" + const second = "data:image/png;base64,BBB" + const files = [{ uri: first, mention: { start: 0, end: 9, text: "[Image 1]" } }] + + expect(promptAttachmentLabel(files, { uri: first })).toBe("[Image 1]") + expect(promptAttachmentLabel([...files, { ...files[0], mention: undefined }], { uri: second })).toBe("[Image 2]") + expect(promptAttachmentLabel([{ uri: first }], { uri: first })).toBe("[Image 1]") + }) + + test("numbers PDFs independently from images", () => { + const files = [{ uri: "data:image/png;base64,AAA" }] + + expect(promptAttachmentLabel(files, { uri: "data:application/pdf;base64,BBB" })).toBe("[PDF 1]") + }) + + test("does not reuse a label when attachment metadata differs", () => { + const uri = "data:image/png;base64,AAA" + const files = [{ uri, name: "one.png", mention: { start: 0, end: 9, text: "[Image 1]" } }] + + expect(promptAttachmentLabel(files, { uri, name: "two.png" })).toBe("[Image 2]") + }) + + test("does not reuse numbers after an earlier attachment is removed", () => { + const files = [{ uri: "data:image/png;base64,BBB", mention: { start: 0, end: 9, text: "[Image 2]" } }] + + expect(promptAttachmentLabel(files, { uri: "data:image/png;base64,CCC" })).toBe("[Image 3]") + }) + + test("preserves mentionless attachments when tracked mentions are synchronized", () => { + const mentionless = { uri: "data:image/png;base64,AAA" } + const emptyMention = { + uri: "data:image/png;base64,CCC", + mention: { start: 0, end: 0, text: "" }, + } + const mentioned = { + uri: "data:image/png;base64,BBB", + mention: { start: 0, end: 9, text: "[Image 1]" }, + } + + const restored = preserveMentionlessPromptAttachments([mentionless, emptyMention, mentioned], [mentioned]) + expect(restored).toEqual([mentionless, emptyMention, mentioned]) + expect(restored.indexOf(mentioned)).toBe(2) + + const another = { + uri: "data:image/png;base64,DDD", + mention: { start: 10, end: 19, text: "[Image 2]" }, + } + expect(preserveMentionlessPromptAttachments([mentioned, mentionless, another], [another, mentioned])).toEqual([ + another, + mentionless, + mentioned, + ]) + }) + + test("deduplicates visible inline image cards without dropping durable references", () => { + const file = { + data: "AAA", + mime: "image/png", + source: { type: "inline" }, + name: "clipboard", + mention: { text: "[Image 1]" }, + } + const files = [file, { ...file, mention: { text: "[Image 1]" } }] + + expect(deduplicateVisibleImages(files)).toEqual([file]) + expect(files).toHaveLength(2) + + const distinct = [ + { ...file, mention: { text: "[Image 2]" } }, + { ...file, mention: undefined }, + ] + expect(deduplicateVisibleImages([file, ...distinct])).toEqual([file, ...distinct]) + }) +}) diff --git a/packages/tui/test/prompt/history.test.ts b/packages/tui/test/prompt/history.test.ts index 5c040359817..a505ccac929 100644 --- a/packages/tui/test/prompt/history.test.ts +++ b/packages/tui/test/prompt/history.test.ts @@ -41,4 +41,21 @@ describe("prompt history", () => { const b = entry("describe this", [{ name: "b.png", uri: "data:image/png;base64,BBB" }]) expect(isDuplicateEntry(a, b)).toBe(false) }) + + test("preserves duplicate attachment mentions for prompt restoration", () => { + const value = entry("[Image 1] [Image 1]", [ + { + name: "clipboard", + uri: "data:image/png;base64,AAA", + mention: { start: 0, end: 9, text: "[Image 1]" }, + }, + { + name: "clipboard", + uri: "data:image/png;base64,AAA", + mention: { start: 10, end: 19, text: "[Image 1]" }, + }, + ]) + + expect(parsePromptHistory(JSON.stringify(value))).toEqual([value]) + }) })