From fa1260d6ca484fe0fd868aa992ad81ab85343cf3 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:50:24 -0500 Subject: [PATCH] fix(tui): send pasted image and PDF paths to local servers (#59833) --- packages/cli/src/commands/handlers/default.ts | 2 + packages/tui/src/app.tsx | 2 + packages/tui/src/component/prompt/index.tsx | 9 ++-- .../src/component/prompt/local-attachment.ts | 14 +++++ packages/tui/src/context/runtime.tsx | 2 + packages/tui/src/prompt/history.tsx | 6 ++- .../tui/test/prompt/local-attachment.test.ts | 53 +++++++++++++++++-- 7 files changed, 80 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/commands/handlers/default.ts b/packages/cli/src/commands/handlers/default.ts index cbf9d43b59d..c23e2be2739 100644 --- a/packages/cli/src/commands/handlers/default.ts +++ b/packages/cli/src/commands/handlers/default.ts @@ -102,6 +102,8 @@ export default Runtime.handler(Commands, (input) => }, server: { endpoint: server.endpoint, + // Managed and standalone servers run here; an explicit --server may be on another machine. + local: requestedServer === undefined, service: service ? { reconnect: (signal) => runServicePromise(service.reconnect(), { signal }), diff --git a/packages/tui/src/app.tsx b/packages/tui/src/app.tsx index 81c0cf98fd4..569f217638a 100644 --- a/packages/tui/src/app.tsx +++ b/packages/tui/src/app.tsx @@ -189,6 +189,7 @@ export type TuiInput = { app: TuiApp server: { endpoint: Endpoint + local?: boolean service?: { reconnect: (signal: AbortSignal) => Promise restart: () => Promise @@ -361,6 +362,7 @@ export const run = Effect.fn("Tui.run")(function* (input: TuiInput) { value={{ platform: process.platform, variables: input.environment, + localServer: input.server.local, }} > [number] = { uri: file.uri, name: file.filename, + path: file.path, mention: { start: extmarkStart, end: extmarkEnd, diff --git a/packages/tui/src/component/prompt/local-attachment.ts b/packages/tui/src/component/prompt/local-attachment.ts index cd5612cba14..a676d46380a 100644 --- a/packages/tui/src/component/prompt/local-attachment.ts +++ b/packages/tui/src/component/prompt/local-attachment.ts @@ -1,4 +1,6 @@ +import { existsSync } from "node:fs" import path from "node:path" +import { pathToFileURL } from "node:url" // Bound filesystem work per terminal paste; the byte budget also bounds staged data. const MAX_PASTED_FILEPATHS = 32 @@ -54,10 +56,22 @@ export async function resolvePastedAttachments(text: string, platform: string) { type: "file" as const, uri: `data:${item.attachment.mime};base64,${Buffer.from(item.attachment.content).toString("base64")}`, filename, + path: path.resolve(item.filepath), } }) } +// A local server reads the file itself so the model sees its path. The pasted bytes are the +// fallback for remote servers and for files deleted before submit, like screenshot drags. +export function promptFilesForServer( + files: readonly T[] | undefined, + localServer: boolean, +) { + return files?.map(({ path: file, ...rest }) => + file && localServer && existsSync(file) ? { ...rest, uri: pathToFileURL(file).href } : rest, + ) +} + const mimeTypes: Record = { ".avif": "image/avif", ".gif": "image/gif", diff --git a/packages/tui/src/context/runtime.tsx b/packages/tui/src/context/runtime.tsx index 31a1142062e..fc996da9982 100644 --- a/packages/tui/src/context/runtime.tsx +++ b/packages/tui/src/context/runtime.tsx @@ -23,6 +23,8 @@ export type TuiPaths = Readonly<{ export type TuiTerminalEnvironment = Readonly<{ platform: string variables?: Readonly> + // The server runs on this machine, so it can read file paths from this terminal. + localServer?: boolean }> export type TuiStartup = Readonly<{ diff --git a/packages/tui/src/prompt/history.tsx b/packages/tui/src/prompt/history.tsx index 00c754c3a0c..f86f059c532 100644 --- a/packages/tui/src/prompt/history.tsx +++ b/packages/tui/src/prompt/history.tsx @@ -17,7 +17,11 @@ export type PastedText = { } } -export type PromptInfo = Types.DeepMutable> & { +// `path` is the local file a pasted attachment was read from. It stays in the TUI. +export type PromptFile = Types.DeepMutable & { path?: string } + +export type PromptInfo = Types.DeepMutable> & { + files?: PromptFile[] pasted: PastedText[] mode?: "normal" | "shell" } diff --git a/packages/tui/test/prompt/local-attachment.test.ts b/packages/tui/test/prompt/local-attachment.test.ts index 187c714c9a6..dc7899f0ce2 100644 --- a/packages/tui/test/prompt/local-attachment.test.ts +++ b/packages/tui/test/prompt/local-attachment.test.ts @@ -4,10 +4,12 @@ import { pathToFileURL } from "node:url" import { MAX_LOCAL_ATTACHMENT_BYTES, parsePastedFilepaths, + promptFilesForServer, readLocalAttachmentWith, resolvePastedAttachments, } from "../../src/component/prompt/local-attachment" import type { LocalFiles } from "../../src/component/prompt/local-attachment" +import type { PromptFile } from "../../src/prompt/history" import { tmpdir } from "../fixture/fixture" function files(input: { mime: string; text?: string; bytes?: Uint8Array }): LocalFiles { @@ -97,7 +99,7 @@ describe("prompt local attachments", () => { for (const input of [file, `'${file}'`, pathToFileURL(file).href]) { expect(await resolvePastedAttachments(input, process.platform)).toEqual([ - { type: "file", uri: "data:image/png;base64,AQID", filename: "one image.png" }, + { type: "file", uri: "data:image/png;base64,AQID", filename: "one image.png", path: file }, ]) } }) @@ -113,8 +115,8 @@ describe("prompt local attachments", () => { `# dropped files\r\n${pathToFileURL(image).href}\r\n${pathToFileURL(pdf).href}`, ]) { expect(await resolvePastedAttachments(input, process.platform)).toEqual([ - { type: "file", uri: "data:image/png;base64,AQID", filename: "one image.png" }, - { type: "file", uri: "data:application/pdf;base64,BAUG", filename: "two file.pdf" }, + { type: "file", uri: "data:image/png;base64,AQID", filename: "one image.png", path: image }, + { type: "file", uri: "data:application/pdf;base64,BAUG", filename: "two file.pdf", path: pdf }, ]) } }) @@ -177,4 +179,49 @@ describe("prompt local attachments", () => { expect(await resolvePastedAttachments(Array(32).fill(file).join(" "), process.platform)).toHaveLength(32) expect(await resolvePastedAttachments(Array(33).fill(file).join(" "), process.platform)).toBeUndefined() }) + + test("sends pasted files by path to a local server and by bytes otherwise", async () => { + await using tmp = await tmpdir() + const image = path.join(tmp.path, "scrëen shot.png") + const photo = path.join(tmp.path, "photo.jpg") + const pdf = path.join(tmp.path, "report.pdf") + await Promise.all([ + Bun.write(image, new Uint8Array([1, 2, 3])), + Bun.write(photo, new Uint8Array([7, 8, 9])), + Bun.write(pdf, new Uint8Array([4, 5, 6])), + ]) + const pasted = await resolvePastedAttachments(`'${image}' ${photo} "${pdf}"`, process.platform) + if (!pasted) throw new Error("expected attachments") + const files: PromptFile[] = [ + ...pasted.map((file, index) => ({ + uri: file.uri ?? "", + name: file.filename, + path: file.path, + mention: { start: index * 10, end: index * 10 + 9, text: `[Image ${index + 1}]` }, + })), + { uri: "data:image/png;base64,Cg==", name: "clipboard" }, + ] + + const mentions = files.map((file) => file.mention) + + expect(promptFilesForServer(files, true)).toEqual([ + { uri: pathToFileURL(image).href, name: "scrëen shot.png", mention: mentions[0] }, + { uri: pathToFileURL(photo).href, name: "photo.jpg", mention: mentions[1] }, + { uri: pathToFileURL(pdf).href, name: "report.pdf", mention: mentions[2] }, + { uri: "data:image/png;base64,Cg==", name: "clipboard" }, + ]) + expect(promptFilesForServer(files, false)).toEqual([ + { uri: "data:image/png;base64,AQID", name: "scrëen shot.png", mention: mentions[0] }, + { uri: "data:image/jpeg;base64,BwgJ", name: "photo.jpg", mention: mentions[1] }, + { uri: "data:application/pdf;base64,BAUG", name: "report.pdf", mention: mentions[2] }, + { uri: "data:image/png;base64,Cg==", name: "clipboard" }, + ]) + + await Bun.file(image).delete() + expect(promptFilesForServer(files, true)?.[0]).toEqual({ + uri: "data:image/png;base64,AQID", + name: "scrëen shot.png", + mention: mentions[0], + }) + }) })