fix(tui): send pasted image and PDF paths to local servers (#59833)

packages/cli/src/commands/handlers/default.ts

@@ -102,6 +102,8 @@ export default Runtime.handler(Commands, (input) =>
       },
       server: {
         endpoint: server.endpoint,
+        // Managed and standalone servers run here; an explicit --server may be on another machine.
+        local: requestedServer === undefined,
         service: service
           ? {
               reconnect: (signal) => runServicePromise(service.reconnect(), { signal }),

packages/tui/src/app.tsx

@@ -189,6 +189,7 @@ export type TuiInput = {
   app: TuiApp
   server: {
     endpoint: Endpoint
+    local?: boolean
     service?: {
       reconnect: (signal: AbortSignal) => Promise<Endpoint>
       restart: () => Promise<void>
@@ -361,6 +362,7 @@ export const run = Effect.fn("Tui.run")(function* (input: TuiInput) {
                               value={{
                                 platform: process.platform,
                                 variables: input.environment,
+                                localServer: input.server.local,
                               }}
                             >
                               <TuiStartupProvider

packages/tui/src/component/prompt/index.tsx

@@ -50,7 +50,7 @@ import { createFadeIn } from "../../util/signal"
 import { DialogSkill } from "../dialog-skill"
 import { useConfig } from "../../config"
 import { usePromptMove } from "./move"
-import { resolvePastedAttachments } from "./local-attachment"
+import { promptFilesForServer, resolvePastedAttachments } from "./local-attachment"
 import { locationKey, useData } from "../../context/data"
 import { useLocation } from "../../context/location"
 import { useArgs } from "../../context/args"
@@ -1325,7 +1325,7 @@ export function Prompt(props: PromptProps) {
           sessionID: target,
           name: slashHead.name,
           text: slashHead.arguments,
-          files: entry.files,
+          files: promptFilesForServer(entry.files, terminalEnvironment.localServer === true),
           agents: entry.agents,
           skills: entry.skills?.length ? entry.skills : undefined,
           delivery,
@@ -1366,7 +1366,7 @@ export function Prompt(props: PromptProps) {
         .prompt({
           sessionID: target,
           text: inputText,
-          files: entry.files,
+          files: promptFilesForServer(entry.files, terminalEnvironment.localServer === true),
           agents: entry.agents,
           skills: entry.skills?.length ? entry.skills : undefined,
           delivery,
@@ -1483,7 +1483,7 @@ export function Prompt(props: PromptProps) {
     }, 0)
   }
 
-  function pasteAttachment(file: { filename?: string; uri: string }) {
+  function pasteAttachment(file: { filename?: string; uri: string; path?: string }) {
     const currentOffset = input.cursorOffset
     const extmarkStart = currentOffset
     const virtualText = promptAttachmentLabel(store.prompt.files, { uri: file.uri, name: file.filename })
@@ -1503,6 +1503,7 @@ export function Prompt(props: PromptProps) {
     const part: NonNullable<PromptInfo["files"]>[number] = {
       uri: file.uri,
       name: file.filename,
+      path: file.path,
       mention: {
         start: extmarkStart,
         end: extmarkEnd,

packages/tui/src/component/prompt/local-attachment.ts

@@ -1,4 +1,6 @@
+import { existsSync } from "node:fs"
 import path from "node:path"
+import { pathToFileURL } from "node:url"
 
 // Bound filesystem work per terminal paste; the byte budget also bounds staged data.
 const MAX_PASTED_FILEPATHS = 32
@@ -54,10 +56,22 @@ export async function resolvePastedAttachments(text: string, platform: string) {
       type: "file" as const,
       uri: `data:${item.attachment.mime};base64,${Buffer.from(item.attachment.content).toString("base64")}`,
       filename,
+      path: path.resolve(item.filepath),
     }
   })
 }
 
+// A local server reads the file itself so the model sees its path. The pasted bytes are the
+// fallback for remote servers and for files deleted before submit, like screenshot drags.
+export function promptFilesForServer<T extends { uri: string; path?: string }>(
+  files: readonly T[] | undefined,
+  localServer: boolean,
+) {
+  return files?.map(({ path: file, ...rest }) =>
+    file && localServer && existsSync(file) ? { ...rest, uri: pathToFileURL(file).href } : rest,
+  )
+}
+
 const mimeTypes: Record<string, string> = {
   ".avif": "image/avif",
   ".gif": "image/gif",

packages/tui/src/context/runtime.tsx

@@ -23,6 +23,8 @@ export type TuiPaths = Readonly<{
 export type TuiTerminalEnvironment = Readonly<{
   platform: string
   variables?: Readonly<Record<string, string>>
+  // The server runs on this machine, so it can read file paths from this terminal.
+  localServer?: boolean
 }>
 
 export type TuiStartup = Readonly<{

packages/tui/src/prompt/history.tsx

@@ -17,7 +17,11 @@ export type PastedText = {
   }
 }
 
-export type PromptInfo = Types.DeepMutable<Pick<PromptInput.Prompt, "text" | "files" | "agents" | "skills">> & {
+// `path` is the local file a pasted attachment was read from. It stays in the TUI.
+export type PromptFile = Types.DeepMutable<PromptInput.FileAttachment> & { path?: string }
+
+export type PromptInfo = Types.DeepMutable<Pick<PromptInput.Prompt, "text" | "agents" | "skills">> & {
+  files?: PromptFile[]
   pasted: PastedText[]
   mode?: "normal" | "shell"
 }

packages/tui/test/prompt/local-attachment.test.ts

@@ -4,10 +4,12 @@ import { pathToFileURL } from "node:url"
 import {
   MAX_LOCAL_ATTACHMENT_BYTES,
   parsePastedFilepaths,
+  promptFilesForServer,
   readLocalAttachmentWith,
   resolvePastedAttachments,
 } from "../../src/component/prompt/local-attachment"
 import type { LocalFiles } from "../../src/component/prompt/local-attachment"
+import type { PromptFile } from "../../src/prompt/history"
 import { tmpdir } from "../fixture/fixture"
 
 function files(input: { mime: string; text?: string; bytes?: Uint8Array }): LocalFiles {
@@ -97,7 +99,7 @@ describe("prompt local attachments", () => {
 
     for (const input of [file, `'${file}'`, pathToFileURL(file).href]) {
       expect(await resolvePastedAttachments(input, process.platform)).toEqual([
-        { type: "file", uri: "data:image/png;base64,AQID", filename: "one image.png" },
+        { type: "file", uri: "data:image/png;base64,AQID", filename: "one image.png", path: file },
       ])
     }
   })
@@ -113,8 +115,8 @@ describe("prompt local attachments", () => {
       `# dropped files\r\n${pathToFileURL(image).href}\r\n${pathToFileURL(pdf).href}`,
     ]) {
       expect(await resolvePastedAttachments(input, process.platform)).toEqual([
-        { type: "file", uri: "data:image/png;base64,AQID", filename: "one image.png" },
-        { type: "file", uri: "data:application/pdf;base64,BAUG", filename: "two file.pdf" },
+        { type: "file", uri: "data:image/png;base64,AQID", filename: "one image.png", path: image },
+        { type: "file", uri: "data:application/pdf;base64,BAUG", filename: "two file.pdf", path: pdf },
       ])
     }
   })
@@ -177,4 +179,49 @@ describe("prompt local attachments", () => {
     expect(await resolvePastedAttachments(Array(32).fill(file).join(" "), process.platform)).toHaveLength(32)
     expect(await resolvePastedAttachments(Array(33).fill(file).join(" "), process.platform)).toBeUndefined()
   })
+
+  test("sends pasted files by path to a local server and by bytes otherwise", async () => {
+    await using tmp = await tmpdir()
+    const image = path.join(tmp.path, "scrëen shot.png")
+    const photo = path.join(tmp.path, "photo.jpg")
+    const pdf = path.join(tmp.path, "report.pdf")
+    await Promise.all([
+      Bun.write(image, new Uint8Array([1, 2, 3])),
+      Bun.write(photo, new Uint8Array([7, 8, 9])),
+      Bun.write(pdf, new Uint8Array([4, 5, 6])),
+    ])
+    const pasted = await resolvePastedAttachments(`'${image}' ${photo} "${pdf}"`, process.platform)
+    if (!pasted) throw new Error("expected attachments")
+    const files: PromptFile[] = [
+      ...pasted.map((file, index) => ({
+        uri: file.uri ?? "",
+        name: file.filename,
+        path: file.path,
+        mention: { start: index * 10, end: index * 10 + 9, text: `[Image ${index + 1}]` },
+      })),
+      { uri: "data:image/png;base64,Cg==", name: "clipboard" },
+    ]
+
+    const mentions = files.map((file) => file.mention)
+
+    expect(promptFilesForServer(files, true)).toEqual([
+      { uri: pathToFileURL(image).href, name: "scrëen shot.png", mention: mentions[0] },
+      { uri: pathToFileURL(photo).href, name: "photo.jpg", mention: mentions[1] },
+      { uri: pathToFileURL(pdf).href, name: "report.pdf", mention: mentions[2] },
+      { uri: "data:image/png;base64,Cg==", name: "clipboard" },
+    ])
+    expect(promptFilesForServer(files, false)).toEqual([
+      { uri: "data:image/png;base64,AQID", name: "scrëen shot.png", mention: mentions[0] },
+      { uri: "data:image/jpeg;base64,BwgJ", name: "photo.jpg", mention: mentions[1] },
+      { uri: "data:application/pdf;base64,BAUG", name: "report.pdf", mention: mentions[2] },
+      { uri: "data:image/png;base64,Cg==", name: "clipboard" },
+    ])
+
+    await Bun.file(image).delete()
+    expect(promptFilesForServer(files, true)?.[0]).toEqual({
+      uri: "data:image/png;base64,AQID",
+      name: "scrëen shot.png",
+      mention: mentions[0],
+    })
+  })
 })