From d432a06b9c07bab0be4e8d642173b1cb14137f11 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Sat, 14 Nov 2026 21:49:50 -0600 Subject: [PATCH] fix(core): preserve UTF-8 across shell output pages (#63984) --- packages/core/src/shell.ts | 32 +++++++++- packages/core/test/shell-daemon.test.ts | 64 ++++++++++++++++++- .../tui/src/component/dialog-shell-output.tsx | 3 +- 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index fec67d1a372..383becbd619 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -1,6 +1,7 @@ export * as Shell from "./shell.js" import path from "path" +import { isUtf8 } from "node:buffer" import { Context, Deferred, @@ -748,13 +749,23 @@ const layer = () => const command = yield* require(id) const cursor = input?.cursor ?? 0 const limit = input?.limit ?? 65536 + const running = command.info.status === "running" const size = yield* sizeOf(command) if (cursor >= size) return { output: "", cursor: size, size, truncated: false } const start = Math.max(0, cursor) - const bytes = yield* readRange(command.info.file, start, Math.min(limit, size - start)) + if (limit <= 0) return { output: "", cursor: start, size, truncated: false } + // A UTF-8 character can straddle either edge by up to three bytes. + const offset = Math.max(0, start - 3) + const bytes = yield* readRange(command.info.file, offset, Math.min(size, start + limit + 3) - offset) + const from = start - offset + const to = Math.min(bytes.length, from + limit) + const first = splitCharacter(bytes, from, running)?.end ?? from + const last = splitCharacter(bytes, to, running) + // Keep the usual limit unless even the first complete character would not fit. + const end = last && last.start === first && last.end !== undefined ? last.end : (last?.start ?? to) return { - output: bytes.toString("utf8"), - cursor: start + bytes.length, + output: bytes.toString("utf8", first, end), + cursor: offset + Math.max(first, end), size, truncated: false, } @@ -1123,6 +1134,21 @@ const layer = () => }), ) +// Find a character crossing this byte offset. A valid prefix at a live EOF has no end yet. +function splitCharacter(bytes: Buffer, cut: number, running: boolean) { + for (let start = cut - 1; start >= Math.max(0, cut - 3); start--) { + const lead = bytes[start] + if ((lead & 0xc0) === 0x80) continue + if (lead < 0xc2 || lead > 0xf4) return + const end = start + (lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4) + if (end <= cut) return + const char = bytes.subarray(start, end) + if (end <= bytes.length) return isUtf8(char) ? { start, end } : undefined + if (running && new TextDecoder().decode(char, { stream: true }) === "") return { start, end: undefined } + return + } +} + // Spans carry only the command text, truncated; never env, stdin or output. Shells outlive the turn that started // them, so the session comes from the shell's own metadata rather than the caller's span annotations. function annotate(info: Info) { diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index 5051819f124..e5a714260bb 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { existsSync, readdirSync, realpathSync } from "node:fs" -import { mkdir, rm, utimes, writeFile } from "node:fs/promises" +import { appendFile, mkdir, rm, utimes, writeFile } from "node:fs/promises" import path from "node:path" import { Context, Effect, Fiber, Layer } from "effect" import { TestClock } from "effect/testing" @@ -103,6 +103,68 @@ describe("Shell on the process daemon", () => { }), ) + it.live("pages shell output at UTF-8 boundaries with byte cursors", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "pages" }) + const prefix = "a".repeat(65535) + for (const char of ["é", "€", "😀"]) { + const text = `${prefix}${char}\uFEFFz` + yield* Effect.promise(() => writeFile(info.file, text)) + const page = yield* shell.output(info.id) + expect(page.cursor).toBe(prefix.length) + expect(page.size).toBe(Buffer.byteLength(text)) + const rest = yield* shell.output(info.id, { cursor: page.cursor }) + expect(page.output + rest.output).toBe(text) + expect(rest.cursor).toBe(page.size) + const tiny = yield* shell.output(info.id, { cursor: page.cursor, limit: 1 }) + expect(tiny).toEqual({ + output: char, + cursor: page.cursor + Buffer.byteLength(char), + size: page.size, + truncated: false, + }) + const tail = yield* shell.output(info.id, { cursor: page.cursor + 1 }) + expect(tail.output).toBe("\uFEFFz") + expect(tail.cursor).toBe(page.size) + } + expect(yield* shell.output(info.id, { cursor: 65536, limit: 0 })).toMatchObject({ output: "", cursor: 65536 }) + }), + ) + }), + ) + + it.live("retries an incomplete live character and consumes malformed bytes at exit", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "partial" }) + yield* Effect.promise(() => writeFile(info.file, Buffer.from([0xff, 0xf0, 0x9f]))) + const page = yield* shell.output(info.id) + expect(page).toEqual({ output: "�", cursor: 1, size: 3, truncated: false }) + expect(yield* shell.output(info.id, { cursor: page.cursor, limit: 1 })).toMatchObject({ + output: "", + cursor: 1, + }) + yield* Effect.promise(() => appendFile(info.file, Buffer.from([0x98, 0x80, 0xe0, 0x80, 0xe2]))) + const rest = yield* shell.output(info.id, { cursor: page.cursor }) + expect(rest).toEqual({ output: "😀��", cursor: 7, size: 8, truncated: false }) + env.daemon.exit(1, 0) + yield* shell.wait(info.id) + expect(yield* shell.output(info.id, { cursor: rest.cursor })).toEqual({ + output: "�", + cursor: 8, + size: 8, + truncated: false, + }) + }), + ) + }), + ) + it.live("counts lines like the spawner and reads only what reached the file after a write error", () => Effect.gen(function* () { const env = yield* daemonShells() diff --git a/packages/tui/src/component/dialog-shell-output.tsx b/packages/tui/src/component/dialog-shell-output.tsx index 3c5c79609bd..c4df17dcdd4 100644 --- a/packages/tui/src/component/dialog-shell-output.tsx +++ b/packages/tui/src/component/dialog-shell-output.tsx @@ -58,6 +58,7 @@ export function DialogShellOutput(props: { shell: ShellInfo; location: LocationR } const page = await client.api.shell.output({ id, location, cursor, limit: PAGE_BYTES }) if (disposed) return false + const advanced = page.data.cursor > cursor cursor = page.data.cursor setOutput((previous) => { const next = (previous ?? "") + page.data.output @@ -65,7 +66,7 @@ export function DialogShellOutput(props: { shell: ShellInfo; location: LocationR return TextBoundary.tail(next, PAGE_BYTES) }) setError("") - return cursor < page.data.size + return advanced && cursor < page.data.size } const poll = () => {