fix(core): preserve UTF-8 across shell output pages (#63984)

This commit is contained in:
Aiden Cline
2026-11-14 21:49:50 -06:00
committed by GitHub
Unverified
parent 0cfe2c2610
commit d432a06b9c
3 changed files with 94 additions and 5 deletions
+29 -3
View File
@@ -1,6 +1,7 @@
export * as Shell from "./shell.js"
import path from "path"
import { isUtf8 } from "node:buffer"
import {
Context,
Deferred,
@@ -748,13 +749,23 @@ const layer = () =>
const command = yield* require(id)
const cursor = input?.cursor ?? 0
const limit = input?.limit ?? 65536
const running = command.info.status === "running"
const size = yield* sizeOf(command)
if (cursor >= size) return { output: "", cursor: size, size, truncated: false }
const start = Math.max(0, cursor)
const bytes = yield* readRange(command.info.file, start, Math.min(limit, size - start))
if (limit <= 0) return { output: "", cursor: start, size, truncated: false }
// A UTF-8 character can straddle either edge by up to three bytes.
const offset = Math.max(0, start - 3)
const bytes = yield* readRange(command.info.file, offset, Math.min(size, start + limit + 3) - offset)
const from = start - offset
const to = Math.min(bytes.length, from + limit)
const first = splitCharacter(bytes, from, running)?.end ?? from
const last = splitCharacter(bytes, to, running)
// Keep the usual limit unless even the first complete character would not fit.
const end = last && last.start === first && last.end !== undefined ? last.end : (last?.start ?? to)
return {
output: bytes.toString("utf8"),
cursor: start + bytes.length,
output: bytes.toString("utf8", first, end),
cursor: offset + Math.max(first, end),
size,
truncated: false,
}
@@ -1123,6 +1134,21 @@ const layer = () =>
}),
)
// Find a character crossing this byte offset. A valid prefix at a live EOF has no end yet.
function splitCharacter(bytes: Buffer, cut: number, running: boolean) {
for (let start = cut - 1; start >= Math.max(0, cut - 3); start--) {
const lead = bytes[start]
if ((lead & 0xc0) === 0x80) continue
if (lead < 0xc2 || lead > 0xf4) return
const end = start + (lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4)
if (end <= cut) return
const char = bytes.subarray(start, end)
if (end <= bytes.length) return isUtf8(char) ? { start, end } : undefined
if (running && new TextDecoder().decode(char, { stream: true }) === "") return { start, end: undefined }
return
}
}
// Spans carry only the command text, truncated; never env, stdin or output. Shells outlive the turn that started
// them, so the session comes from the shell's own metadata rather than the caller's span annotations.
function annotate(info: Info) {
+63 -1
View File
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { existsSync, readdirSync, realpathSync } from "node:fs"
import { mkdir, rm, utimes, writeFile } from "node:fs/promises"
import { appendFile, mkdir, rm, utimes, writeFile } from "node:fs/promises"
import path from "node:path"
import { Context, Effect, Fiber, Layer } from "effect"
import { TestClock } from "effect/testing"
@@ -103,6 +103,68 @@ describe("Shell on the process daemon", () => {
}),
)
it.live("pages shell output at UTF-8 boundaries with byte cursors", () =>
Effect.gen(function* () {
const env = yield* daemonShells()
yield* env.withShell(env.directory, (shell) =>
Effect.gen(function* () {
const info = yield* shell.create({ shell: "/bin/sh", command: "pages" })
const prefix = "a".repeat(65535)
for (const char of ["é", "€", "😀"]) {
const text = `${prefix}${char}\uFEFFz`
yield* Effect.promise(() => writeFile(info.file, text))
const page = yield* shell.output(info.id)
expect(page.cursor).toBe(prefix.length)
expect(page.size).toBe(Buffer.byteLength(text))
const rest = yield* shell.output(info.id, { cursor: page.cursor })
expect(page.output + rest.output).toBe(text)
expect(rest.cursor).toBe(page.size)
const tiny = yield* shell.output(info.id, { cursor: page.cursor, limit: 1 })
expect(tiny).toEqual({
output: char,
cursor: page.cursor + Buffer.byteLength(char),
size: page.size,
truncated: false,
})
const tail = yield* shell.output(info.id, { cursor: page.cursor + 1 })
expect(tail.output).toBe("\uFEFFz")
expect(tail.cursor).toBe(page.size)
}
expect(yield* shell.output(info.id, { cursor: 65536, limit: 0 })).toMatchObject({ output: "", cursor: 65536 })
}),
)
}),
)
it.live("retries an incomplete live character and consumes malformed bytes at exit", () =>
Effect.gen(function* () {
const env = yield* daemonShells()
yield* env.withShell(env.directory, (shell) =>
Effect.gen(function* () {
const info = yield* shell.create({ shell: "/bin/sh", command: "partial" })
yield* Effect.promise(() => writeFile(info.file, Buffer.from([0xff, 0xf0, 0x9f])))
const page = yield* shell.output(info.id)
expect(page).toEqual({ output: "�", cursor: 1, size: 3, truncated: false })
expect(yield* shell.output(info.id, { cursor: page.cursor, limit: 1 })).toMatchObject({
output: "",
cursor: 1,
})
yield* Effect.promise(() => appendFile(info.file, Buffer.from([0x98, 0x80, 0xe0, 0x80, 0xe2])))
const rest = yield* shell.output(info.id, { cursor: page.cursor })
expect(rest).toEqual({ output: "😀��", cursor: 7, size: 8, truncated: false })
env.daemon.exit(1, 0)
yield* shell.wait(info.id)
expect(yield* shell.output(info.id, { cursor: rest.cursor })).toEqual({
output: "�",
cursor: 8,
size: 8,
truncated: false,
})
}),
)
}),
)
it.live("counts lines like the spawner and reads only what reached the file after a write error", () =>
Effect.gen(function* () {
const env = yield* daemonShells()
@@ -58,6 +58,7 @@ export function DialogShellOutput(props: { shell: ShellInfo; location: LocationR
}
const page = await client.api.shell.output({ id, location, cursor, limit: PAGE_BYTES })
if (disposed) return false
const advanced = page.data.cursor > cursor
cursor = page.data.cursor
setOutput((previous) => {
const next = (previous ?? "") + page.data.output
@@ -65,7 +66,7 @@ export function DialogShellOutput(props: { shell: ShellInfo; location: LocationR
return TextBoundary.tail(next, PAGE_BYTES)
})
setError("")
return cursor < page.data.size
return advanced && cursor < page.data.size
}
const poll = () => {