mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
fix(core): preserve UTF-8 across shell output pages (#63984)
This commit is contained in:
committed by
GitHub
Unverified
parent
0cfe2c2610
commit
d432a06b9c
@@ -1,6 +1,7 @@
|
||||
export * as Shell from "./shell.js"
|
||||
|
||||
import path from "path"
|
||||
import { isUtf8 } from "node:buffer"
|
||||
import {
|
||||
Context,
|
||||
Deferred,
|
||||
@@ -748,13 +749,23 @@ const layer = () =>
|
||||
const command = yield* require(id)
|
||||
const cursor = input?.cursor ?? 0
|
||||
const limit = input?.limit ?? 65536
|
||||
const running = command.info.status === "running"
|
||||
const size = yield* sizeOf(command)
|
||||
if (cursor >= size) return { output: "", cursor: size, size, truncated: false }
|
||||
const start = Math.max(0, cursor)
|
||||
const bytes = yield* readRange(command.info.file, start, Math.min(limit, size - start))
|
||||
if (limit <= 0) return { output: "", cursor: start, size, truncated: false }
|
||||
// A UTF-8 character can straddle either edge by up to three bytes.
|
||||
const offset = Math.max(0, start - 3)
|
||||
const bytes = yield* readRange(command.info.file, offset, Math.min(size, start + limit + 3) - offset)
|
||||
const from = start - offset
|
||||
const to = Math.min(bytes.length, from + limit)
|
||||
const first = splitCharacter(bytes, from, running)?.end ?? from
|
||||
const last = splitCharacter(bytes, to, running)
|
||||
// Keep the usual limit unless even the first complete character would not fit.
|
||||
const end = last && last.start === first && last.end !== undefined ? last.end : (last?.start ?? to)
|
||||
return {
|
||||
output: bytes.toString("utf8"),
|
||||
cursor: start + bytes.length,
|
||||
output: bytes.toString("utf8", first, end),
|
||||
cursor: offset + Math.max(first, end),
|
||||
size,
|
||||
truncated: false,
|
||||
}
|
||||
@@ -1123,6 +1134,21 @@ const layer = () =>
|
||||
}),
|
||||
)
|
||||
|
||||
// Find a character crossing this byte offset. A valid prefix at a live EOF has no end yet.
|
||||
function splitCharacter(bytes: Buffer, cut: number, running: boolean) {
|
||||
for (let start = cut - 1; start >= Math.max(0, cut - 3); start--) {
|
||||
const lead = bytes[start]
|
||||
if ((lead & 0xc0) === 0x80) continue
|
||||
if (lead < 0xc2 || lead > 0xf4) return
|
||||
const end = start + (lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4)
|
||||
if (end <= cut) return
|
||||
const char = bytes.subarray(start, end)
|
||||
if (end <= bytes.length) return isUtf8(char) ? { start, end } : undefined
|
||||
if (running && new TextDecoder().decode(char, { stream: true }) === "") return { start, end: undefined }
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Spans carry only the command text, truncated; never env, stdin or output. Shells outlive the turn that started
|
||||
// them, so the session comes from the shell's own metadata rather than the caller's span annotations.
|
||||
function annotate(info: Info) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { existsSync, readdirSync, realpathSync } from "node:fs"
|
||||
import { mkdir, rm, utimes, writeFile } from "node:fs/promises"
|
||||
import { appendFile, mkdir, rm, utimes, writeFile } from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import { Context, Effect, Fiber, Layer } from "effect"
|
||||
import { TestClock } from "effect/testing"
|
||||
@@ -103,6 +103,68 @@ describe("Shell on the process daemon", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("pages shell output at UTF-8 boundaries with byte cursors", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const info = yield* shell.create({ shell: "/bin/sh", command: "pages" })
|
||||
const prefix = "a".repeat(65535)
|
||||
for (const char of ["é", "€", "😀"]) {
|
||||
const text = `${prefix}${char}\uFEFFz`
|
||||
yield* Effect.promise(() => writeFile(info.file, text))
|
||||
const page = yield* shell.output(info.id)
|
||||
expect(page.cursor).toBe(prefix.length)
|
||||
expect(page.size).toBe(Buffer.byteLength(text))
|
||||
const rest = yield* shell.output(info.id, { cursor: page.cursor })
|
||||
expect(page.output + rest.output).toBe(text)
|
||||
expect(rest.cursor).toBe(page.size)
|
||||
const tiny = yield* shell.output(info.id, { cursor: page.cursor, limit: 1 })
|
||||
expect(tiny).toEqual({
|
||||
output: char,
|
||||
cursor: page.cursor + Buffer.byteLength(char),
|
||||
size: page.size,
|
||||
truncated: false,
|
||||
})
|
||||
const tail = yield* shell.output(info.id, { cursor: page.cursor + 1 })
|
||||
expect(tail.output).toBe("\uFEFFz")
|
||||
expect(tail.cursor).toBe(page.size)
|
||||
}
|
||||
expect(yield* shell.output(info.id, { cursor: 65536, limit: 0 })).toMatchObject({ output: "", cursor: 65536 })
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("retries an incomplete live character and consumes malformed bytes at exit", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const info = yield* shell.create({ shell: "/bin/sh", command: "partial" })
|
||||
yield* Effect.promise(() => writeFile(info.file, Buffer.from([0xff, 0xf0, 0x9f])))
|
||||
const page = yield* shell.output(info.id)
|
||||
expect(page).toEqual({ output: "�", cursor: 1, size: 3, truncated: false })
|
||||
expect(yield* shell.output(info.id, { cursor: page.cursor, limit: 1 })).toMatchObject({
|
||||
output: "",
|
||||
cursor: 1,
|
||||
})
|
||||
yield* Effect.promise(() => appendFile(info.file, Buffer.from([0x98, 0x80, 0xe0, 0x80, 0xe2])))
|
||||
const rest = yield* shell.output(info.id, { cursor: page.cursor })
|
||||
expect(rest).toEqual({ output: "😀��", cursor: 7, size: 8, truncated: false })
|
||||
env.daemon.exit(1, 0)
|
||||
yield* shell.wait(info.id)
|
||||
expect(yield* shell.output(info.id, { cursor: rest.cursor })).toEqual({
|
||||
output: "�",
|
||||
cursor: 8,
|
||||
size: 8,
|
||||
truncated: false,
|
||||
})
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("counts lines like the spawner and reads only what reached the file after a write error", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
|
||||
@@ -58,6 +58,7 @@ export function DialogShellOutput(props: { shell: ShellInfo; location: LocationR
|
||||
}
|
||||
const page = await client.api.shell.output({ id, location, cursor, limit: PAGE_BYTES })
|
||||
if (disposed) return false
|
||||
const advanced = page.data.cursor > cursor
|
||||
cursor = page.data.cursor
|
||||
setOutput((previous) => {
|
||||
const next = (previous ?? "") + page.data.output
|
||||
@@ -65,7 +66,7 @@ export function DialogShellOutput(props: { shell: ShellInfo; location: LocationR
|
||||
return TextBoundary.tail(next, PAGE_BYTES)
|
||||
})
|
||||
setError("")
|
||||
return cursor < page.data.size
|
||||
return advanced && cursor < page.data.size
|
||||
}
|
||||
|
||||
const poll = () => {
|
||||
|
||||
Reference in New Issue
Block a user