fix: keep text cuts from splitting characters (#63967)

This commit is contained in:
Aiden Cline
2026-11-14 16:00:40 -06:00
committed by GitHub
Unverified
parent 46744b6469
commit 38bd73789a
14 changed files with 320 additions and 22 deletions
+2 -1
View File
@@ -5,6 +5,7 @@ import { ChildProcess } from "effect/process"
import { Entry, Match } from "@opencode/schema/filesystem"
import { makeLocationNode } from "@opencode/util/effect/app-node"
import { collectStream, waitForAbort } from "@opencode/util/process"
import { TextBoundary } from "@opencode/util/text-boundary"
import { Environment } from "./environment/index.js"
import { NonNegativeInt, PositiveInt, RelativePath } from "./schema.js"
import { RipgrepBinary } from "./ripgrep/binary.js"
@@ -252,7 +253,7 @@ const layer = Layer.effect(
lines: {
text:
match.data.lines.text.length > MAX_LINE_CHARS
? Buffer.from(match.data.lines.text.slice(0, MAX_LINE_CHARS), "utf8").toString() + "..."
? TextBoundary.head(match.data.lines.text, MAX_LINE_CHARS) + "..."
: match.data.lines.text,
},
path: { text: normalizePath(match.data.path.text) },
+5 -3
View File
@@ -8,6 +8,7 @@ import { Database } from "../database/database.js"
import { Bus } from "../bus.js"
import { makeLocationNode } from "@opencode/util/effect/app-node"
import { isExactRootFallback } from "@opencode/util/session-title-fallback"
import { TextBoundary } from "@opencode/util/text-boundary"
import { llmClient } from "../effect/app-node-platform.js"
import { SessionContext } from "./context.js"
import { SessionEvent } from "./event.js"
@@ -29,7 +30,8 @@ export interface Interface {
export class Service extends Context.Service<Service, Interface>()("@opencode/SessionTitle") {}
const truncate = (value: string) => (value.length <= MAX_LENGTH ? value : `${value.slice(0, MAX_LENGTH - 3)}...`)
const truncate = (value: string) =>
value.length <= MAX_LENGTH ? value : `${TextBoundary.head(value, MAX_LENGTH - 3)}...`
// Framing the input as text to name keeps models from answering or carrying out the request instead.
const messagePrompt = (text: string) =>
`Here is the first message of the conversation:\n\n<message>\n${text}\n</message>\n\nReply with only the title.`
@@ -119,7 +121,7 @@ export const layer = Layer.effect(
const text = !isUntitled(session)
? yield* store.context(session.id).pipe(
Effect.map((messages) => {
const original = `Original request:\n${firstUser.text.slice(0, MAX_FIRST_MESSAGE_LENGTH)}`
const original = `Original request:\n${TextBoundary.head(firstUser.text, MAX_FIRST_MESSAGE_LENGTH)}`
const recent = messages
.flatMap((message) => {
if (message.type === "user" && message.id !== firstUser.id) return [`User: ${message.text.trim()}`]
@@ -134,7 +136,7 @@ export const layer = Layer.effect(
if (!recent) return conversationPrompt(original)
const prefix = `${original}\n\nRecent conversation:\n`
const budget = MAX_CONTEXT_LENGTH - conversationPrompt("").length - prefix.length
return conversationPrompt(`${prefix}${recent.slice(-budget)}`)
return conversationPrompt(`${prefix}${TextBoundary.tail(recent, budget)}`)
}),
Effect.orElseSucceed(() => messagePrompt(firstUser.text)),
)
+2 -8
View File
@@ -22,6 +22,7 @@ import { AppProcess } from "@opencode/util/process"
import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner"
import { makeGlobalNode, makeLocationNode } from "@opencode/util/effect/app-node"
import { FSUtil } from "@opencode/util/fs-util"
import { TextBoundary } from "@opencode/util/text-boundary"
import { Bus } from "./bus.js"
import { Environment } from "./environment/index.js"
import { FileRetention } from "./file-retention.js"
@@ -825,7 +826,7 @@ const layer = () =>
// Read the byte before the window too, to tell whether the window starts on a line boundary.
const before = Math.max(0, start - 1)
const window = yield* readRange(info.file, before, size - before)
const page = window.subarray(charBoundary(window, start - before)).toString("utf8")
const page = TextBoundary.utf8Tail(window, size - start)
const partial = start > 0 && window[0] !== 10
const endsWithNewline = page.endsWith("\n")
const lines = page.split("\n")
@@ -1142,13 +1143,6 @@ function spanCommand(command: string) {
return `${command.slice(0, SPAN_COMMAND_LIMIT).toWellFormed()}…`
}
// Moves past UTF-8 continuation bytes (at most 3) so a byte cut never splits a character.
function charBoundary(bytes: Buffer, offset: number) {
const limit = Math.min(bytes.length, offset + 3)
for (let index = offset; index < limit; index++) if ((bytes[index] & 0xc0) !== 0x80) return index
return limit
}
function formatSize(bytes: number) {
const kb = bytes / 1024
return kb > 1024 ? `${(kb / 1024).toFixed(1)} MB` : `${Math.round(kb)} KB`
+3 -6
View File
@@ -3,6 +3,7 @@ export * as ReadToolFileSystem from "./read-filesystem.js"
import path from "path"
import { pathToFileURL } from "url"
import { makeLocationNode } from "@opencode/util/effect/app-node"
import { TextBoundary } from "@opencode/util/text-boundary"
import { Context, Effect, Layer, Schema } from "effect"
import { lookup } from "mime-types"
import { Environment } from "../environment/index.js"
@@ -374,12 +375,8 @@ const list = (items: ReadonlyArray<Environment.DirEntry>, page: PageInput, sep =
})
}
const clampLine = (line: string) => {
if (line.length <= MAX_LINE_LENGTH) return line
const code = line.charCodeAt(MAX_LINE_LENGTH - 1)
const end = code >= 0xd800 && code <= 0xdbff ? MAX_LINE_LENGTH - 1 : MAX_LINE_LENGTH
return line.slice(0, end) + MAX_LINE_SUFFIX
}
const clampLine = (line: string) =>
line.length <= MAX_LINE_LENGTH ? line : TextBoundary.head(line, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX
const layer = Layer.effect(
Service,
+22
View File
@@ -184,6 +184,28 @@ describe("Ripgrep", () => {
}),
),
)
it.live("ends oversized line previews on a whole character", () =>
withTempDir((tmp) =>
Effect.gen(function* () {
yield* Effect.promise(() =>
fs.writeFile(
path.join(tmp.path, "emoji.txt"),
[1_998, 1_999, 2_000].map((count) => `${"x".repeat(count)}😀tail\n`).join(""),
),
)
const ripgrep = yield* Ripgrep.Service
const matches = yield* ripgrep.grep({ cwd: tmp.path, pattern: "tail", limit: 10 })
expect(matches.map((match) => match.text)).toEqual([
`${"x".repeat(1_998)}😀...`,
`${"x".repeat(1_999)}...`,
`${"x".repeat(2_000)}...`,
])
}),
),
)
})
describe("RipgrepBinary.verify", () => {
+46
View File
@@ -634,6 +634,52 @@ it.effect("bounds regeneration context while preserving the original request and
}),
)
it.effect("keeps the clipped regeneration context well formed", () =>
Effect.gen(function* () {
yield* enableTitleAgent
const sessionID = Session.ID.make("ses_title_regenerate_emoji")
yield* insertSession(sessionID, "Original title")
// The leading x puts both cuts, the original request's head and the recent conversation's tail, inside an emoji.
yield* prompt(sessionID, `x${"😀".repeat(1_500)}`)
yield* prompt(sessionID, `x${"😀".repeat(5_000)}`)
const title = yield* SessionTitle.Service
yield* title.generate(sessionID)
const content = requests[0]?.messages[0]?.content[0]
expect(content?.type).toBe("text")
if (content?.type !== "text") return
expect(content.text.length).toBeLessThanOrEqual(8_000)
expect(content.text.length).toBeGreaterThan(7_900)
expect(content.text.isWellFormed()).toBe(true)
}),
)
for (const [prefix, stored] of [
[95, `${"a".repeat(95)}😀...`],
[96, `${"a".repeat(96)}...`],
[97, `${"a".repeat(97)}...`],
] as const)
it.effect(`clips a long generated title without splitting an emoji after ${prefix} letters`, () =>
Effect.gen(function* () {
yield* enableTitleAgent
titleStream = () =>
Stream.make(
LLMEvent.textDelta({ id: "title", text: `${"a".repeat(prefix)}😀${"b".repeat(20)}` }),
LLMEvent.finish({ reason: { normalized: "stop" } }),
)
const sessionID = Session.ID.make(`ses_title_emoji_${prefix}`)
yield* insertSession(sessionID)
yield* prompt(sessionID, "Name this session")
const title = yield* SessionTitle.Service
yield* title.generate(sessionID)
const store = yield* SessionStore.Service
expect((yield* store.get(sessionID))?.title).toBe(stored)
}),
)
it.effect("preserves the existing title when regeneration fails", () =>
Effect.gen(function* () {
yield* enableTitleAgent
@@ -173,6 +173,31 @@ describe("transitionShellFollower", () => {
expect(stopped.effects).toEqual([{ type: "read", attempt: 3, cursor: 1 }])
})
test("keeps the retained tail on a whole character when the window cuts an emoji", () => {
// Each page is valid text; only the rolling 65,536-unit window would start on the second half of the emoji.
const first = fresh(false)
if (first.status !== "reading") throw new Error(`expected a read, got ${first.status}`)
const next = transitionShellFollower(first, {
type: "read.succeeded",
attempt: first.attempt,
page: { output: "😀", cursor: 4, size: 65_539 },
}).state
if (next.status !== "reading") throw new Error(`expected a read, got ${next.status}`)
const done = transitionShellFollower(next, {
type: "read.succeeded",
attempt: next.attempt,
page: { output: "x".repeat(65_535), cursor: 65_539, size: 65_539 },
})
expect(done.state).toMatchObject({ status: "complete", output: "x".repeat(65_535) })
expect(done.effects).toContainEqual({ type: "emit", output: "x".repeat(65_535) })
expect(done.effects).toContainEqual({ type: "cache", cursor: 65_539, output: "x".repeat(65_535), complete: true })
})
test("a missing shell is remembered and never read again", () => {
const missing = transitionShellFollower(fresh(true), { type: "read.missing", attempt: 1 })
@@ -1,3 +1,5 @@
import { TextBoundary } from "@opencode/util/text-boundary"
// Same page size as the TUI shell output viewer; only this much recent output is retained and rendered.
export const SHELL_OUTPUT_TAIL_BYTES = 64 * 1024
@@ -144,7 +146,7 @@ export function transitionShellFollower(state: ShellFollowerState, event: ShellF
const page = event.page
const advanced = page.cursor > state.cursor
const cursor = Math.max(state.cursor, page.cursor)
const output = (state.output + page.output).slice(-SHELL_OUTPUT_TAIL_BYTES)
const output = TextBoundary.tail(state.output + page.output, SHELL_OUTPUT_TAIL_BYTES)
const emit = { type: "emit", output } as const
const attempt = state.attempt + 1
@@ -3,6 +3,7 @@ import { useTerminalDimensions } from "@opentui/solid"
import { isShellNotFoundError, type LocationRef, type ShellInfo } from "@opencode/client"
import { createEffect, createMemo, createSignal, onCleanup, Show, untrack } from "solid-js"
import stripAnsi from "strip-ansi"
import { TextBoundary } from "@opencode/util/text-boundary"
import { useNewDesigns } from "../config"
import { useClient } from "../context/client"
import { useClipboard } from "../context/clipboard"
@@ -61,7 +62,7 @@ export function DialogShellOutput(props: { shell: ShellInfo; location: LocationR
setOutput((previous) => {
const next = (previous ?? "") + page.data.output
if (next.length > PAGE_BYTES) setOmitted(true)
return next.slice(-PAGE_BYTES)
return TextBoundary.tail(next, PAGE_BYTES)
})
setError("")
return cursor < page.data.size
+2 -1
View File
@@ -120,6 +120,7 @@ import { stringWidth } from "../../util/string-width"
import { useArgs } from "../../context/args"
import { withTimestampedFallback } from "@opencode/util/session-title-fallback"
import { SessionTurn } from "@opencode/util/session-turn"
import { TextBoundary } from "@opencode/util/text-boundary"
import { useSessionTabs, type ScrollAnchor } from "../../context/session-tabs"
import { createSingleFlight } from "../../util/single-flight"
import { createDelayedPresence } from "../../util/delayed-presence"
@@ -3069,7 +3070,7 @@ function ShellDisplay(props: {
const next = stripAnsi(output + response.data.output)
if (next.length <= SHELL_DISPLAY_LIMIT) return next
setOutputTruncated(true)
return next.slice(-SHELL_DISPLAY_LIMIT)
return TextBoundary.tail(next, SHELL_DISPLAY_LIMIT)
})
if (response.data.cursor <= cursor) break
cursor = response.data.cursor
@@ -35,6 +35,7 @@ async function setup(width: number, output = "", newDesigns = false) {
}
const state = { output, missing: false, failure: false }
const requests: { url: URL; method: string }[] = []
const copied: string[] = []
const events = createEventStream()
const envelope = (data: unknown) => json({ location, data })
const api = createApi(
@@ -67,7 +68,11 @@ async function setup(width: number, output = "", newDesigns = false) {
const app = await testRender(
() => (
<TestTuiContexts directory={temporary.path} paths={{ state: temporary.path }}>
<TestTuiContexts
directory={temporary.path}
paths={{ state: temporary.path }}
clipboard={{ read: async () => undefined, write: async (text) => void copied.push(text) }}
>
<TuiAppProvider value={{ name: "test", version: "test", channel: "test" }}>
<StorageProvider>
<ConfigProvider config={createTuiResolvedConfig({ experimental: { "new-designs": newDesigns } })}>
@@ -101,6 +106,7 @@ async function setup(width: number, output = "", newDesigns = false) {
shell,
location,
requests,
copied,
events,
async [Symbol.asyncDispose]() {
app.renderer.destroy()
@@ -234,3 +240,16 @@ test("new designs show footer actions instead of the legacy hint line", async ()
expect(frame).toContain("copy output o")
expect(frame).not.toContain("back esc")
})
test("the retained 64 KiB tail starts on a whole character", async () => {
await using app = await setup(100, "😀", true)
app.mockInput.pressEnter()
await app.waitForFrame((frame) => frame.includes("Shell output") && frame.includes("😀"))
// The second page brings the text to 64 KiB + 1 code units, so the tail would start on the emoji's second half.
const rest = ("x".repeat(1023) + "\n").repeat(63) + "x".repeat(1023)
app.state.output += rest
await app.waitForFrame((frame) => frame.includes("Earlier output omitted"), { maxPasses: 100 })
app.mockInput.pressKey("o")
await app.waitFor(() => app.copied.length > 0)
expect(app.copied).toEqual([rest])
})
@@ -0,0 +1,113 @@
import { expect, test } from "bun:test"
import { TextRenderable, type Renderable } from "@opentui/core"
import { createTestRenderer } from "@opentui/core/testing"
import { Effect, FileSystem } from "effect"
import { Global } from "@opencode/util/global"
import type { SessionMessageInfo } from "@opencode/client"
import { createEventStream, createFetch, directory, json } from "./fixture/tui-client"
import { tmpdir } from "./fixture/fixture"
const session = {
id: "ses_shell_output",
title: "Shell output",
projectID: "proj_test",
location: { directory },
cost: 0,
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
time: { created: 0, updated: 0 },
}
const messages = [
{ type: "user", id: "user-1", text: "Run it", time: { created: 0 } },
{
type: "assistant",
id: "assistant-1",
agent: "build",
model: { providerID: "fixture", id: "fixture" },
finish: "stop",
time: { created: 1, completed: 3 },
content: [
{
type: "tool",
id: "call-background",
name: "shell",
time: { created: 1, completed: 2 },
state: {
input: { command: "render-frames" },
status: "completed",
content: [],
metadata: { shellID: "sh_background", status: "running" },
},
},
{ type: "text", text: "Started it" },
],
},
] as SessionMessageInfo[]
test("an expanded background shell keeps its 1 MiB tail on a whole character", async () => {
await using state = await tmpdir()
// Two 1 MiB byte pages decode to 1 MiB + 1 code units, so the retained tail would start on the emoji's second half.
const rest = ("x".repeat(1023) + "\n").repeat(1023) + "x".repeat(1023)
const bytes = Buffer.from(`😀${rest}`)
const setup = await createTestRenderer({ width: 100, height: 40, kittyKeyboard: true })
await setup.renderer.setupTerminal()
setup.renderer.start()
const calls = createFetch((url) => {
if (url.pathname === "/api/session") return json({ data: [session], cursor: {} })
if (url.pathname === `/api/session/${session.id}`) return json({ data: session })
if (url.pathname === `/api/session/${session.id}/message`) return json({ data: messages.toReversed(), cursor: {} })
if (url.pathname === `/api/session/${session.id}/inbox` || url.pathname === `/api/session/${session.id}/permission`)
return json({ data: [] })
if (url.pathname === "/api/shell/sh_background/output") {
const cursor = Number(url.searchParams.get("cursor"))
const end = Math.min(bytes.length, cursor + Number(url.searchParams.get("limit")))
return json({ data: { output: bytes.subarray(cursor, end).toString(), cursor: end, size: bytes.length } })
}
}, createEventStream())
const server = Bun.serve({ port: 0, idleTimeout: 0, fetch: (request) => calls.fetch(request) })
const { run } = await import("../src/app")
const task = Effect.runPromise(
run({
app: { name: "test", version: "test", channel: "test" },
server: { endpoint: { url: server.url.toString() } },
config: {
get: async () => ({ animations: false, tabs: { enabled: false }, session: { verbosity: "high" } }),
update: async () => ({}),
},
packages: { prepare: async () => ({ directory: "" }) },
args: { sessionID: session.id },
terminalHandoff: async () => ({ renderer: setup.renderer, mode: "dark", complete: () => {} }),
log: () => {},
}).pipe(Effect.provide(Global.layerWith({ state: state.path })), Effect.provide(FileSystem.layerNoop({}))),
)
try {
await setup.waitForFrame((frame) => frame.includes("Started it"))
const row = setup
.captureCharFrame()
.split("\n")
.findIndex((line) => line.includes("$ render-frames"))
await setup.mockMouse.click(10, row)
const output = await waitForText(setup.renderer.root, (text) => text.startsWith("[earlier output omitted]"))
expect(output).toBe(`[earlier output omitted]\n${rest}`)
} finally {
setup.renderer.destroy()
await task
await server.stop()
}
})
async function waitForText(root: Renderable, matches: (text: string) => boolean) {
const deadline = Date.now() + 5_000
while (Date.now() < deadline) {
const found = texts(root).find(matches)
if (found !== undefined) return found
await Bun.sleep(10)
}
throw new Error("Timed out waiting for shell output")
}
function texts(node: Renderable): string[] {
return [
...(node instanceof TextRenderable ? [node.plainText] : []),
...node.getChildren().flatMap((child) => texts(child as Renderable)),
]
}
+39
View File
@@ -0,0 +1,39 @@
import { describe, expect, test } from "bun:test"
import { TextBoundary } from "./text-boundary.js"
const encoder = new TextEncoder()
const samples = ["", "ascii", "ab😀cd", "😀é€\uFFFD😀"]
describe("TextBoundary", () => {
test.each(samples)("every head and tail of %j keeps whole characters within the limit", (text) => {
for (let limit = 0; limit <= text.length + 1; limit++) {
const head = TextBoundary.head(text, limit)
const tail = TextBoundary.tail(text, limit)
expect(text.startsWith(head)).toBe(true)
expect(text.endsWith(tail)).toBe(true)
for (const part of [head, tail]) {
expect(part.isWellFormed()).toBe(true)
expect(part.length).toBeLessThanOrEqual(limit)
expect(part.length).toBeGreaterThanOrEqual(Math.min(limit, text.length) - 1)
}
}
})
test.each(samples)("every UTF-8 tail of %j keeps whole characters within the limit", (text) => {
const bytes = encoder.encode(text)
for (let limit = 0; limit <= bytes.length + 1; limit++) {
const tail = TextBoundary.utf8Tail(bytes, limit)
expect(text.endsWith(tail)).toBe(true)
expect(encoder.encode(tail).length).toBeLessThanOrEqual(limit)
expect(encoder.encode(tail).length).toBeGreaterThanOrEqual(Math.min(limit, bytes.length) - 3)
}
})
test("keeps malformed input that is not at the cut", () => {
expect(TextBoundary.head("a\ud83d", 2)).toBe("a\ud83d")
expect(TextBoundary.head("a\ud83db", 2)).toBe("a")
expect(TextBoundary.tail("\ude00a", 2)).toBe("\ude00a")
expect(TextBoundary.tail("b\ude00a", 2)).toBe("a")
expect(TextBoundary.utf8Tail(Uint8Array.of(0x80, 0x41), 2)).toBe("\uFFFDA")
})
})
+36
View File
@@ -0,0 +1,36 @@
export * as TextBoundary from "./text-boundary.js"
/**
* Clip text to a budget without ending or starting on half a character. `head` and `tail` count UTF-16 code units,
* like `.length`; `utf8Tail` counts bytes of UTF-8 output that has not been decoded yet. The result can be shorter
* than the budget by the part of the character at the cut, and `head("😀", 1)` is empty. Boundaries are code points,
* not graphemes, and malformed input is kept as it is everywhere except at the cut.
*/
/** The first `limit` code units of `text`, without a high surrogate whose pair the limit cut off. */
export function head(text: string, limit: number) {
if (text.length <= limit) return text
const code = text.charCodeAt(limit - 1)
return text.slice(0, code >= 0xd800 && code <= 0xdbff ? limit - 1 : limit)
}
/** The last `limit` code units of `text`, without a low surrogate whose pair the limit cut off. */
export function tail(text: string, limit: number) {
if (text.length <= limit) return text
const start = text.length - limit
const code = text.charCodeAt(start)
return text.slice(code >= 0xdc00 && code <= 0xdfff ? start + 1 : start)
}
/** The last `limit` bytes of `bytes`, decoded, after skipping up to 3 continuation bytes of a cut character. */
export function utf8Tail(bytes: Uint8Array, limit: number) {
if (bytes.length <= limit) return decoder.decode(bytes)
const start = bytes.length - limit
const end = Math.min(bytes.length, start + 3)
let index = start
while (index < end && (bytes[index] & 0xc0) === 0x80) index++
return decoder.decode(bytes.subarray(index))
}
// Like Buffer#toString, keep a leading byte order mark.
const decoder = new TextDecoder("utf-8", { ignoreBOM: true })