mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
fix: keep text cuts from splitting characters (#63967)
This commit is contained in:
committed by
GitHub
Unverified
parent
46744b6469
commit
38bd73789a
@@ -5,6 +5,7 @@ import { ChildProcess } from "effect/process"
|
||||
import { Entry, Match } from "@opencode/schema/filesystem"
|
||||
import { makeLocationNode } from "@opencode/util/effect/app-node"
|
||||
import { collectStream, waitForAbort } from "@opencode/util/process"
|
||||
import { TextBoundary } from "@opencode/util/text-boundary"
|
||||
import { Environment } from "./environment/index.js"
|
||||
import { NonNegativeInt, PositiveInt, RelativePath } from "./schema.js"
|
||||
import { RipgrepBinary } from "./ripgrep/binary.js"
|
||||
@@ -252,7 +253,7 @@ const layer = Layer.effect(
|
||||
lines: {
|
||||
text:
|
||||
match.data.lines.text.length > MAX_LINE_CHARS
|
||||
? Buffer.from(match.data.lines.text.slice(0, MAX_LINE_CHARS), "utf8").toString() + "..."
|
||||
? TextBoundary.head(match.data.lines.text, MAX_LINE_CHARS) + "..."
|
||||
: match.data.lines.text,
|
||||
},
|
||||
path: { text: normalizePath(match.data.path.text) },
|
||||
|
||||
@@ -8,6 +8,7 @@ import { Database } from "../database/database.js"
|
||||
import { Bus } from "../bus.js"
|
||||
import { makeLocationNode } from "@opencode/util/effect/app-node"
|
||||
import { isExactRootFallback } from "@opencode/util/session-title-fallback"
|
||||
import { TextBoundary } from "@opencode/util/text-boundary"
|
||||
import { llmClient } from "../effect/app-node-platform.js"
|
||||
import { SessionContext } from "./context.js"
|
||||
import { SessionEvent } from "./event.js"
|
||||
@@ -29,7 +30,8 @@ export interface Interface {
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/SessionTitle") {}
|
||||
|
||||
const truncate = (value: string) => (value.length <= MAX_LENGTH ? value : `${value.slice(0, MAX_LENGTH - 3)}...`)
|
||||
const truncate = (value: string) =>
|
||||
value.length <= MAX_LENGTH ? value : `${TextBoundary.head(value, MAX_LENGTH - 3)}...`
|
||||
// Framing the input as text to name keeps models from answering or carrying out the request instead.
|
||||
const messagePrompt = (text: string) =>
|
||||
`Here is the first message of the conversation:\n\n<message>\n${text}\n</message>\n\nReply with only the title.`
|
||||
@@ -119,7 +121,7 @@ export const layer = Layer.effect(
|
||||
const text = !isUntitled(session)
|
||||
? yield* store.context(session.id).pipe(
|
||||
Effect.map((messages) => {
|
||||
const original = `Original request:\n${firstUser.text.slice(0, MAX_FIRST_MESSAGE_LENGTH)}`
|
||||
const original = `Original request:\n${TextBoundary.head(firstUser.text, MAX_FIRST_MESSAGE_LENGTH)}`
|
||||
const recent = messages
|
||||
.flatMap((message) => {
|
||||
if (message.type === "user" && message.id !== firstUser.id) return [`User: ${message.text.trim()}`]
|
||||
@@ -134,7 +136,7 @@ export const layer = Layer.effect(
|
||||
if (!recent) return conversationPrompt(original)
|
||||
const prefix = `${original}\n\nRecent conversation:\n`
|
||||
const budget = MAX_CONTEXT_LENGTH - conversationPrompt("").length - prefix.length
|
||||
return conversationPrompt(`${prefix}${recent.slice(-budget)}`)
|
||||
return conversationPrompt(`${prefix}${TextBoundary.tail(recent, budget)}`)
|
||||
}),
|
||||
Effect.orElseSucceed(() => messagePrompt(firstUser.text)),
|
||||
)
|
||||
|
||||
@@ -22,6 +22,7 @@ import { AppProcess } from "@opencode/util/process"
|
||||
import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner"
|
||||
import { makeGlobalNode, makeLocationNode } from "@opencode/util/effect/app-node"
|
||||
import { FSUtil } from "@opencode/util/fs-util"
|
||||
import { TextBoundary } from "@opencode/util/text-boundary"
|
||||
import { Bus } from "./bus.js"
|
||||
import { Environment } from "./environment/index.js"
|
||||
import { FileRetention } from "./file-retention.js"
|
||||
@@ -825,7 +826,7 @@ const layer = () =>
|
||||
// Read the byte before the window too, to tell whether the window starts on a line boundary.
|
||||
const before = Math.max(0, start - 1)
|
||||
const window = yield* readRange(info.file, before, size - before)
|
||||
const page = window.subarray(charBoundary(window, start - before)).toString("utf8")
|
||||
const page = TextBoundary.utf8Tail(window, size - start)
|
||||
const partial = start > 0 && window[0] !== 10
|
||||
const endsWithNewline = page.endsWith("\n")
|
||||
const lines = page.split("\n")
|
||||
@@ -1142,13 +1143,6 @@ function spanCommand(command: string) {
|
||||
return `${command.slice(0, SPAN_COMMAND_LIMIT).toWellFormed()}…`
|
||||
}
|
||||
|
||||
// Moves past UTF-8 continuation bytes (at most 3) so a byte cut never splits a character.
|
||||
function charBoundary(bytes: Buffer, offset: number) {
|
||||
const limit = Math.min(bytes.length, offset + 3)
|
||||
for (let index = offset; index < limit; index++) if ((bytes[index] & 0xc0) !== 0x80) return index
|
||||
return limit
|
||||
}
|
||||
|
||||
function formatSize(bytes: number) {
|
||||
const kb = bytes / 1024
|
||||
return kb > 1024 ? `${(kb / 1024).toFixed(1)} MB` : `${Math.round(kb)} KB`
|
||||
|
||||
@@ -3,6 +3,7 @@ export * as ReadToolFileSystem from "./read-filesystem.js"
|
||||
import path from "path"
|
||||
import { pathToFileURL } from "url"
|
||||
import { makeLocationNode } from "@opencode/util/effect/app-node"
|
||||
import { TextBoundary } from "@opencode/util/text-boundary"
|
||||
import { Context, Effect, Layer, Schema } from "effect"
|
||||
import { lookup } from "mime-types"
|
||||
import { Environment } from "../environment/index.js"
|
||||
@@ -374,12 +375,8 @@ const list = (items: ReadonlyArray<Environment.DirEntry>, page: PageInput, sep =
|
||||
})
|
||||
}
|
||||
|
||||
const clampLine = (line: string) => {
|
||||
if (line.length <= MAX_LINE_LENGTH) return line
|
||||
const code = line.charCodeAt(MAX_LINE_LENGTH - 1)
|
||||
const end = code >= 0xd800 && code <= 0xdbff ? MAX_LINE_LENGTH - 1 : MAX_LINE_LENGTH
|
||||
return line.slice(0, end) + MAX_LINE_SUFFIX
|
||||
}
|
||||
const clampLine = (line: string) =>
|
||||
line.length <= MAX_LINE_LENGTH ? line : TextBoundary.head(line, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX
|
||||
|
||||
const layer = Layer.effect(
|
||||
Service,
|
||||
|
||||
@@ -184,6 +184,28 @@ describe("Ripgrep", () => {
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
it.live("ends oversized line previews on a whole character", () =>
|
||||
withTempDir((tmp) =>
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.promise(() =>
|
||||
fs.writeFile(
|
||||
path.join(tmp.path, "emoji.txt"),
|
||||
[1_998, 1_999, 2_000].map((count) => `${"x".repeat(count)}😀tail\n`).join(""),
|
||||
),
|
||||
)
|
||||
|
||||
const ripgrep = yield* Ripgrep.Service
|
||||
const matches = yield* ripgrep.grep({ cwd: tmp.path, pattern: "tail", limit: 10 })
|
||||
|
||||
expect(matches.map((match) => match.text)).toEqual([
|
||||
`${"x".repeat(1_998)}😀...`,
|
||||
`${"x".repeat(1_999)}...`,
|
||||
`${"x".repeat(2_000)}...`,
|
||||
])
|
||||
}),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
describe("RipgrepBinary.verify", () => {
|
||||
|
||||
@@ -634,6 +634,52 @@ it.effect("bounds regeneration context while preserving the original request and
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps the clipped regeneration context well formed", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* enableTitleAgent
|
||||
const sessionID = Session.ID.make("ses_title_regenerate_emoji")
|
||||
yield* insertSession(sessionID, "Original title")
|
||||
// The leading x puts both cuts, the original request's head and the recent conversation's tail, inside an emoji.
|
||||
yield* prompt(sessionID, `x${"😀".repeat(1_500)}`)
|
||||
yield* prompt(sessionID, `x${"😀".repeat(5_000)}`)
|
||||
|
||||
const title = yield* SessionTitle.Service
|
||||
yield* title.generate(sessionID)
|
||||
|
||||
const content = requests[0]?.messages[0]?.content[0]
|
||||
expect(content?.type).toBe("text")
|
||||
if (content?.type !== "text") return
|
||||
expect(content.text.length).toBeLessThanOrEqual(8_000)
|
||||
expect(content.text.length).toBeGreaterThan(7_900)
|
||||
expect(content.text.isWellFormed()).toBe(true)
|
||||
}),
|
||||
)
|
||||
|
||||
for (const [prefix, stored] of [
|
||||
[95, `${"a".repeat(95)}😀...`],
|
||||
[96, `${"a".repeat(96)}...`],
|
||||
[97, `${"a".repeat(97)}...`],
|
||||
] as const)
|
||||
it.effect(`clips a long generated title without splitting an emoji after ${prefix} letters`, () =>
|
||||
Effect.gen(function* () {
|
||||
yield* enableTitleAgent
|
||||
titleStream = () =>
|
||||
Stream.make(
|
||||
LLMEvent.textDelta({ id: "title", text: `${"a".repeat(prefix)}😀${"b".repeat(20)}` }),
|
||||
LLMEvent.finish({ reason: { normalized: "stop" } }),
|
||||
)
|
||||
const sessionID = Session.ID.make(`ses_title_emoji_${prefix}`)
|
||||
yield* insertSession(sessionID)
|
||||
yield* prompt(sessionID, "Name this session")
|
||||
|
||||
const title = yield* SessionTitle.Service
|
||||
yield* title.generate(sessionID)
|
||||
|
||||
const store = yield* SessionStore.Service
|
||||
expect((yield* store.get(sessionID))?.title).toBe(stored)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves the existing title when regeneration fails", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* enableTitleAgent
|
||||
|
||||
@@ -173,6 +173,31 @@ describe("transitionShellFollower", () => {
|
||||
expect(stopped.effects).toEqual([{ type: "read", attempt: 3, cursor: 1 }])
|
||||
})
|
||||
|
||||
test("keeps the retained tail on a whole character when the window cuts an emoji", () => {
|
||||
// Each page is valid text; only the rolling 65,536-unit window would start on the second half of the emoji.
|
||||
const first = fresh(false)
|
||||
|
||||
if (first.status !== "reading") throw new Error(`expected a read, got ${first.status}`)
|
||||
|
||||
const next = transitionShellFollower(first, {
|
||||
type: "read.succeeded",
|
||||
attempt: first.attempt,
|
||||
page: { output: "😀", cursor: 4, size: 65_539 },
|
||||
}).state
|
||||
|
||||
if (next.status !== "reading") throw new Error(`expected a read, got ${next.status}`)
|
||||
|
||||
const done = transitionShellFollower(next, {
|
||||
type: "read.succeeded",
|
||||
attempt: next.attempt,
|
||||
page: { output: "x".repeat(65_535), cursor: 65_539, size: 65_539 },
|
||||
})
|
||||
|
||||
expect(done.state).toMatchObject({ status: "complete", output: "x".repeat(65_535) })
|
||||
expect(done.effects).toContainEqual({ type: "emit", output: "x".repeat(65_535) })
|
||||
expect(done.effects).toContainEqual({ type: "cache", cursor: 65_539, output: "x".repeat(65_535), complete: true })
|
||||
})
|
||||
|
||||
test("a missing shell is remembered and never read again", () => {
|
||||
const missing = transitionShellFollower(fresh(true), { type: "read.missing", attempt: 1 })
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { TextBoundary } from "@opencode/util/text-boundary"
|
||||
|
||||
// Same page size as the TUI shell output viewer; only this much recent output is retained and rendered.
|
||||
export const SHELL_OUTPUT_TAIL_BYTES = 64 * 1024
|
||||
|
||||
@@ -144,7 +146,7 @@ export function transitionShellFollower(state: ShellFollowerState, event: ShellF
|
||||
const page = event.page
|
||||
const advanced = page.cursor > state.cursor
|
||||
const cursor = Math.max(state.cursor, page.cursor)
|
||||
const output = (state.output + page.output).slice(-SHELL_OUTPUT_TAIL_BYTES)
|
||||
const output = TextBoundary.tail(state.output + page.output, SHELL_OUTPUT_TAIL_BYTES)
|
||||
const emit = { type: "emit", output } as const
|
||||
const attempt = state.attempt + 1
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import { useTerminalDimensions } from "@opentui/solid"
|
||||
import { isShellNotFoundError, type LocationRef, type ShellInfo } from "@opencode/client"
|
||||
import { createEffect, createMemo, createSignal, onCleanup, Show, untrack } from "solid-js"
|
||||
import stripAnsi from "strip-ansi"
|
||||
import { TextBoundary } from "@opencode/util/text-boundary"
|
||||
import { useNewDesigns } from "../config"
|
||||
import { useClient } from "../context/client"
|
||||
import { useClipboard } from "../context/clipboard"
|
||||
@@ -61,7 +62,7 @@ export function DialogShellOutput(props: { shell: ShellInfo; location: LocationR
|
||||
setOutput((previous) => {
|
||||
const next = (previous ?? "") + page.data.output
|
||||
if (next.length > PAGE_BYTES) setOmitted(true)
|
||||
return next.slice(-PAGE_BYTES)
|
||||
return TextBoundary.tail(next, PAGE_BYTES)
|
||||
})
|
||||
setError("")
|
||||
return cursor < page.data.size
|
||||
|
||||
@@ -120,6 +120,7 @@ import { stringWidth } from "../../util/string-width"
|
||||
import { useArgs } from "../../context/args"
|
||||
import { withTimestampedFallback } from "@opencode/util/session-title-fallback"
|
||||
import { SessionTurn } from "@opencode/util/session-turn"
|
||||
import { TextBoundary } from "@opencode/util/text-boundary"
|
||||
import { useSessionTabs, type ScrollAnchor } from "../../context/session-tabs"
|
||||
import { createSingleFlight } from "../../util/single-flight"
|
||||
import { createDelayedPresence } from "../../util/delayed-presence"
|
||||
@@ -3069,7 +3070,7 @@ function ShellDisplay(props: {
|
||||
const next = stripAnsi(output + response.data.output)
|
||||
if (next.length <= SHELL_DISPLAY_LIMIT) return next
|
||||
setOutputTruncated(true)
|
||||
return next.slice(-SHELL_DISPLAY_LIMIT)
|
||||
return TextBoundary.tail(next, SHELL_DISPLAY_LIMIT)
|
||||
})
|
||||
if (response.data.cursor <= cursor) break
|
||||
cursor = response.data.cursor
|
||||
|
||||
@@ -35,6 +35,7 @@ async function setup(width: number, output = "", newDesigns = false) {
|
||||
}
|
||||
const state = { output, missing: false, failure: false }
|
||||
const requests: { url: URL; method: string }[] = []
|
||||
const copied: string[] = []
|
||||
const events = createEventStream()
|
||||
const envelope = (data: unknown) => json({ location, data })
|
||||
const api = createApi(
|
||||
@@ -67,7 +68,11 @@ async function setup(width: number, output = "", newDesigns = false) {
|
||||
|
||||
const app = await testRender(
|
||||
() => (
|
||||
<TestTuiContexts directory={temporary.path} paths={{ state: temporary.path }}>
|
||||
<TestTuiContexts
|
||||
directory={temporary.path}
|
||||
paths={{ state: temporary.path }}
|
||||
clipboard={{ read: async () => undefined, write: async (text) => void copied.push(text) }}
|
||||
>
|
||||
<TuiAppProvider value={{ name: "test", version: "test", channel: "test" }}>
|
||||
<StorageProvider>
|
||||
<ConfigProvider config={createTuiResolvedConfig({ experimental: { "new-designs": newDesigns } })}>
|
||||
@@ -101,6 +106,7 @@ async function setup(width: number, output = "", newDesigns = false) {
|
||||
shell,
|
||||
location,
|
||||
requests,
|
||||
copied,
|
||||
events,
|
||||
async [Symbol.asyncDispose]() {
|
||||
app.renderer.destroy()
|
||||
@@ -234,3 +240,16 @@ test("new designs show footer actions instead of the legacy hint line", async ()
|
||||
expect(frame).toContain("copy output o")
|
||||
expect(frame).not.toContain("back esc")
|
||||
})
|
||||
|
||||
test("the retained 64 KiB tail starts on a whole character", async () => {
|
||||
await using app = await setup(100, "😀", true)
|
||||
app.mockInput.pressEnter()
|
||||
await app.waitForFrame((frame) => frame.includes("Shell output") && frame.includes("😀"))
|
||||
// The second page brings the text to 64 KiB + 1 code units, so the tail would start on the emoji's second half.
|
||||
const rest = ("x".repeat(1023) + "\n").repeat(63) + "x".repeat(1023)
|
||||
app.state.output += rest
|
||||
await app.waitForFrame((frame) => frame.includes("Earlier output omitted"), { maxPasses: 100 })
|
||||
app.mockInput.pressKey("o")
|
||||
await app.waitFor(() => app.copied.length > 0)
|
||||
expect(app.copied).toEqual([rest])
|
||||
})
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import { TextRenderable, type Renderable } from "@opentui/core"
|
||||
import { createTestRenderer } from "@opentui/core/testing"
|
||||
import { Effect, FileSystem } from "effect"
|
||||
import { Global } from "@opencode/util/global"
|
||||
import type { SessionMessageInfo } from "@opencode/client"
|
||||
import { createEventStream, createFetch, directory, json } from "./fixture/tui-client"
|
||||
import { tmpdir } from "./fixture/fixture"
|
||||
|
||||
const session = {
|
||||
id: "ses_shell_output",
|
||||
title: "Shell output",
|
||||
projectID: "proj_test",
|
||||
location: { directory },
|
||||
cost: 0,
|
||||
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
time: { created: 0, updated: 0 },
|
||||
}
|
||||
const messages = [
|
||||
{ type: "user", id: "user-1", text: "Run it", time: { created: 0 } },
|
||||
{
|
||||
type: "assistant",
|
||||
id: "assistant-1",
|
||||
agent: "build",
|
||||
model: { providerID: "fixture", id: "fixture" },
|
||||
finish: "stop",
|
||||
time: { created: 1, completed: 3 },
|
||||
content: [
|
||||
{
|
||||
type: "tool",
|
||||
id: "call-background",
|
||||
name: "shell",
|
||||
time: { created: 1, completed: 2 },
|
||||
state: {
|
||||
input: { command: "render-frames" },
|
||||
status: "completed",
|
||||
content: [],
|
||||
metadata: { shellID: "sh_background", status: "running" },
|
||||
},
|
||||
},
|
||||
{ type: "text", text: "Started it" },
|
||||
],
|
||||
},
|
||||
] as SessionMessageInfo[]
|
||||
|
||||
test("an expanded background shell keeps its 1 MiB tail on a whole character", async () => {
|
||||
await using state = await tmpdir()
|
||||
// Two 1 MiB byte pages decode to 1 MiB + 1 code units, so the retained tail would start on the emoji's second half.
|
||||
const rest = ("x".repeat(1023) + "\n").repeat(1023) + "x".repeat(1023)
|
||||
const bytes = Buffer.from(`😀${rest}`)
|
||||
const setup = await createTestRenderer({ width: 100, height: 40, kittyKeyboard: true })
|
||||
await setup.renderer.setupTerminal()
|
||||
setup.renderer.start()
|
||||
const calls = createFetch((url) => {
|
||||
if (url.pathname === "/api/session") return json({ data: [session], cursor: {} })
|
||||
if (url.pathname === `/api/session/${session.id}`) return json({ data: session })
|
||||
if (url.pathname === `/api/session/${session.id}/message`) return json({ data: messages.toReversed(), cursor: {} })
|
||||
if (url.pathname === `/api/session/${session.id}/inbox` || url.pathname === `/api/session/${session.id}/permission`)
|
||||
return json({ data: [] })
|
||||
if (url.pathname === "/api/shell/sh_background/output") {
|
||||
const cursor = Number(url.searchParams.get("cursor"))
|
||||
const end = Math.min(bytes.length, cursor + Number(url.searchParams.get("limit")))
|
||||
return json({ data: { output: bytes.subarray(cursor, end).toString(), cursor: end, size: bytes.length } })
|
||||
}
|
||||
}, createEventStream())
|
||||
const server = Bun.serve({ port: 0, idleTimeout: 0, fetch: (request) => calls.fetch(request) })
|
||||
const { run } = await import("../src/app")
|
||||
const task = Effect.runPromise(
|
||||
run({
|
||||
app: { name: "test", version: "test", channel: "test" },
|
||||
server: { endpoint: { url: server.url.toString() } },
|
||||
config: {
|
||||
get: async () => ({ animations: false, tabs: { enabled: false }, session: { verbosity: "high" } }),
|
||||
update: async () => ({}),
|
||||
},
|
||||
packages: { prepare: async () => ({ directory: "" }) },
|
||||
args: { sessionID: session.id },
|
||||
terminalHandoff: async () => ({ renderer: setup.renderer, mode: "dark", complete: () => {} }),
|
||||
log: () => {},
|
||||
}).pipe(Effect.provide(Global.layerWith({ state: state.path })), Effect.provide(FileSystem.layerNoop({}))),
|
||||
)
|
||||
try {
|
||||
await setup.waitForFrame((frame) => frame.includes("Started it"))
|
||||
const row = setup
|
||||
.captureCharFrame()
|
||||
.split("\n")
|
||||
.findIndex((line) => line.includes("$ render-frames"))
|
||||
await setup.mockMouse.click(10, row)
|
||||
const output = await waitForText(setup.renderer.root, (text) => text.startsWith("[earlier output omitted]"))
|
||||
expect(output).toBe(`[earlier output omitted]\n${rest}`)
|
||||
} finally {
|
||||
setup.renderer.destroy()
|
||||
await task
|
||||
await server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
async function waitForText(root: Renderable, matches: (text: string) => boolean) {
|
||||
const deadline = Date.now() + 5_000
|
||||
while (Date.now() < deadline) {
|
||||
const found = texts(root).find(matches)
|
||||
if (found !== undefined) return found
|
||||
await Bun.sleep(10)
|
||||
}
|
||||
throw new Error("Timed out waiting for shell output")
|
||||
}
|
||||
|
||||
function texts(node: Renderable): string[] {
|
||||
return [
|
||||
...(node instanceof TextRenderable ? [node.plainText] : []),
|
||||
...node.getChildren().flatMap((child) => texts(child as Renderable)),
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { TextBoundary } from "./text-boundary.js"
|
||||
|
||||
const encoder = new TextEncoder()
|
||||
const samples = ["", "ascii", "ab😀cd", "😀é€\uFFFD😀"]
|
||||
|
||||
describe("TextBoundary", () => {
|
||||
test.each(samples)("every head and tail of %j keeps whole characters within the limit", (text) => {
|
||||
for (let limit = 0; limit <= text.length + 1; limit++) {
|
||||
const head = TextBoundary.head(text, limit)
|
||||
const tail = TextBoundary.tail(text, limit)
|
||||
expect(text.startsWith(head)).toBe(true)
|
||||
expect(text.endsWith(tail)).toBe(true)
|
||||
for (const part of [head, tail]) {
|
||||
expect(part.isWellFormed()).toBe(true)
|
||||
expect(part.length).toBeLessThanOrEqual(limit)
|
||||
expect(part.length).toBeGreaterThanOrEqual(Math.min(limit, text.length) - 1)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test.each(samples)("every UTF-8 tail of %j keeps whole characters within the limit", (text) => {
|
||||
const bytes = encoder.encode(text)
|
||||
for (let limit = 0; limit <= bytes.length + 1; limit++) {
|
||||
const tail = TextBoundary.utf8Tail(bytes, limit)
|
||||
expect(text.endsWith(tail)).toBe(true)
|
||||
expect(encoder.encode(tail).length).toBeLessThanOrEqual(limit)
|
||||
expect(encoder.encode(tail).length).toBeGreaterThanOrEqual(Math.min(limit, bytes.length) - 3)
|
||||
}
|
||||
})
|
||||
|
||||
test("keeps malformed input that is not at the cut", () => {
|
||||
expect(TextBoundary.head("a\ud83d", 2)).toBe("a\ud83d")
|
||||
expect(TextBoundary.head("a\ud83db", 2)).toBe("a")
|
||||
expect(TextBoundary.tail("\ude00a", 2)).toBe("\ude00a")
|
||||
expect(TextBoundary.tail("b\ude00a", 2)).toBe("a")
|
||||
expect(TextBoundary.utf8Tail(Uint8Array.of(0x80, 0x41), 2)).toBe("\uFFFDA")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,36 @@
|
||||
export * as TextBoundary from "./text-boundary.js"
|
||||
|
||||
/**
|
||||
* Clip text to a budget without ending or starting on half a character. `head` and `tail` count UTF-16 code units,
|
||||
* like `.length`; `utf8Tail` counts bytes of UTF-8 output that has not been decoded yet. The result can be shorter
|
||||
* than the budget by the part of the character at the cut, and `head("😀", 1)` is empty. Boundaries are code points,
|
||||
* not graphemes, and malformed input is kept as it is everywhere except at the cut.
|
||||
*/
|
||||
|
||||
/** The first `limit` code units of `text`, without a high surrogate whose pair the limit cut off. */
|
||||
export function head(text: string, limit: number) {
|
||||
if (text.length <= limit) return text
|
||||
const code = text.charCodeAt(limit - 1)
|
||||
return text.slice(0, code >= 0xd800 && code <= 0xdbff ? limit - 1 : limit)
|
||||
}
|
||||
|
||||
/** The last `limit` code units of `text`, without a low surrogate whose pair the limit cut off. */
|
||||
export function tail(text: string, limit: number) {
|
||||
if (text.length <= limit) return text
|
||||
const start = text.length - limit
|
||||
const code = text.charCodeAt(start)
|
||||
return text.slice(code >= 0xdc00 && code <= 0xdfff ? start + 1 : start)
|
||||
}
|
||||
|
||||
/** The last `limit` bytes of `bytes`, decoded, after skipping up to 3 continuation bytes of a cut character. */
|
||||
export function utf8Tail(bytes: Uint8Array, limit: number) {
|
||||
if (bytes.length <= limit) return decoder.decode(bytes)
|
||||
const start = bytes.length - limit
|
||||
const end = Math.min(bytes.length, start + 3)
|
||||
let index = start
|
||||
while (index < end && (bytes[index] & 0xc0) === 0x80) index++
|
||||
return decoder.decode(bytes.subarray(index))
|
||||
}
|
||||
|
||||
// Like Buffer#toString, keep a leading byte order mark.
|
||||
const decoder = new TextDecoder("utf-8", { ignoreBOM: true })
|
||||
Reference in New Issue
Block a user