diff --git a/packages/core/src/ripgrep.ts b/packages/core/src/ripgrep.ts index 81f0368a6c2..2055f7f6685 100644 --- a/packages/core/src/ripgrep.ts +++ b/packages/core/src/ripgrep.ts @@ -5,6 +5,7 @@ import { ChildProcess } from "effect/process" import { Entry, Match } from "@opencode/schema/filesystem" import { makeLocationNode } from "@opencode/util/effect/app-node" import { collectStream, waitForAbort } from "@opencode/util/process" +import { TextBoundary } from "@opencode/util/text-boundary" import { Environment } from "./environment/index.js" import { NonNegativeInt, PositiveInt, RelativePath } from "./schema.js" import { RipgrepBinary } from "./ripgrep/binary.js" @@ -252,7 +253,7 @@ const layer = Layer.effect( lines: { text: match.data.lines.text.length > MAX_LINE_CHARS - ? Buffer.from(match.data.lines.text.slice(0, MAX_LINE_CHARS), "utf8").toString() + "..." + ? TextBoundary.head(match.data.lines.text, MAX_LINE_CHARS) + "..." : match.data.lines.text, }, path: { text: normalizePath(match.data.path.text) }, diff --git a/packages/core/src/session/title.ts b/packages/core/src/session/title.ts index c5534dd8b64..d69dc95ffab 100644 --- a/packages/core/src/session/title.ts +++ b/packages/core/src/session/title.ts @@ -8,6 +8,7 @@ import { Database } from "../database/database.js" import { Bus } from "../bus.js" import { makeLocationNode } from "@opencode/util/effect/app-node" import { isExactRootFallback } from "@opencode/util/session-title-fallback" +import { TextBoundary } from "@opencode/util/text-boundary" import { llmClient } from "../effect/app-node-platform.js" import { SessionContext } from "./context.js" import { SessionEvent } from "./event.js" @@ -29,7 +30,8 @@ export interface Interface { export class Service extends Context.Service()("@opencode/SessionTitle") {} -const truncate = (value: string) => (value.length <= MAX_LENGTH ? value : `${value.slice(0, MAX_LENGTH - 3)}...`) +const truncate = (value: string) => + value.length <= MAX_LENGTH ? value : `${TextBoundary.head(value, MAX_LENGTH - 3)}...` // Framing the input as text to name keeps models from answering or carrying out the request instead. const messagePrompt = (text: string) => `Here is the first message of the conversation:\n\n\n${text}\n\n\nReply with only the title.` @@ -119,7 +121,7 @@ export const layer = Layer.effect( const text = !isUntitled(session) ? yield* store.context(session.id).pipe( Effect.map((messages) => { - const original = `Original request:\n${firstUser.text.slice(0, MAX_FIRST_MESSAGE_LENGTH)}` + const original = `Original request:\n${TextBoundary.head(firstUser.text, MAX_FIRST_MESSAGE_LENGTH)}` const recent = messages .flatMap((message) => { if (message.type === "user" && message.id !== firstUser.id) return [`User: ${message.text.trim()}`] @@ -134,7 +136,7 @@ export const layer = Layer.effect( if (!recent) return conversationPrompt(original) const prefix = `${original}\n\nRecent conversation:\n` const budget = MAX_CONTEXT_LENGTH - conversationPrompt("").length - prefix.length - return conversationPrompt(`${prefix}${recent.slice(-budget)}`) + return conversationPrompt(`${prefix}${TextBoundary.tail(recent, budget)}`) }), Effect.orElseSucceed(() => messagePrompt(firstUser.text)), ) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 46d5d516736..fec67d1a372 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -22,6 +22,7 @@ import { AppProcess } from "@opencode/util/process" import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner" import { makeGlobalNode, makeLocationNode } from "@opencode/util/effect/app-node" import { FSUtil } from "@opencode/util/fs-util" +import { TextBoundary } from "@opencode/util/text-boundary" import { Bus } from "./bus.js" import { Environment } from "./environment/index.js" import { FileRetention } from "./file-retention.js" @@ -825,7 +826,7 @@ const layer = () => // Read the byte before the window too, to tell whether the window starts on a line boundary. const before = Math.max(0, start - 1) const window = yield* readRange(info.file, before, size - before) - const page = window.subarray(charBoundary(window, start - before)).toString("utf8") + const page = TextBoundary.utf8Tail(window, size - start) const partial = start > 0 && window[0] !== 10 const endsWithNewline = page.endsWith("\n") const lines = page.split("\n") @@ -1142,13 +1143,6 @@ function spanCommand(command: string) { return `${command.slice(0, SPAN_COMMAND_LIMIT).toWellFormed()}…` } -// Moves past UTF-8 continuation bytes (at most 3) so a byte cut never splits a character. -function charBoundary(bytes: Buffer, offset: number) { - const limit = Math.min(bytes.length, offset + 3) - for (let index = offset; index < limit; index++) if ((bytes[index] & 0xc0) !== 0x80) return index - return limit -} - function formatSize(bytes: number) { const kb = bytes / 1024 return kb > 1024 ? `${(kb / 1024).toFixed(1)} MB` : `${Math.round(kb)} KB` diff --git a/packages/core/src/tool/read-filesystem.ts b/packages/core/src/tool/read-filesystem.ts index 5f1a91dd3d2..d552b155781 100644 --- a/packages/core/src/tool/read-filesystem.ts +++ b/packages/core/src/tool/read-filesystem.ts @@ -3,6 +3,7 @@ export * as ReadToolFileSystem from "./read-filesystem.js" import path from "path" import { pathToFileURL } from "url" import { makeLocationNode } from "@opencode/util/effect/app-node" +import { TextBoundary } from "@opencode/util/text-boundary" import { Context, Effect, Layer, Schema } from "effect" import { lookup } from "mime-types" import { Environment } from "../environment/index.js" @@ -374,12 +375,8 @@ const list = (items: ReadonlyArray, page: PageInput, sep = }) } -const clampLine = (line: string) => { - if (line.length <= MAX_LINE_LENGTH) return line - const code = line.charCodeAt(MAX_LINE_LENGTH - 1) - const end = code >= 0xd800 && code <= 0xdbff ? MAX_LINE_LENGTH - 1 : MAX_LINE_LENGTH - return line.slice(0, end) + MAX_LINE_SUFFIX -} +const clampLine = (line: string) => + line.length <= MAX_LINE_LENGTH ? line : TextBoundary.head(line, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX const layer = Layer.effect( Service, diff --git a/packages/core/test/ripgrep.test.ts b/packages/core/test/ripgrep.test.ts index 4f84d779051..e0ebd2cc80e 100644 --- a/packages/core/test/ripgrep.test.ts +++ b/packages/core/test/ripgrep.test.ts @@ -184,6 +184,28 @@ describe("Ripgrep", () => { }), ), ) + + it.live("ends oversized line previews on a whole character", () => + withTempDir((tmp) => + Effect.gen(function* () { + yield* Effect.promise(() => + fs.writeFile( + path.join(tmp.path, "emoji.txt"), + [1_998, 1_999, 2_000].map((count) => `${"x".repeat(count)}πŸ˜€tail\n`).join(""), + ), + ) + + const ripgrep = yield* Ripgrep.Service + const matches = yield* ripgrep.grep({ cwd: tmp.path, pattern: "tail", limit: 10 }) + + expect(matches.map((match) => match.text)).toEqual([ + `${"x".repeat(1_998)}πŸ˜€...`, + `${"x".repeat(1_999)}...`, + `${"x".repeat(2_000)}...`, + ]) + }), + ), + ) }) describe("RipgrepBinary.verify", () => { diff --git a/packages/core/test/session-title.test.ts b/packages/core/test/session-title.test.ts index 5143eb718c3..24380c57183 100644 --- a/packages/core/test/session-title.test.ts +++ b/packages/core/test/session-title.test.ts @@ -634,6 +634,52 @@ it.effect("bounds regeneration context while preserving the original request and }), ) +it.effect("keeps the clipped regeneration context well formed", () => + Effect.gen(function* () { + yield* enableTitleAgent + const sessionID = Session.ID.make("ses_title_regenerate_emoji") + yield* insertSession(sessionID, "Original title") + // The leading x puts both cuts, the original request's head and the recent conversation's tail, inside an emoji. + yield* prompt(sessionID, `x${"πŸ˜€".repeat(1_500)}`) + yield* prompt(sessionID, `x${"πŸ˜€".repeat(5_000)}`) + + const title = yield* SessionTitle.Service + yield* title.generate(sessionID) + + const content = requests[0]?.messages[0]?.content[0] + expect(content?.type).toBe("text") + if (content?.type !== "text") return + expect(content.text.length).toBeLessThanOrEqual(8_000) + expect(content.text.length).toBeGreaterThan(7_900) + expect(content.text.isWellFormed()).toBe(true) + }), +) + +for (const [prefix, stored] of [ + [95, `${"a".repeat(95)}πŸ˜€...`], + [96, `${"a".repeat(96)}...`], + [97, `${"a".repeat(97)}...`], +] as const) + it.effect(`clips a long generated title without splitting an emoji after ${prefix} letters`, () => + Effect.gen(function* () { + yield* enableTitleAgent + titleStream = () => + Stream.make( + LLMEvent.textDelta({ id: "title", text: `${"a".repeat(prefix)}πŸ˜€${"b".repeat(20)}` }), + LLMEvent.finish({ reason: { normalized: "stop" } }), + ) + const sessionID = Session.ID.make(`ses_title_emoji_${prefix}`) + yield* insertSession(sessionID) + yield* prompt(sessionID, "Name this session") + + const title = yield* SessionTitle.Service + yield* title.generate(sessionID) + + const store = yield* SessionStore.Service + expect((yield* store.get(sessionID))?.title).toBe(stored) + }), + ) + it.effect("preserves the existing title when regeneration fails", () => Effect.gen(function* () { yield* enableTitleAgent diff --git a/packages/session-ui/src/tools/shell-output-machine.test.ts b/packages/session-ui/src/tools/shell-output-machine.test.ts index fac0e19d208..dd52a566a35 100644 --- a/packages/session-ui/src/tools/shell-output-machine.test.ts +++ b/packages/session-ui/src/tools/shell-output-machine.test.ts @@ -173,6 +173,31 @@ describe("transitionShellFollower", () => { expect(stopped.effects).toEqual([{ type: "read", attempt: 3, cursor: 1 }]) }) + test("keeps the retained tail on a whole character when the window cuts an emoji", () => { + // Each page is valid text; only the rolling 65,536-unit window would start on the second half of the emoji. + const first = fresh(false) + + if (first.status !== "reading") throw new Error(`expected a read, got ${first.status}`) + + const next = transitionShellFollower(first, { + type: "read.succeeded", + attempt: first.attempt, + page: { output: "πŸ˜€", cursor: 4, size: 65_539 }, + }).state + + if (next.status !== "reading") throw new Error(`expected a read, got ${next.status}`) + + const done = transitionShellFollower(next, { + type: "read.succeeded", + attempt: next.attempt, + page: { output: "x".repeat(65_535), cursor: 65_539, size: 65_539 }, + }) + + expect(done.state).toMatchObject({ status: "complete", output: "x".repeat(65_535) }) + expect(done.effects).toContainEqual({ type: "emit", output: "x".repeat(65_535) }) + expect(done.effects).toContainEqual({ type: "cache", cursor: 65_539, output: "x".repeat(65_535), complete: true }) + }) + test("a missing shell is remembered and never read again", () => { const missing = transitionShellFollower(fresh(true), { type: "read.missing", attempt: 1 }) diff --git a/packages/session-ui/src/tools/shell-output-machine.ts b/packages/session-ui/src/tools/shell-output-machine.ts index d8e2b961509..fd1a3920004 100644 --- a/packages/session-ui/src/tools/shell-output-machine.ts +++ b/packages/session-ui/src/tools/shell-output-machine.ts @@ -1,3 +1,5 @@ +import { TextBoundary } from "@opencode/util/text-boundary" + // Same page size as the TUI shell output viewer; only this much recent output is retained and rendered. export const SHELL_OUTPUT_TAIL_BYTES = 64 * 1024 @@ -144,7 +146,7 @@ export function transitionShellFollower(state: ShellFollowerState, event: ShellF const page = event.page const advanced = page.cursor > state.cursor const cursor = Math.max(state.cursor, page.cursor) - const output = (state.output + page.output).slice(-SHELL_OUTPUT_TAIL_BYTES) + const output = TextBoundary.tail(state.output + page.output, SHELL_OUTPUT_TAIL_BYTES) const emit = { type: "emit", output } as const const attempt = state.attempt + 1 diff --git a/packages/tui/src/component/dialog-shell-output.tsx b/packages/tui/src/component/dialog-shell-output.tsx index a7ca8dd8100..3c5c79609bd 100644 --- a/packages/tui/src/component/dialog-shell-output.tsx +++ b/packages/tui/src/component/dialog-shell-output.tsx @@ -3,6 +3,7 @@ import { useTerminalDimensions } from "@opentui/solid" import { isShellNotFoundError, type LocationRef, type ShellInfo } from "@opencode/client" import { createEffect, createMemo, createSignal, onCleanup, Show, untrack } from "solid-js" import stripAnsi from "strip-ansi" +import { TextBoundary } from "@opencode/util/text-boundary" import { useNewDesigns } from "../config" import { useClient } from "../context/client" import { useClipboard } from "../context/clipboard" @@ -61,7 +62,7 @@ export function DialogShellOutput(props: { shell: ShellInfo; location: LocationR setOutput((previous) => { const next = (previous ?? "") + page.data.output if (next.length > PAGE_BYTES) setOmitted(true) - return next.slice(-PAGE_BYTES) + return TextBoundary.tail(next, PAGE_BYTES) }) setError("") return cursor < page.data.size diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index 1fcacd17048..8f04fb09387 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -120,6 +120,7 @@ import { stringWidth } from "../../util/string-width" import { useArgs } from "../../context/args" import { withTimestampedFallback } from "@opencode/util/session-title-fallback" import { SessionTurn } from "@opencode/util/session-turn" +import { TextBoundary } from "@opencode/util/text-boundary" import { useSessionTabs, type ScrollAnchor } from "../../context/session-tabs" import { createSingleFlight } from "../../util/single-flight" import { createDelayedPresence } from "../../util/delayed-presence" @@ -3069,7 +3070,7 @@ function ShellDisplay(props: { const next = stripAnsi(output + response.data.output) if (next.length <= SHELL_DISPLAY_LIMIT) return next setOutputTruncated(true) - return next.slice(-SHELL_DISPLAY_LIMIT) + return TextBoundary.tail(next, SHELL_DISPLAY_LIMIT) }) if (response.data.cursor <= cursor) break cursor = response.data.cursor diff --git a/packages/tui/test/component/dialog-shell-output.test.tsx b/packages/tui/test/component/dialog-shell-output.test.tsx index 869f274c98b..ded26552d07 100644 --- a/packages/tui/test/component/dialog-shell-output.test.tsx +++ b/packages/tui/test/component/dialog-shell-output.test.tsx @@ -35,6 +35,7 @@ async function setup(width: number, output = "", newDesigns = false) { } const state = { output, missing: false, failure: false } const requests: { url: URL; method: string }[] = [] + const copied: string[] = [] const events = createEventStream() const envelope = (data: unknown) => json({ location, data }) const api = createApi( @@ -67,7 +68,11 @@ async function setup(width: number, output = "", newDesigns = false) { const app = await testRender( () => ( - + undefined, write: async (text) => void copied.push(text) }} + > @@ -101,6 +106,7 @@ async function setup(width: number, output = "", newDesigns = false) { shell, location, requests, + copied, events, async [Symbol.asyncDispose]() { app.renderer.destroy() @@ -234,3 +240,16 @@ test("new designs show footer actions instead of the legacy hint line", async () expect(frame).toContain("copy output o") expect(frame).not.toContain("back esc") }) + +test("the retained 64 KiB tail starts on a whole character", async () => { + await using app = await setup(100, "πŸ˜€", true) + app.mockInput.pressEnter() + await app.waitForFrame((frame) => frame.includes("Shell output") && frame.includes("πŸ˜€")) + // The second page brings the text to 64 KiB + 1 code units, so the tail would start on the emoji's second half. + const rest = ("x".repeat(1023) + "\n").repeat(63) + "x".repeat(1023) + app.state.output += rest + await app.waitForFrame((frame) => frame.includes("Earlier output omitted"), { maxPasses: 100 }) + app.mockInput.pressKey("o") + await app.waitFor(() => app.copied.length > 0) + expect(app.copied).toEqual([rest]) +}) diff --git a/packages/tui/test/session-shell-output.test.tsx b/packages/tui/test/session-shell-output.test.tsx new file mode 100644 index 00000000000..6787043a8a3 --- /dev/null +++ b/packages/tui/test/session-shell-output.test.tsx @@ -0,0 +1,113 @@ +import { expect, test } from "bun:test" +import { TextRenderable, type Renderable } from "@opentui/core" +import { createTestRenderer } from "@opentui/core/testing" +import { Effect, FileSystem } from "effect" +import { Global } from "@opencode/util/global" +import type { SessionMessageInfo } from "@opencode/client" +import { createEventStream, createFetch, directory, json } from "./fixture/tui-client" +import { tmpdir } from "./fixture/fixture" + +const session = { + id: "ses_shell_output", + title: "Shell output", + projectID: "proj_test", + location: { directory }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 0, updated: 0 }, +} +const messages = [ + { type: "user", id: "user-1", text: "Run it", time: { created: 0 } }, + { + type: "assistant", + id: "assistant-1", + agent: "build", + model: { providerID: "fixture", id: "fixture" }, + finish: "stop", + time: { created: 1, completed: 3 }, + content: [ + { + type: "tool", + id: "call-background", + name: "shell", + time: { created: 1, completed: 2 }, + state: { + input: { command: "render-frames" }, + status: "completed", + content: [], + metadata: { shellID: "sh_background", status: "running" }, + }, + }, + { type: "text", text: "Started it" }, + ], + }, +] as SessionMessageInfo[] + +test("an expanded background shell keeps its 1 MiB tail on a whole character", async () => { + await using state = await tmpdir() + // Two 1 MiB byte pages decode to 1 MiB + 1 code units, so the retained tail would start on the emoji's second half. + const rest = ("x".repeat(1023) + "\n").repeat(1023) + "x".repeat(1023) + const bytes = Buffer.from(`πŸ˜€${rest}`) + const setup = await createTestRenderer({ width: 100, height: 40, kittyKeyboard: true }) + await setup.renderer.setupTerminal() + setup.renderer.start() + const calls = createFetch((url) => { + if (url.pathname === "/api/session") return json({ data: [session], cursor: {} }) + if (url.pathname === `/api/session/${session.id}`) return json({ data: session }) + if (url.pathname === `/api/session/${session.id}/message`) return json({ data: messages.toReversed(), cursor: {} }) + if (url.pathname === `/api/session/${session.id}/inbox` || url.pathname === `/api/session/${session.id}/permission`) + return json({ data: [] }) + if (url.pathname === "/api/shell/sh_background/output") { + const cursor = Number(url.searchParams.get("cursor")) + const end = Math.min(bytes.length, cursor + Number(url.searchParams.get("limit"))) + return json({ data: { output: bytes.subarray(cursor, end).toString(), cursor: end, size: bytes.length } }) + } + }, createEventStream()) + const server = Bun.serve({ port: 0, idleTimeout: 0, fetch: (request) => calls.fetch(request) }) + const { run } = await import("../src/app") + const task = Effect.runPromise( + run({ + app: { name: "test", version: "test", channel: "test" }, + server: { endpoint: { url: server.url.toString() } }, + config: { + get: async () => ({ animations: false, tabs: { enabled: false }, session: { verbosity: "high" } }), + update: async () => ({}), + }, + packages: { prepare: async () => ({ directory: "" }) }, + args: { sessionID: session.id }, + terminalHandoff: async () => ({ renderer: setup.renderer, mode: "dark", complete: () => {} }), + log: () => {}, + }).pipe(Effect.provide(Global.layerWith({ state: state.path })), Effect.provide(FileSystem.layerNoop({}))), + ) + try { + await setup.waitForFrame((frame) => frame.includes("Started it")) + const row = setup + .captureCharFrame() + .split("\n") + .findIndex((line) => line.includes("$ render-frames")) + await setup.mockMouse.click(10, row) + const output = await waitForText(setup.renderer.root, (text) => text.startsWith("[earlier output omitted]")) + expect(output).toBe(`[earlier output omitted]\n${rest}`) + } finally { + setup.renderer.destroy() + await task + await server.stop() + } +}) + +async function waitForText(root: Renderable, matches: (text: string) => boolean) { + const deadline = Date.now() + 5_000 + while (Date.now() < deadline) { + const found = texts(root).find(matches) + if (found !== undefined) return found + await Bun.sleep(10) + } + throw new Error("Timed out waiting for shell output") +} + +function texts(node: Renderable): string[] { + return [ + ...(node instanceof TextRenderable ? [node.plainText] : []), + ...node.getChildren().flatMap((child) => texts(child as Renderable)), + ] +} diff --git a/packages/util/src/text-boundary.test.ts b/packages/util/src/text-boundary.test.ts new file mode 100644 index 00000000000..2288cb7ccb4 --- /dev/null +++ b/packages/util/src/text-boundary.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, test } from "bun:test" +import { TextBoundary } from "./text-boundary.js" + +const encoder = new TextEncoder() +const samples = ["", "ascii", "abπŸ˜€cd", "πŸ˜€Γ©β‚¬\uFFFDπŸ˜€"] + +describe("TextBoundary", () => { + test.each(samples)("every head and tail of %j keeps whole characters within the limit", (text) => { + for (let limit = 0; limit <= text.length + 1; limit++) { + const head = TextBoundary.head(text, limit) + const tail = TextBoundary.tail(text, limit) + expect(text.startsWith(head)).toBe(true) + expect(text.endsWith(tail)).toBe(true) + for (const part of [head, tail]) { + expect(part.isWellFormed()).toBe(true) + expect(part.length).toBeLessThanOrEqual(limit) + expect(part.length).toBeGreaterThanOrEqual(Math.min(limit, text.length) - 1) + } + } + }) + + test.each(samples)("every UTF-8 tail of %j keeps whole characters within the limit", (text) => { + const bytes = encoder.encode(text) + for (let limit = 0; limit <= bytes.length + 1; limit++) { + const tail = TextBoundary.utf8Tail(bytes, limit) + expect(text.endsWith(tail)).toBe(true) + expect(encoder.encode(tail).length).toBeLessThanOrEqual(limit) + expect(encoder.encode(tail).length).toBeGreaterThanOrEqual(Math.min(limit, bytes.length) - 3) + } + }) + + test("keeps malformed input that is not at the cut", () => { + expect(TextBoundary.head("a\ud83d", 2)).toBe("a\ud83d") + expect(TextBoundary.head("a\ud83db", 2)).toBe("a") + expect(TextBoundary.tail("\ude00a", 2)).toBe("\ude00a") + expect(TextBoundary.tail("b\ude00a", 2)).toBe("a") + expect(TextBoundary.utf8Tail(Uint8Array.of(0x80, 0x41), 2)).toBe("\uFFFDA") + }) +}) diff --git a/packages/util/src/text-boundary.ts b/packages/util/src/text-boundary.ts new file mode 100644 index 00000000000..2633b6a2860 --- /dev/null +++ b/packages/util/src/text-boundary.ts @@ -0,0 +1,36 @@ +export * as TextBoundary from "./text-boundary.js" + +/** + * Clip text to a budget without ending or starting on half a character. `head` and `tail` count UTF-16 code units, + * like `.length`; `utf8Tail` counts bytes of UTF-8 output that has not been decoded yet. The result can be shorter + * than the budget by the part of the character at the cut, and `head("πŸ˜€", 1)` is empty. Boundaries are code points, + * not graphemes, and malformed input is kept as it is everywhere except at the cut. + */ + +/** The first `limit` code units of `text`, without a high surrogate whose pair the limit cut off. */ +export function head(text: string, limit: number) { + if (text.length <= limit) return text + const code = text.charCodeAt(limit - 1) + return text.slice(0, code >= 0xd800 && code <= 0xdbff ? limit - 1 : limit) +} + +/** The last `limit` code units of `text`, without a low surrogate whose pair the limit cut off. */ +export function tail(text: string, limit: number) { + if (text.length <= limit) return text + const start = text.length - limit + const code = text.charCodeAt(start) + return text.slice(code >= 0xdc00 && code <= 0xdfff ? start + 1 : start) +} + +/** The last `limit` bytes of `bytes`, decoded, after skipping up to 3 continuation bytes of a cut character. */ +export function utf8Tail(bytes: Uint8Array, limit: number) { + if (bytes.length <= limit) return decoder.decode(bytes) + const start = bytes.length - limit + const end = Math.min(bytes.length, start + 3) + let index = start + while (index < end && (bytes[index] & 0xc0) === 0x80) index++ + return decoder.decode(bytes.subarray(index)) +} + +// Like Buffer#toString, keep a leading byte order mark. +const decoder = new TextDecoder("utf-8", { ignoreBOM: true })