diff --git a/packages/core/src/tool-output.ts b/packages/core/src/tool-output.ts index f9ec06fe93c..c5551b8eabf 100644 --- a/packages/core/src/tool-output.ts +++ b/packages/core/src/tool-output.ts @@ -75,48 +75,56 @@ const layer = Layer.effect( const truncate = Effect.fnUntraced(function* (result: Result) { if (result.metadata?.truncated !== undefined) return result const content = result.content - const text = content.flatMap((item) => (item.type === "text" ? [item.text] : [])).join("\n") + // A line break joins text parts unless the text before already ends in one. + const text = content.reduce( + (joined, item) => + item.type !== "text" + ? joined + : joined === "" || joined.endsWith("\n") + ? joined + item.text + : `${joined}\n${item.text}`, + "", + ) const limits = state.get() - const lines = text.split("\n") - if (text.endsWith("\n")) lines.pop() - const totalBytes = Buffer.byteLength(text, "utf-8") - if (lines.length <= limits.maxLines && totalBytes <= limits.maxBytes) + const totalLines = countLines(text) + if (totalLines <= limits.maxLines && Buffer.byteLength(text, "utf-8") <= limits.maxBytes) return { ...result, metadata: { ...result.metadata, truncated: false } } - const kept: string[] = [] - let bytes = 0 - for (const line of lines.slice(0, limits.maxLines)) { - const size = Buffer.byteLength(line, "utf-8") + (kept.length > 0 ? 1 : 0) - if (bytes + size > limits.maxBytes) break - kept.push(line) - bytes += size - } + // Cut once, at the line or byte limit, whichever comes first, so a line too long for the space left shows its + // start. The preview never ends on a line break. + const end = Math.min(afterLines(text, limits.maxLines), byteBoundary(text, limits.maxBytes)) + const shown = text.slice(0, text.endsWith("\r\n", end) ? end - 2 : text.endsWith("\n", end) ? end - 1 : end) const file = path.join(directory, fileName()) const failure = yield* fs.ensureDir(directory).pipe( Effect.andThen(fs.writeFileString(file, text)), Effect.as(undefined), Effect.catch((error) => - Effect.logWarning("failed to save full tool output", { file, cause: error }).pipe( - Effect.as(describe(error)), - ), + Effect.logWarning("failed to save full tool output", { file, cause: error }).pipe(Effect.as(describe(error))), ), ) - const shown = kept.length > 0 ? `lines 1-${kept.length}` : "0 lines" - const marker = `[showing ${shown} of ${lines.length}; ${ + // Counted before the final line break is stripped, so a blank last line still counts. + const shownLines = countLines(text.slice(0, end)) + const lastLineCut = + shownLines > 0 && + shown.length < text.length && + !text.startsWith("\n", shown.length) && + !text.startsWith("\r\n", shown.length) + const range = shownLines === 0 ? "0 lines" : `lines 1-${shownLines}` + const marker = `[showing ${range} of ${totalLines}${lastLineCut ? " (last line cut)" : ""}; ${ failure === undefined ? `full output saved to ${file}` : `full output could not be saved: ${failure}` }]` const bounded: Tool.Content[] = [] - let remaining = kept.join("\n").length - let seenText = false + let remaining = shown.length + let separator = false let marked = false for (const item of content) { if (item.type === "file") { bounded.push(item) continue } - if (seenText && remaining > 0) remaining-- - seenText = true - if (remaining >= item.text.length) { + if (separator && remaining > 0) remaining-- + separator = item.text !== "" && !item.text.endsWith("\n") + if (!marked && remaining >= item.text.length) { bounded.push(item) remaining -= item.text.length continue @@ -172,3 +180,41 @@ export const node = makeLocationNode({ layer, deps: [FSUtil.node, Global.node, cleanupNode], }) + +/** How many lines `text` has, not counting an empty one after a final line break. Counts without splitting. */ +function countLines(text: string) { + if (text === "") return 0 + let count = 1 + for (let index = text.indexOf("\n"); index !== -1 && index < text.length - 1; index = text.indexOf("\n", index + 1)) + count++ + return count +} + +/** Where the line after the first `count` lines starts, or the end of `text`. */ +function afterLines(text: string, count: number) { + let index = 0 + for (let line = 0; line < count; line++) { + const next = text.indexOf("\n", index) + if (next === -1) return text.length + index = next + 1 + } + return index +} + +const encoder = new TextEncoder() +const graphemes = new Intl.Segmenter(undefined, { granularity: "grapheme" }) + +/** The longest start of `text` that fits in `budget` UTF-8 bytes without splitting a character, emoji included. */ +function byteBoundary(text: string, budget: number) { + // encodeInto only encodes what fits in the buffer, and stops at a whole code point. + const index = encoder.encodeInto(text, new Uint8Array(Math.max(0, budget))).read + if (index === text.length || index === 0) return index + const before = text.charCodeAt(index - 1) + const after = text.charCodeAt(index) + // Two ASCII characters always have a boundary between them, except the two halves of \r\n. + if (before < 0x80 && after < 0x80) return before === 13 && after === 10 ? index - 1 : index + // Otherwise back off to the start of the grapheme the cut is in. A line break always ends one, so segmenting from the + // start of the line keeps flag pairs and long emoji sequences exact; one code point past the cut gives the context. + const start = text.lastIndexOf("\n", index - 1) + 1 + return start + (graphemes.segment(text.slice(start, index + 2)).containing(index - start)?.index ?? index - start) +} diff --git a/packages/core/test/tool-output.test.ts b/packages/core/test/tool-output.test.ts index 439fda7ee70..ba509d6eb6b 100644 --- a/packages/core/test/tool-output.test.ts +++ b/packages/core/test/tool-output.test.ts @@ -59,7 +59,9 @@ describe("ToolOutput", () => { { type: "text", text: "one\ntwo" }, { type: "text", - text: expect.stringMatching(/^\[showing lines 1-2 of 3; full output could not be saved: E[A-Z]+: [^\n]+\]$/), + text: expect.stringMatching( + /^\[showing lines 1-2 of 3; full output could not be saved: E[A-Z]+: [^\n]+\]$/, + ), }, ]) }), @@ -67,16 +69,16 @@ describe("ToolOutput", () => { ), ) - it.live("reports lines shown under the byte limit", () => + it.live("cuts at the byte limit, inside a line when it falls there", () => withStore( (output) => Effect.gen(function* () { const result = yield* output.truncate({ content: [{ type: "text", text: "one\ntwo" }] }) expect(result.content).toEqual([ - { type: "text", text: "one" }, + { type: "text", text: "one\nt" }, { type: "text", - text: expect.stringMatching(/^\[showing lines 1-1 of 2; full output saved to .+\]$/), + text: expect.stringMatching(/^\[showing lines 1-2 of 2 \(last line cut\); full output saved to .+\]$/), }, ]) }), @@ -84,6 +86,165 @@ describe("ToolOutput", () => { ), ) + it.live("shows the start of a single line longer than the byte limit", () => + withStore( + (output) => + Effect.gen(function* () { + const text = JSON.stringify({ items: Array.from({ length: 50 }, (_, index) => index) }) + const result = yield* output.truncate({ content: [{ type: "text", text }] }) + expect(result.content).toEqual([ + { type: "text", text: text.slice(0, 20) }, + { + type: "text", + text: expect.stringMatching( + new RegExp("^\\[showing lines 1-1 of 1 \\(last line cut\\); full output saved to .+\\]$"), + ), + }, + ]) + }), + { maxLines: 100, maxBytes: 20 }, + ), + ) + + it.live("never splits a character or an emoji at the cut", () => + withStore( + (output) => + Effect.gen(function* () { + const family = "๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ" + const cases = [ + { text: "abโœ“โœ“โœ“", shown: "abโœ“" }, + { text: `ab${family}cd`, shown: "ab" }, + { text: "ab๐Ÿ‘๐Ÿฝcd", shown: "ab" }, + { text: "ab๐Ÿ‡ฏ๐Ÿ‡ตcd", shown: "ab" }, + { text: "ab๐’ณ๐’ณ", shown: "ab๐’ณ" }, + ] + for (const item of cases) { + const result = yield* output.truncate({ content: [{ type: "text", text: item.text }] }) + const first = result.content[0] + expect(first).toEqual({ type: "text", text: item.shown }) + expect(first?.type === "text" && first.text.isWellFormed()).toBe(true) + } + }), + { maxLines: 100, maxBytes: 7 }, + ), + ) + + it.live("keeps flags and long clusters whole however long the text before them", () => + withStore( + (output) => + Effect.gen(function* () { + const flags = yield* output.truncate({ content: [{ type: "text", text: "a" + "๐Ÿ‡ฏ๐Ÿ‡ต".repeat(40) }] }) + expect(flags.content[0]).toEqual({ type: "text", text: "a" + "๐Ÿ‡ฏ๐Ÿ‡ต".repeat(20) }) + const cluster = "๐Ÿ‘" + "๐Ÿฝ\u0301".repeat(60) + const long = yield* output.truncate({ content: [{ type: "text", text: `ab${cluster}z` }] }) + expect(long.content[0]).toEqual({ type: "text", text: "ab" }) + }), + { maxLines: 100, maxBytes: 165 }, + ), + ) + + it.live("cuts inside a later text part and leaves out the parts after it", () => + withStore( + (output) => + Effect.gen(function* () { + const inside = yield* output.truncate({ + content: [ + { type: "text", text: "" }, + { type: "text", text: "abc" }, + { type: "text", text: "defgh" }, + { type: "text", text: "" }, + ], + }) + expect(inside.content).toEqual([ + { type: "text", text: "" }, + { type: "text", text: "abc" }, + { type: "text", text: "de" }, + { + type: "text", + text: expect.stringMatching(/^\[showing lines 1-2 of 2 \(last line cut\);/), + }, + ]) + const between = yield* output.truncate({ + content: [ + { type: "text", text: "abcde" }, + { type: "text", text: "fgh" }, + ], + }) + expect(between.content).toEqual([ + { type: "text", text: "abcde" }, + { type: "text", text: expect.stringMatching(/^\[showing lines 1-1 of 2;/) }, + ]) + }), + { maxLines: 100, maxBytes: 6 }, + ), + ) + + it.live("ends a line cut before its line break", () => + withStore( + (output) => + Effect.gen(function* () { + const crlf = yield* output.truncate({ content: [{ type: "text", text: "one\r\ntwo\r\nthree" }] }) + expect(crlf.content).toEqual([ + { type: "text", text: "one" }, + { type: "text", text: expect.stringMatching(/^\[showing lines 1-1 of 3;/) }, + ]) + const blank = yield* output.truncate({ content: [{ type: "text", text: "\nfoo\nbar" }] }) + expect(blank.content).toEqual([{ type: "text", text: expect.stringMatching(/^\[showing lines 1-1 of 3;/) }]) + }), + { maxLines: 1, maxBytes: 1_000 }, + ), + ) + + it.live("counts a blank last line of the preview", () => + withStore( + (output) => + Effect.gen(function* () { + const result = yield* output.truncate({ content: [{ type: "text", text: "a\n\n\nb" }] }) + expect(result.content).toEqual([ + { type: "text", text: "a\n" }, + { type: "text", text: expect.stringMatching(/^\[showing lines 1-2 of 4;/) }, + ]) + }), + { maxLines: 2, maxBytes: 1_000 }, + ), + ) + + it.live("shows nothing under a zero line limit", () => + withStore( + (output) => + Effect.gen(function* () { + const result = yield* output.truncate({ content: [{ type: "text", text: "secret" }] }) + expect(result.content).toEqual([{ type: "text", text: expect.stringMatching(/^\[showing 0 lines of 1;/) }]) + }), + { maxLines: 0, maxBytes: 1_000 }, + ), + ) + + it.live("joins text parts with a line break only where one is missing", () => + withStore( + (output, fs) => + Effect.gen(function* () { + const result = yield* output.truncate({ + content: [ + { type: "text", text: "one\n" }, + { type: "text", text: "two" }, + { type: "text", text: "three" }, + ], + }) + const outputPath = result.metadata?.outputPath + expect(typeof outputPath).toBe("string") + if (typeof outputPath !== "string") return + expect(yield* fs.readFileString(outputPath)).toBe("one\ntwo\nthree") + expect(result.content).toEqual([ + { type: "text", text: "one\n" }, + { type: "text", text: "two" }, + { type: "text", text: expect.stringMatching(/^\[showing lines 1-2 of 3;/) }, + ]) + }), + { maxLines: 2, maxBytes: 1_000 }, + ), + ) + it.live("preserves mixed content ordering", () => withStore( (output) => @@ -144,7 +305,10 @@ describe("ToolOutput", () => { const result = yield* output.truncate({ content: [{ type: "text", text: "one\n" }] }) expect(result.content).toEqual([ { type: "text", text: "one" }, - { type: "text", text: expect.stringMatching(/^\[showing lines 1-1 of 1; full output saved to /) }, + { + type: "text", + text: expect.stringMatching(/^\[showing lines 1-1 of 1; full output saved to /), + }, ]) }), { maxLines: 2, maxBytes: 3 },