fix(core): enforce default read limits for small text files (#63810)

This commit is contained in:
Aiden Cline
2026-11-13 11:30:12 -06:00
committed by GitHub
Unverified
parent f6e0c2d083
commit b3c1936a97
2 changed files with 58 additions and 7 deletions
+14 -7
View File
@@ -155,13 +155,20 @@ export const read = Effect.fn("ReadTool.read")(function* (
const paged = first.info.size > MAX_READ_BYTES || page.offset !== undefined || page.limit !== undefined
if (!paged) {
if (first.bytes.includes(0)) return yield* new BinaryFileError({ resource })
return {
type: "file" as const,
uri: virtual ? resource : pathToFileURL(input).href,
name: virtual ? path.posix.basename(input) : path.basename(input),
content: new TextDecoder().decode(first.bytes).split("\n").map(clampLine).join("\n"),
encoding: "utf8" as const,
mime: mimeType(input),
const lines = new TextDecoder().decode(first.bytes).split("\n")
// A final separator ends the last physical line; keep it in whole-file content.
const count = lines.at(-1) === "" ? lines.length - 1 : lines.length
if (count <= MAX_READ_LINES) {
const content = lines.map(clampLine).join("\n")
if (Buffer.byteLength(content) <= MAX_READ_BYTES)
return {
type: "file" as const,
uri: virtual ? resource : pathToFileURL(input).href,
name: virtual ? path.posix.basename(input) : path.basename(input),
content,
encoding: "utf8" as const,
mime: mimeType(input),
}
}
}
@@ -112,6 +112,50 @@ describe("ReadTool text serialization", () => {
})
describe("ReadToolFileSystem", () => {
it.live("applies the default line limit to small files at physical line boundaries", () =>
Effect.gen(function* () {
const { environment, files, directory } = yield* fixture
const file = absolute(path.join(directory, "dense.txt"))
const read = (page?: ReadToolFileSystem.PageInput) => ReadToolFileSystem.read(environment, file, "dense.txt", page)
const prefix = "x\r\n".repeat(1_999)
for (const ending of ["last", "last\n", "last\r\n", "\n"]) {
const content = prefix + ending
yield* files.writeFileString(file, content)
expect(yield* read()).toMatchObject({ type: "file", content })
}
yield* files.writeFileString(file, prefix + "\r\nlast")
const page = yield* read()
expect(page.type).toBe("text-page")
if (page.type !== "text-page") return
expect(page).toMatchObject({ content: "x\n".repeat(1_999), offset: 1, next: 2_001, truncated: true })
expect(ReadTool.toModelContent("dense.txt", undefined, page)).toEndWith(
"2000: \n[Output truncated. Continue reading with offset: 2001]",
)
expect(yield* read({ offset: page.next })).toMatchObject({ content: "last", offset: 2_001, truncated: false })
}),
)
it.live("applies the byte limit to the returned content of small files", () =>
Effect.gen(function* () {
const { environment, files, directory } = yield* fixture
const file = absolute(path.join(directory, "bytes.txt"))
const read = () => ReadToolFileSystem.read(environment, file, "bytes.txt")
const exact = ("é".repeat(799) + "x\n").repeat(32)
yield* files.writeFileString(file, exact)
expect(yield* read()).toMatchObject({ type: "file", content: exact })
// Each truncation marker takes more bytes than the one character it replaces.
yield* files.writeFileString(file, ("x".repeat(2_001) + "\n").repeat(25) + "é".repeat(500))
const page = yield* read()
expect(page.type).toBe("text-page")
if (page.type !== "text-page") return
expect(page).toMatchObject({ offset: 1, next: 26, truncated: true })
expect(page.content.split("\n")).toHaveLength(25)
expect(Buffer.byteLength(page.content)).toBeLessThanOrEqual(50 * 1024)
}),
)
it.effect("preserves the environment not-found error", () =>
Effect.gen(function* () {
const { environment, directory } = yield* fixture