diff --git a/packages/core/src/tool/read-filesystem.ts b/packages/core/src/tool/read-filesystem.ts index 11930dcba8a..98962a9ee54 100644 --- a/packages/core/src/tool/read-filesystem.ts +++ b/packages/core/src/tool/read-filesystem.ts @@ -155,13 +155,20 @@ export const read = Effect.fn("ReadTool.read")(function* ( const paged = first.info.size > MAX_READ_BYTES || page.offset !== undefined || page.limit !== undefined if (!paged) { if (first.bytes.includes(0)) return yield* new BinaryFileError({ resource }) - return { - type: "file" as const, - uri: virtual ? resource : pathToFileURL(input).href, - name: virtual ? path.posix.basename(input) : path.basename(input), - content: new TextDecoder().decode(first.bytes).split("\n").map(clampLine).join("\n"), - encoding: "utf8" as const, - mime: mimeType(input), + const lines = new TextDecoder().decode(first.bytes).split("\n") + // A final separator ends the last physical line; keep it in whole-file content. + const count = lines.at(-1) === "" ? lines.length - 1 : lines.length + if (count <= MAX_READ_LINES) { + const content = lines.map(clampLine).join("\n") + if (Buffer.byteLength(content) <= MAX_READ_BYTES) + return { + type: "file" as const, + uri: virtual ? resource : pathToFileURL(input).href, + name: virtual ? path.posix.basename(input) : path.basename(input), + content, + encoding: "utf8" as const, + mime: mimeType(input), + } } } diff --git a/packages/core/test/tool-read-filesystem.test.ts b/packages/core/test/tool-read-filesystem.test.ts index f4997f9b4b2..7b32291646f 100644 --- a/packages/core/test/tool-read-filesystem.test.ts +++ b/packages/core/test/tool-read-filesystem.test.ts @@ -112,6 +112,50 @@ describe("ReadTool text serialization", () => { }) describe("ReadToolFileSystem", () => { + it.live("applies the default line limit to small files at physical line boundaries", () => + Effect.gen(function* () { + const { environment, files, directory } = yield* fixture + const file = absolute(path.join(directory, "dense.txt")) + const read = (page?: ReadToolFileSystem.PageInput) => ReadToolFileSystem.read(environment, file, "dense.txt", page) + const prefix = "x\r\n".repeat(1_999) + for (const ending of ["last", "last\n", "last\r\n", "\n"]) { + const content = prefix + ending + yield* files.writeFileString(file, content) + expect(yield* read()).toMatchObject({ type: "file", content }) + } + + yield* files.writeFileString(file, prefix + "\r\nlast") + const page = yield* read() + expect(page.type).toBe("text-page") + if (page.type !== "text-page") return + expect(page).toMatchObject({ content: "x\n".repeat(1_999), offset: 1, next: 2_001, truncated: true }) + expect(ReadTool.toModelContent("dense.txt", undefined, page)).toEndWith( + "2000: \n[Output truncated. Continue reading with offset: 2001]", + ) + expect(yield* read({ offset: page.next })).toMatchObject({ content: "last", offset: 2_001, truncated: false }) + }), + ) + + it.live("applies the byte limit to the returned content of small files", () => + Effect.gen(function* () { + const { environment, files, directory } = yield* fixture + const file = absolute(path.join(directory, "bytes.txt")) + const read = () => ReadToolFileSystem.read(environment, file, "bytes.txt") + const exact = ("é".repeat(799) + "x\n").repeat(32) + yield* files.writeFileString(file, exact) + expect(yield* read()).toMatchObject({ type: "file", content: exact }) + + // Each truncation marker takes more bytes than the one character it replaces. + yield* files.writeFileString(file, ("x".repeat(2_001) + "\n").repeat(25) + "é".repeat(500)) + const page = yield* read() + expect(page.type).toBe("text-page") + if (page.type !== "text-page") return + expect(page).toMatchObject({ offset: 1, next: 26, truncated: true }) + expect(page.content.split("\n")).toHaveLength(25) + expect(Buffer.byteLength(page.content)).toBeLessThanOrEqual(50 * 1024) + }), + ) + it.effect("preserves the environment not-found error", () => Effect.gen(function* () { const { environment, directory } = yield* fixture