From 464688ba686b715e28326e209b08116ad60b86e0 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:30:09 -0600 Subject: [PATCH] fix(core): stop keeping the whole file in memory for tail reads (#63030) --- packages/core/src/tool/read-filesystem.ts | 357 +++++++----------- .../core/test/tool-read-filesystem.test.ts | 74 ++++ 2 files changed, 219 insertions(+), 212 deletions(-) diff --git a/packages/core/src/tool/read-filesystem.ts b/packages/core/src/tool/read-filesystem.ts index 7fe5d34b51f..11930dcba8a 100644 --- a/packages/core/src/tool/read-filesystem.ts +++ b/packages/core/src/tool/read-filesystem.ts @@ -17,7 +17,8 @@ export const MAX_MEDIA_INGEST_BYTES = 20 * 1024 * 1024 const FIRST_CHUNK = 256 * 1024 const TAIL_LINES = 100 const MAX_LINE_LENGTH = 2_000 -const TREE_BASE = 6 +// Enough UTF-8 bytes for the clamped prefix, including a BOM and a partial code point at the end. +const LINE_BYTES = 4 * (MAX_LINE_LENGTH + 1) const MAX_LINE_SUFFIX = `... (line truncated to ${MAX_LINE_LENGTH} chars)` export const MEDIA_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp", "application/pdf"]) @@ -123,8 +124,7 @@ export const read = Effect.fn("ReadTool.read")(function* ( const first = yield* files.read(input, { offset: 0, length: FIRST_CHUNK }).pipe( Effect.catchTag("Environment.WrongKind", (error) => Effect.gen(function* () { - if (error.actual !== "directory") - return yield* new PathKindError({ resource, expected: "a file or directory" }) + if (error.actual !== "directory") return yield* new PathKindError({ resource, expected: "a file or directory" }) if (page.offset !== undefined && page.offset < 0) return yield* new DirectoryOffsetError({ resource }) return yield* files.list(input).pipe( Effect.map((entries) => list(entries, page, virtual ? "/" : path.sep)), @@ -165,70 +165,7 @@ export const read = Effect.fn("ReadTool.read")(function* ( } } - if (first.bytes.length >= first.info.size) return yield* completePage(input, resource, first.bytes, page) - - const offset = page.offset || 1 - const limit = Math.min(page.limit || MAX_READ_LINES, MAX_READ_LINES) - const leaves = [textLeaf(first.bytes)] - let bytes = first.bytes.length - let lines = leaves[0].summary.lines - let ended = false - while (true) { - const eof = ended || bytes >= first.info.size - const total = lineCount(lines, leaves.at(-1)?.bytes.at(-1)) - if (eof && offset > total) { - const skipped = Math.max(0, total - Math.min(page.limit || TAIL_LINES, TAIL_LINES)) - // Decode from the newline ending the last skipped line, so a byte order mark - // starting the tail stays content. That empty first line is never in the tail. - const tree = textTree(leaves) - const tail = textPage( - textSlice(leaves, Math.max(0, textOffset(tree, skipped) - 1), tree.summary.bytes), - true, - page, - ) - if (tail === undefined) return yield* Effect.die("Read page did not settle for a complete file") - return yield* makeTextPage( - input, - resource, - { ...tail, offset: Math.max(0, skipped - 1) + tail.offset }, - leaves.some((leaf) => leaf.bytes.includes(0)), - ) - } - if (eof || (offset > 0 && lines >= offset - 1)) { - // A negative offset only resolves at EOF, counting back from the last line. - const from = offset < 0 ? Math.max(1, total + offset + 1) : offset - const tree = textTree(leaves) - const start = textOffset(tree, from - 1) - // Decode only the requested lines plus one byte, enough to tell whether more follow. - const end = Math.min(tree.summary.bytes, textOffset(tree, from - 1 + limit) + 1) - const result = textPage(textSlice(leaves, start, end), eof && end === tree.summary.bytes, { limit }) - if (result !== undefined) { - const translated = { - ...result, - offset: from, - ...(result.next === undefined ? { next: undefined } : { next: from + result.next - 1 }), - } - const consumed = start + result.consumed - let checked = 0 - const binary = leaves.some((leaf) => { - const length = Math.min(leaf.summary.bytes, consumed - checked) - checked += leaf.summary.bytes - return length > 0 && leaf.bytes.subarray(0, length).includes(0) - }) - return yield* makeTextPage(input, resource, translated, binary) - } - } - - const next = yield* readFile(files, input, resource, { offset: bytes, length: FIRST_CHUNK }) - if (next.bytes.length === 0) { - ended = true - continue - } - const leaf = textLeaf(next.bytes) - leaves.push(leaf) - bytes += leaf.summary.bytes - lines += leaf.summary.lines - } + return yield* textPage(files, input, resource, first, page) }) const readFile = ( @@ -243,36 +180,155 @@ const readFile = ( Effect.catchTag("Environment.WrongKind", () => Effect.fail(new PathKindError({ resource, expected: "a file" }))), ) -const completePage = Effect.fnUntraced(function* ( +// The first line ending in a chunk may have started many chunks earlier. +type Checkpoint = { byte: number; length: number; firstLine: number; start: number; lastLine: number } + +const textPage = Effect.fnUntraced(function* ( + files: Files, input: AbsolutePath, resource: string, - bytes: Uint8Array, + first: { bytes: Uint8Array; info: { size: number } }, page: PageInput, ) { - const total = lineCount(textLeaf(bytes).summary.lines, bytes.at(-1)) - const result = textPage( - bytes, - true, - page.offset !== undefined && page.offset < 0 ? { ...page, offset: Math.max(1, total + page.offset + 1) } : page, - ) - if (result === undefined) return yield* Effect.die("Read page did not settle for a complete file") - return yield* makeTextPage(input, resource, result, bytes.subarray(0, result.consumed).includes(0)) -}) + const requested = page.offset || 1 + const limit = Math.min(page.limit || MAX_READ_LINES, MAX_READ_LINES) + const checkpoints: Checkpoint[] = [] + const entries: string[] = [] + let size = 0 + let from = requested + let tail = false + let consumed = 0 + let next: number | undefined + let firstNul = Infinity + const decoder = new TextDecoder("utf-8", { ignoreBOM: true }) -const makeTextPage = Effect.fnUntraced(function* ( - input: AbsolutePath, - resource: string, - result: NonNullable>, - binary: boolean, -) { - if (binary) return yield* new BinaryFileError({ resource }) + const range = (offset: number, length: number) => readFile(files, input, resource, { offset, length }) + const reread = Effect.fnUntraced(function* (offset: number, length: number) { + const { bytes } = yield* range(offset, length) + if (bytes.length === length) return bytes + const result = new Uint8Array(length) + result.set(bytes) + let filled = bytes.length + while (filled < length) { + const more = yield* range(offset + filled, length - filled) + if (more.bytes.length === 0) break + result.set(more.bytes, filled) + filled += more.bytes.length + } + return result.subarray(0, filled) + }) + + // Select complete lines in this chunk. Only a split line's displayable prefix ever needs a separate read. + const collect = Effect.fnUntraced(function* (point: Checkpoint, chunk: Uint8Array) { + let start = point.start + let local = Math.max(0, start - point.byte) + for (let line = point.firstLine; line < from && local < chunk.length; local++) { + if (chunk[local] !== 10) continue + line++ + start = point.byte + local + 1 + } + for (let line = Math.max(point.firstLine, from); line <= point.lastLine; line++) { + if (!tail && (entries.length >= limit || size >= MAX_READ_BYTES)) { + next = line + return + } + const lf = chunk.indexOf(10, local) + const end = point.byte + (lf < 0 ? chunk.length : lf) + const length = Math.min(LINE_BYTES, end - start) + const bytes = + start >= point.byte + ? chunk.subarray(start - point.byte, start - point.byte + length) + : start + length <= first.bytes.length + ? first.bytes.subarray(start, start + length) + : yield* reread(start, length) + let text = decoder.decode(bytes) + // Single-chunk pages decode from byte zero. Streamed pages decode from the selected line, + // except a past-end tail, whose embedded BOM remains content. + if ( + text.startsWith("\uFEFF") && + (start === 0 || (!tail && line === from && first.bytes.length < first.info.size)) + ) + text = text.slice(1) + text = clampLine(text.endsWith("\r") ? text.slice(0, -1) : text) + const cost = Buffer.byteLength(text) + (entries.length > 0 ? 1 : 0) + if (!tail && size + cost > MAX_READ_BYTES) { + next = line + return + } + entries.push(text) + size += cost + consumed = end + Number(lf >= 0) + local = lf + 1 + start = end + 1 + } + }) + + let byte = 0 + let total = 0 + let start = 0 + let chunk = first.bytes + let lastByte = 0 + while (chunk.length > 0) { + lastByte = byte + if (firstNul === Infinity) { + const nul = chunk.indexOf(0) + if (nul >= 0) firstNul = byte + nul + } + let count = 0 + for (let i = 0; i < chunk.length; i++) count += Number(chunk[i] === 10) + if (count > 0) { + const point = { byte, length: chunk.length, firstLine: total + 1, start, lastLine: total + count } + checkpoints.push(point) + total += count + if (requested > 0 && total >= requested) yield* collect(point, chunk) + start = byte + chunk.lastIndexOf(10) + 1 + } + byte += chunk.length + // Any byte after a full page starts another line, even if that line has no newline for megabytes. + if (next === undefined && (entries.length >= limit || size >= MAX_READ_BYTES) && consumed < byte) + next = requested + entries.length + if (next !== undefined || byte >= first.info.size) break + const more = yield* range(byte, FIRST_CHUNK) + if (more.bytes.length === 0) break + chunk = more.bytes + } + + // EOF ends an unterminated line. A trailing newline already ended the last one. + if (next === undefined && start < byte) { + total++ + const point = { byte: lastByte, length: chunk.length, firstLine: total, start, lastLine: total } + checkpoints.push(point) + if (requested > 0 && total >= requested) yield* collect(point, chunk) + } + if (requested < 0 || requested > total) { + tail = requested > total + from = Math.max(1, total + 1 + (tail ? -Math.min(page.limit || TAIL_LINES, TAIL_LINES) : requested)) + // Each checkpoint covers disjoint line numbers. Seek to the chunk ending the first selected line; + // chunks in the middle of a long line have no checkpoint and are never reread. + for (const point of checkpoints) { + if (point.lastLine < from) continue + const bytes = + point.byte === 0 ? first.bytes : point.byte === lastByte ? chunk : yield* reread(point.byte, point.length) + yield* collect(point, bytes) + if (next !== undefined) break + } + if (tail) { + while (size > MAX_READ_BYTES) { + size -= Buffer.byteLength(entries.shift() ?? "") + 1 + from++ + } + consumed = byte + } + } + + if (firstNul < consumed) return yield* new BinaryFileError({ resource }) return new TextPage({ type: "text-page", - content: result.entries.join("\n"), + content: entries.join("\n"), mime: mimeType(input), - offset: result.offset, - truncated: result.next !== undefined, - ...(result.next === undefined ? {} : { next: result.next }), + offset: from, + truncated: next !== undefined, + ...(next === undefined ? {} : { next }), }) }) @@ -309,51 +365,6 @@ const list = (items: ReadonlyArray, page: PageInput, sep = }) } -const textPage = (bytes: Uint8Array, eof: boolean, page: PageInput) => { - const offset = page.offset || 1 - const limit = Math.min(page.limit || MAX_READ_LINES, MAX_READ_LINES) - const decoded = new TextDecoder().decode(bytes) - const split = decoded.split("\n") - const complete = eof ? (split.at(-1) === "" ? split.slice(0, -1) : split) : split.slice(0, -1) - const available = complete.map((line) => (line.endsWith("\r") ? line.slice(0, -1) : line)) - - const entries: string[] = [] - let size = 0 - // An offset past the end returns the file's last lines that fit, ending at the last line. - if (eof && offset > Math.max(1, available.length)) { - for (const value of available.slice(-Math.min(page.limit || TAIL_LINES, TAIL_LINES)).toReversed()) { - const text = clampLine(value) - size += Buffer.byteLength(text, "utf-8") + (entries.length > 0 ? 1 : 0) - if (size > MAX_READ_BYTES) break - entries.unshift(text) - } - return { entries, offset: available.length - entries.length + 1, next: undefined, consumed: bytes.length } - } - let next: number | undefined - for (const [index, value] of available.slice(offset - 1).entries()) { - const line = offset + index - if (entries.length >= limit || size >= MAX_READ_BYTES) { - next = line - break - } - const text = clampLine(value) - const lineSize = Buffer.byteLength(text, "utf-8") + (entries.length > 0 ? 1 : 0) - if (size + lineSize > MAX_READ_BYTES) { - next = line - break - } - entries.push(text) - size += lineSize - } - if (next === undefined && entries.length >= limit && (!eof || offset - 1 + entries.length < available.length)) - next = offset + entries.length - if (!eof && next === undefined) return - - const consumedLines = next === undefined ? available.length : next - 1 - const consumed = consumedLines === 0 ? 0 : (nthNewline(bytes, consumedLines) ?? bytes.length) - return { entries, offset, next, consumed } -} - const clampLine = (line: string) => { if (line.length <= MAX_LINE_LENGTH) return line const code = line.charCodeAt(MAX_LINE_LENGTH - 1) @@ -361,84 +372,6 @@ const clampLine = (line: string) => { return line.slice(0, end) + MAX_LINE_SUFFIX } -// A trailing newline ends the last line instead of starting an empty one. -const lineCount = (newlines: number, last: number | undefined) => newlines + (last === undefined || last === 10 ? 0 : 1) - -type TextSummary = { readonly bytes: number; readonly lines: number } -// Request-local augmented rope. Subtree byte and newline weights locate a line -// like an order-statistic query without repeatedly decoding the accumulated text. -// https://doi.org/10.1002/spe.4380251203 -type TextNode = - | { readonly type: "leaf"; readonly bytes: Uint8Array; readonly summary: TextSummary } - | { readonly type: "branch"; readonly children: ReadonlyArray; readonly summary: TextSummary } - -const textLeaf = (bytes: Uint8Array): Extract => { - let lines = 0 - for (const byte of bytes) if (byte === 10) lines++ - return { type: "leaf", bytes, summary: { bytes: bytes.length, lines } } -} - -const textTree = (nodes: ReadonlyArray): TextNode => { - if (nodes.length === 1) return nodes[0] - return textTree( - Array.from({ length: Math.ceil(nodes.length / (TREE_BASE * 2)) }, (_, index) => { - const children = nodes.slice(index * TREE_BASE * 2, (index + 1) * TREE_BASE * 2) - return { - type: "branch" as const, - children, - summary: { - bytes: children.reduce((total, child) => total + child.summary.bytes, 0), - lines: children.reduce((total, child) => total + child.summary.lines, 0), - }, - } - }), - ) -} - -const textOffset = (tree: TextNode, newline: number) => { - if (newline === 0) return 0 - let node = tree - let remaining = newline - let offset = 0 - while (node.type === "branch") { - const child = node.children.find((candidate) => { - if (remaining <= candidate.summary.lines) return true - remaining -= candidate.summary.lines - offset += candidate.summary.bytes - return false - }) - if (!child) return tree.summary.bytes - node = child - } - const end = nthNewline(node.bytes, remaining) - return end === undefined ? tree.summary.bytes : offset + end -} - -const textSlice = ( - leaves: ReadonlyArray>, - start: number, - end: number, -) => { - let position = 0 - return Buffer.concat( - leaves.flatMap((leaf) => { - const leafStart = position - position += leaf.summary.bytes - if (position <= start || leafStart >= end) return [] - return [leaf.bytes.subarray(Math.max(0, start - leafStart), end - leafStart)] - }), - ) -} - -const nthNewline = (bytes: Uint8Array, count: number) => { - let found = 0 - for (const [index, byte] of bytes.entries()) { - if (byte !== 10) continue - found++ - if (found === count) return index + 1 - } -} - const layer = Layer.effect( Service, Effect.gen(function* () { diff --git a/packages/core/test/tool-read-filesystem.test.ts b/packages/core/test/tool-read-filesystem.test.ts index f60db5839a5..f4997f9b4b2 100644 --- a/packages/core/test/tool-read-filesystem.test.ts +++ b/packages/core/test/tool-read-filesystem.test.ts @@ -392,6 +392,53 @@ describe("ReadToolFileSystem", () => { }), ) + it.effect("reads the end of a file many chunks long", () => + Effect.gen(function* () { + const { environment, files, directory } = yield* fixture + const read = (name: string, content: string | Uint8Array, page: ReadToolFileSystem.PageInput) => + (typeof content === "string" + ? files.writeFileString(path.join(directory, name), content) + : files.writeFile(path.join(directory, name), content) + ).pipe( + Effect.andThen(ReadToolFileSystem.read(environment, absolute(path.join(directory, name)), name, page)), + Effect.map((result) => String(ReadTool.toModelContent(name, page.offset, result)).split("\n")), + ) + // 60,000 lines of 51 bytes span 12 chunks. + const lines = Array.from({ length: 60_000 }, (_, index) => `${index + 1}`.padEnd(50, "x")) + const content = lines.join("\n") + "\n" + const expected = (from: number, to: number) => + lines.slice(from - 1, to).map((line, index) => `${from + index}: ${line}`) + + expect(yield* read("negative.txt", content, { offset: -5 })).toEqual([ + "Read file negative.txt, lines 59996-60000", + ...expected(59_996, 60_000), + ]) + expect(yield* read("past.txt", content, { offset: 70_000, limit: 3 })).toEqual([ + "Read file past.txt, lines 59998-60000 (offset 70000 is past the end; the file has 60000 lines)", + ...expected(59_998, 60_000), + ]) + expect(yield* read("deep.txt", content, { offset: 45_000, limit: 2 })).toEqual([ + "Read file deep.txt, lines 45000-45001", + ...expected(45_000, 45_001), + "[Output truncated. Continue reading with offset: 45002]", + ]) + // Far from the end, a negative offset still reads from the line it counts back to. + expect(yield* read("far.txt", content, { offset: -10_000, limit: 3 })).toEqual([ + "Read file far.txt, lines 50001-50003", + ...expected(50_001, 50_003), + "[Output truncated. Continue reading with offset: 50004]", + ]) + + // A null byte in an earlier scanned chunk still marks the file as binary. + const early = new TextEncoder().encode(content) + early[10] = 0 + const negative = yield* read("early-nul.txt", early, { offset: -5 }).pipe(Effect.flip) + expect(negative).toBeInstanceOf(ReadToolFileSystem.BinaryFileError) + const past = yield* read("early-nul-past.txt", early, { offset: 70_000 }).pipe(Effect.flip) + expect(past).toBeInstanceOf(ReadToolFileSystem.BinaryFileError) + }), + ) + it.effect("rejects a negative offset for a directory", () => Effect.gen(function* () { const { environment, files, directory } = yield* fixture @@ -496,6 +543,33 @@ describe("ReadToolFileSystem", () => { }), ) + it.live("ignores a NUL after a chunked page but rejects it for a past-end offset", () => + Effect.gen(function* () { + const { environment, files, directory } = yield* fixture + const file = absolute(path.join(directory, "nul.txt")) + yield* files.writeFileString(file, `${"x".repeat(256 * 1024)}\nselected\n\0\ntail`) + const read = (offset: number) => ReadToolFileSystem.read(environment, file, "nul.txt", { offset, limit: 1 }) + + for (const offset of [2, -3]) { + expect(yield* read(offset)).toMatchObject({ content: "selected", offset: 2, truncated: true, next: 3 }) + } + expect(yield* read(5).pipe(Effect.flip)).toBeInstanceOf(ReadToolFileSystem.BinaryFileError) + }), + ) + + it.live("keeps a split byte order mark in a past-end tail and strips it at a negative offset", () => + Effect.gen(function* () { + const { environment, files, directory } = yield* fixture + const file = absolute(path.join(directory, "marked.txt")) + // The first byte of the three-byte UTF-8 mark is the last byte of the first chunk. + yield* files.writeFileString(file, `${"x".repeat(256 * 1024 - 2)}\n\uFEFFselected`) + const read = (offset: number) => ReadToolFileSystem.read(environment, file, "marked.txt", { offset, limit: 1 }) + + expect(yield* read(3)).toMatchObject({ content: "\uFEFFselected", offset: 2, truncated: false }) + expect(yield* read(-1)).toMatchObject({ content: "selected", offset: 2, truncated: false }) + }), + ) + it.effect("checks skipped lines for null bytes", () => Effect.gen(function* () { const { environment, files, directory } = yield* fixture