diff --git a/packages/core/src/session/compaction.ts b/packages/core/src/session/compaction.ts index a6b7f5d05cb..7197ea85bd4 100644 --- a/packages/core/src/session/compaction.ts +++ b/packages/core/src/session/compaction.ts @@ -117,7 +117,7 @@ type Precomputed = { const NOTHING_TO_COMPACT: Failure = { error: { type: "compaction.unavailable", message: "Nothing to compact yet" } } /** After each "too long" rejection, the next attempt aims at this share of the first rejected request's size. */ -const SHRINK_STEPS = [0.7, 0.5, 0.35] +const SHRINK_STEPS = [0.85, 0.7, 0.5, 0.35] // The least of the window kept free for the last reply before compaction and for the summary itself. const RESERVE_MIN = 16_000 /** @@ -130,6 +130,17 @@ const RESERVE_SHARE = 0.15 const UNKNOWN_WINDOW = 200_000 /** A summary gets at least this much output, or the room left if less, even when the estimate margin leaves less. */ const SUMMARY_OUTPUT_FLOOR = 32_000 +/** + * How far one exchange of a summary transcript is shortened, in the order the levels apply: whole; tool outputs cut; + * tool outputs left out and tool inputs cut; only the user's text, cut; omitted. + */ +const OUTPUTS_CLIPPED = 1 +const OUTPUTS_REMOVED = 2 +const USER_ONLY = 3 +const OMITTED = 4 +const TRANSCRIPT_LEVELS = [0, OUTPUTS_CLIPPED, OUTPUTS_REMOVED, USER_ONLY, OMITTED] +/** Characters kept, from the start and end, of a cut tool input, tool output, or user message. */ +const TRANSCRIPT_CLIP = 2_000 const IMAGE_TOKEN_ESTIMATE = 1_500 const PDF_TOKEN_ESTIMATE = 2_000 /** Share of the compaction ceiling at which a background summary starts. */ @@ -549,11 +560,12 @@ export const layer = Layer.effect( * and otherwise `shrink.history` as text aimed at `target` (see `flattenAndDropOldest`). * Limits and estimates can be wrong, so until the provider rejects something as too long, a request that cannot * be made to fit is sent whole: unchanged, or as text once its media has to go. A "too long" rejection means the - * estimate ran low, so the next attempts aim at 70%, 50%, then 35% of the first rejected request's estimate, and a - * rejection after those gives up. A "payload too large" rejection is about bytes, which inline media almost always - * accounts for, so one on the unchanged request resends it as text, which carries no media, without counting; one - * on text counts as "too long". Other provider errors resend the same request under the session's retry policy. A - * `Failure` from `send` is a reply that cannot be used, and is never retried. + * estimate ran low, so the next attempts aim at 85%, 70%, 50%, then 35% of the first rejected request's estimate, + * each shortening the transcript again from scratch, and a rejection after those gives up. A "payload too large" + * rejection is about bytes, which inline media almost always accounts for, so one on the unchanged request resends + * it as text, which carries no media, without counting; one on text counts as "too long". Other provider errors + * resend the same request under the session's retry policy. A `Failure` from `send` is a reply that cannot be + * used, and is never retried. */ const deliver = Effect.fnUntraced(function* ( trigger: Trigger, @@ -637,9 +649,13 @@ export const layer = Layer.effect( * History that has to change is sent as text rather than as edited messages: `[previous compaction][transcript]`, * the conversation since that compaction as one text message. A previous summary always stays in front, while * its verbatim recent part becomes the oldest transcript entry; a previous native window stays exactly as the - * provider made it. The transcript leaves out reasoning, media, and instruction updates, and cuts long tool - * outputs. Only if it still does not fit are whole exchanges dropped, oldest first. Undefined when even the - * newest exchange does not fit. + * provider made it. The transcript has no tool definitions, which the summary does not need and which no longer + * share a cached prefix, and leaves out reasoning, media, and instruction updates. + * + * When the whole transcript does not fit, exchanges are shortened oldest first, one level at a time, until it + * does (see `flattenMessage` for the levels): every exchange reaches a level before any reaches the next, so + * detail is lost from the oldest history first and whole exchanges are omitted only as a last resort. The newest + * exchange is never omitted; undefined when the transcript cannot fit even so. */ const flattenAndDropOldest = (request: LLMRequest, context: SessionContext.Loaded, target: number) => { const previous = previousCompaction(context.messages) @@ -657,30 +673,58 @@ export const layer = Layer.effect( : request.messages.slice(0, end) const recent = summary ? SessionRecentText.kept(summary) : "" - // A user message and everything after it, up to the next one, is one exchange: kept or dropped whole. - const exchanges = request.messages.slice(end).reduce( - (groups, message) => { - const text = flattenMessage(message) - if (!text) return groups - if (message.role === "user" || groups.length === 0) groups.push(text) - else groups[groups.length - 1] += `\n${text}` - return groups - }, - recent ? [recent] : [], - ) - + // A user message and everything after it, up to the next one, is one exchange. Each is rendered at every level + // once, so shortening is running arithmetic over these sizes. The previous summary's recent part is already + // shortened text: whole until it is omitted. + const exchanges = [ + ...(recent ? [TRANSCRIPT_LEVELS.map((level) => (level === OMITTED ? "" : recent))] : []), + ...request.messages + .slice(end) + .reduce((groups, message) => { + if (message.role === "user" || groups.length === 0) groups.push([message]) + else groups[groups.length - 1].push(message) + return groups + }, []) + .map((group) => + TRANSCRIPT_LEVELS.map((level) => + group + .map((message) => flattenMessage(message, level)) + .filter(Boolean) + .join("\n"), + ), + ) + .filter((renders) => renders[0]), + ] + if (exchanges.length === 0) return undefined const note = (omitted: number) => omitted ? `[${omitted} older ${omitted === 1 ? "exchange" : "exchanges"} omitted]\n\n` : "" + const size = (text: string) => (text ? Token.estimate(text) + 1 : 0) // Room for the longest possible note is set aside before choosing what to keep. const room = target - - estimateRequest({ system: request.system, tools: request.tools, messages: lead }) - + estimateRequest({ system: request.system, tools: [], messages: lead }) - Token.estimate(note(exchanges.length)) - const kept = exchanges.slice(oldestToDrop(exchanges, (text) => Token.estimate(text) + 1, room)) - if (kept.length === 0) return undefined - return LLMRequest.update(request, { - messages: [...lead, Message.user(note(exchanges.length - kept.length) + kept.join("\n\n"))], - }) + + const levels = exchanges.map(() => 0) + let total = exchanges.reduce((sum, renders) => sum + size(renders[0]), 0) + const newest = exchanges.length - 1 + for (const level of TRANSCRIPT_LEVELS.slice(1)) { + for (let index = 0; index < exchanges.length && total > room; index++) { + // The newest exchange is what the summary most needs; it is shortened but never left out. + if (index === newest && !exchanges[index][level]) continue + total += size(exchanges[index][level]) - size(exchanges[index][levels[index]]) + levels[index] = level + } + } + if (total > room) return undefined + + // Includes exchanges with no user text, which are already gone at `USER_ONLY`. + const omitted = levels.filter((level, index) => !exchanges[index][level]).length + const text = exchanges + .map((renders, index) => renders[levels[index]]) + .filter(Boolean) + .join("\n\n") + return LLMRequest.update(request, { tools: [], messages: [...lead, Message.user(note(omitted) + text)] }) } const stream = ( @@ -963,29 +1007,48 @@ const oldestToDrop = (items: ReadonlyArray, size: (item: T) => number, bud return start } -/** One request message as a flattened transcript line. Empty for what the transcript leaves out. */ -const flattenMessage = (message: Message) => { - if (message.role === "system") return "" +/** One request message as transcript lines at a shortening level. Empty for what that level leaves out. */ +const flattenMessage = (message: Message, level: number) => { + if (message.role === "system" || level === OMITTED) return "" + if (level >= USER_ONLY && message.role !== "user") return "" return message.content .flatMap((part) => { - if (part.type === "text") - return part.text ? [`[${message.role === "user" ? "User" : "Assistant"}]: ${part.text}`] : [] + if (part.type === "text") { + if (!part.text) return [] + const text = level >= USER_ONLY ? clip(part.text) : part.text + return [`[${message.role === "user" ? "User" : "Assistant"}]: ${text}`] + } + if (level >= USER_ONLY) return [] if (part.type === "media") return [`[${part.media.mediaType} omitted]`] - if (part.type === "tool-call") return [`[Assistant tool call]: ${part.name}(${JSON.stringify(part.input) ?? ""})`] + if (part.type === "tool-call") { + const input = JSON.stringify(part.input) ?? "" + return [`[Assistant tool call]: ${part.name}(${level >= OUTPUTS_REMOVED ? clip(input) : input})`] + } if (part.type !== "tool-result") return [] const result = part.result + const label = result.type === "error" ? "Tool error" : "Tool result" + if (level >= OUTPUTS_REMOVED) return [`[${label} omitted]`] const output = result.type === "content" ? SessionRecentText.serializeToolContent(result.value) : typeof result.value === "string" ? result.value : (JSON.stringify(result.value) ?? "") - const label = result.type === "error" ? "Tool error" : "Tool result" - return [`[${label}]: ${SessionRecentText.truncateToolOutput(output)}`] + return [`[${label}]: ${level >= OUTPUTS_CLIPPED ? clip(output) : output}`] }) .join("\n") } +/** `value` cut to `TRANSCRIPT_CLIP` characters from its start and end, never splitting a surrogate pair. */ +const clip = (value: string) => { + if (value.length <= TRANSCRIPT_CLIP) return value + const chars = Array.from(value) + if (chars.length <= TRANSCRIPT_CLIP) return value + const half = TRANSCRIPT_CLIP / 2 + const omitted = chars.length - TRANSCRIPT_CLIP + return `${chars.slice(0, half).join("")}\n[… ${omitted} characters omitted …]\n${chars.slice(-half).join("")}` +} + /** The newest whole, real user messages within `keep` tokens: no synthetic guidance, no half of an attachment. */ export const recentUserMessages = ( messages: ReadonlyArray, diff --git a/packages/core/src/session/recent-text.ts b/packages/core/src/session/recent-text.ts index 21cf1710fe4..1f5c923249b 100644 --- a/packages/core/src/session/recent-text.ts +++ b/packages/core/src/session/recent-text.ts @@ -39,7 +39,9 @@ export const messageToText = (message: SessionMessage.Info): string => { return message.content .flatMap((part) => { if (part.type === "text") return [`[Assistant]: ${part.text}`] - if (part.type === "reasoning") return part.text ? [`[Assistant reasoning]: ${part.text}`] : [] + // Providers return summaries of the real reasoning, and written out as text it can read like an attempt to + // extract the model's thinking. + if (part.type === "reasoning") return [] const input = typeof part.state.input === "string" ? part.state.input : JSON.stringify(part.state.input) const call = `[Assistant tool call]: ${part.name}(${input})` diff --git a/packages/core/test/session-compaction.test.ts b/packages/core/test/session-compaction.test.ts index 52f9e248f14..be32f66ed84 100644 --- a/packages/core/test/session-compaction.test.ts +++ b/packages/core/test/session-compaction.test.ts @@ -722,14 +722,17 @@ it.effect("compaction lowers the output limit when it sends a request it could n Effect.gen(function* () { requests = [] const session = yield* insertSession(Session.ID.make("ses_unshrinkable_compaction")) - // One exchange estimated at 250k tokens, past the 200k window, so no older exchange can be dropped. + // A previous summary estimated at 250k tokens, past the 200k window; it always stays whole. const messages = [ - SessionMessage.User.make({ + SessionMessage.CompactionCompleted.make({ id: SessionMessage.ID.create(), - type: "user", - text: "x".repeat(1_000_000), + type: "compaction", + status: "completed", + reason: "manual", + summary: "x".repeat(1_000_000), time: { created: DateTime.makeUnsafe(0) }, }), + textUser("Next question"), ] expect(yield* compactManually(session, messages)).toEqual({ status: "completed" }) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 7289ba02cb7..3dfbe54c9d7 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -2829,7 +2829,7 @@ describe("SessionRunnerLLM", () => { }) } - scenario("stops after three smaller compaction inputs overflow", function* (s) { + scenario("stops after four smaller compaction inputs overflow", function* (s) { // Large enough that the conversation, not the system prompt, is most of the request. const filler = "context ".repeat(1_000) yield* s.llm.push(...Array.from({ length: 8 }, (_, index) => TestLLM.text(`Answer ${index}`, `answer-${index}`))) @@ -2843,17 +2843,18 @@ describe("SessionRunnerLLM", () => { Stream.fail( new AIError({ reason: new InvalidRequestError({ message: "Too long", classification: "context-overflow" }) }), ) - yield* s.llm.push(overflow(), overflow(), overflow(), overflow()) + yield* s.llm.push(overflow(), overflow(), overflow(), overflow(), overflow()) const compaction = yield* s.session.compact({ sessionID }) yield* s.resume - expect(s.requests).toHaveLength(4) + expect(s.requests).toHaveLength(5) expect(userTexts(s.requests[2])[0].length).toBeLessThan(userTexts(s.requests[1])[0].length) expect(userTexts(s.requests[3])[0].length).toBeLessThan(userTexts(s.requests[2])[0].length) + expect(userTexts(s.requests[4])[0].length).toBeLessThan(userTexts(s.requests[3])[0].length) expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({ status: "failed" }) yield* s.llm.push(TestLLM.text("Continued", "continued")) yield* s.runPrompt("Continue") - expect(userTexts(s.requests[4])).toContain(`Request 0: ${filler}`) + expect(userTexts(s.requests[5])).toContain(`Request 0: ${filler}`) }) scenario("resends compaction as text after payload too large, then shrinks later rejections", function* (s) { @@ -2873,12 +2874,12 @@ describe("SessionRunnerLLM", () => { ) s.currentModel = unknownContextModel s.requests.length = 0 - yield* s.llm.push(...Array.from({ length: 5 }, () => Stream.fail(payloadTooLarge()))) + yield* s.llm.push(...Array.from({ length: 6 }, () => Stream.fail(payloadTooLarge()))) const compaction = yield* s.session.compact({ sessionID }) yield* s.resume // The first rejection resends everything as text, which carries no media; later ones shrink like "too long". - expect(s.requests).toHaveLength(5) + expect(s.requests).toHaveLength(6) expect(s.requests[0]?.messages.some((message) => message.content.some((part) => part.type === "media"))).toBeTrue() expect(s.requests[1]?.messages.every((message) => message.role === "user")).toBeTrue() expect(userTexts(s.requests[1])[0]).toContain("[image/png omitted]") @@ -2886,10 +2887,11 @@ describe("SessionRunnerLLM", () => { expect(userTexts(s.requests[2])[0].length).toBeLessThan(userTexts(s.requests[1])[0].length) expect(userTexts(s.requests[3])[0].length).toBeLessThan(userTexts(s.requests[2])[0].length) expect(userTexts(s.requests[4])[0].length).toBeLessThan(userTexts(s.requests[3])[0].length) + expect(userTexts(s.requests[5])[0].length).toBeLessThan(userTexts(s.requests[4])[0].length) expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({ status: "failed" }) }) - scenario("resends whole history as text after payload too large when it cannot be shortened", function* (s) { + scenario("shortens the only exchange's user text when it does not fit", function* (s) { const image = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" yield* s.session.prompt({ sessionID, @@ -2901,16 +2903,15 @@ describe("SessionRunnerLLM", () => { yield* s.resume s.currentModel = testModel("smaller-history", { context: 7_000, output: 1_000 }) s.requests.length = 0 - yield* s.llm.push(Stream.fail(payloadTooLarge()), TestLLM.text("## Objective\n- Recovered", "summary")) + yield* s.llm.push(TestLLM.text("## Objective\n- Recovered", "summary")) const compaction = yield* s.session.compact({ sessionID }) yield* s.resume - // Even the latest exchange is over the estimated limit, so the first send is unchanged and the second keeps it all. - expect(s.requests).toHaveLength(2) - expect(s.requests[0]?.messages.some((message) => message.content.some((part) => part.type === "media"))).toBeTrue() - expect(s.requests[1]?.messages.every((message) => message.role === "user")).toBeTrue() - expect(userTexts(s.requests[1])[0]).toContain("[image/png omitted]") - expect(userTexts(s.requests[1])[0]).toContain(`Request 0: ${"x".repeat(40_000)}`) + // The only exchange is over the estimated limit, so its text is cut to its start and end rather than sent whole. + expect(s.requests).toHaveLength(1) + expect(s.requests[0]?.messages.every((message) => message.role === "user")).toBeTrue() + expect(userTexts(s.requests[0])[0]).toContain("characters omitted") + expect(userTexts(s.requests[0])[0]).not.toContain("x".repeat(2_001)) expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({ status: "completed" }) }) @@ -2952,14 +2953,14 @@ describe("SessionRunnerLLM", () => { ) yield* s.runPrompt("Continue") - // The history fits the model's window by estimate, but the provider just rejected it, so older exchanges go. + // The history fits the model's window by estimate, but the provider just rejected it, so older exchanges are cut. expect(s.requests).toHaveLength(3) - expect(userTexts(s.requests[1])[0]).toContain("older exchanges omitted") - expect(userTexts(s.requests[1])[0]).not.toContain("Request 0:") + expect(userTexts(s.requests[1])[0]).toContain("characters omitted") + expect(userTexts(s.requests[1])[0]).not.toContain(`Request 0: ${filler}`) expect(userTexts(s.requests[1])[0]).toContain("Request 3:") }) - scenario("serializes history and omits media after a summary input overflow", function* (s) { + scenario("serializes history without tools or media after a summary input overflow", function* (s) { const image = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" yield* s.session.prompt({ sessionID, @@ -2995,20 +2996,61 @@ describe("SessionRunnerLLM", () => { expect(s.requests[1]?.messages.every((message) => message.role === "user")).toBeTrue() expect(userTexts(s.requests[1])[0]).toContain("[image/png omitted]") expect(userTexts(s.requests[1])[0]).toContain("[Tool result]:") - expect(userTexts(s.requests[1])[0]).toContain(`${"x".repeat(1_250)}\n[truncated]`) - expect(userTexts(s.requests[1])[0]).not.toContain("x".repeat(1_251)) + // Tool output is only cut when the transcript would not fit whole. + expect(userTexts(s.requests[1])[0]).toContain("x".repeat(5_000)) expect(s.requests[1]?.system).toEqual(s.requests[0]?.system) - expect(s.requests[1]?.tools).toEqual(s.requests[0]?.tools) + expect(s.requests[1]?.tools).toEqual([]) expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({ status: "completed" }) }) + scenario("drops tool output in serialized compaction history before omitting exchanges", function* (s) { + const service = yield* SessionCompaction.Service + yield* service.transform((editor) => editor.configure({ buffer: 3_000 })) + s.currentModel = testModel("large-history", { context: 1_000_000, output: 32_000 }) + yield* s.llm.push( + ...Array.from({ length: 2 }, (_, index) => + TestLLM.stop( + ...Array.from({ length: 12 }, (_, call) => [ + LLMEvent.toolCall({ + id: `hosted-${index}-${call}`, + name: "web_search", + input: { query: "q" }, + providerExecuted: true, + }), + LLMEvent.toolResult({ + id: `hosted-${index}-${call}`, + name: "web_search", + result: { type: "text", value: "x".repeat(2_000) }, + providerExecuted: true, + }), + ]).flat(), + LLMEvent.textStart({ id: `answer-${index}` }), + LLMEvent.textDelta({ id: `answer-${index}`, text: `Answer ${index}` }), + LLMEvent.textEnd({ id: `answer-${index}` }), + ), + ), + ) + yield* Effect.forEach([0, 1], (index) => s.runPrompt(`Request ${index}`)) + s.currentModel = testModel("smaller-history", { context: 7_000, output: 1_000 }) + s.requests.length = 0 + yield* s.llm.push(TestLLM.text("## Objective\n- Recovered", "summary")) + yield* s.session.compact({ sessionID }) + yield* s.resume + + expect(s.requests).toHaveLength(1) + expect(userTexts(s.requests[0])[0]).toContain("Request 0") + // The oldest exchange loses its tool outputs rather than being dropped. + expect(userTexts(s.requests[0])[0]).toContain("[Tool result omitted]") + expect(userTexts(s.requests[0])[0]).not.toMatch(/older exchanges? omitted/) + }) + scenario("fits serialized compaction history by omitting oldest exchanges", function* (s) { const service = yield* SessionCompaction.Service yield* service.transform((editor) => editor.configure({ buffer: 3_000 })) s.currentModel = testModel("large-history", { context: 1_000_000, output: 32_000 }) - yield* s.llm.push(...Array.from({ length: 4 }, (_, index) => TestLLM.text(`Answer ${index}`, `answer-${index}`))) + yield* s.llm.push(...Array.from({ length: 16 }, (_, index) => TestLLM.text(`Answer ${index}`, `answer-${index}`))) yield* Effect.forEach( - Array.from({ length: 4 }, (_, index) => index), + Array.from({ length: 16 }, (_, index) => index), (index) => s.runPrompt(`Request ${index}: ${"x".repeat(8_000)}`), ) s.currentModel = testModel("smaller-history", { context: 7_000, output: 1_000 }) @@ -3017,13 +3059,12 @@ describe("SessionRunnerLLM", () => { const compaction = yield* s.session.compact({ sessionID }) yield* s.resume + // Even cut to the user's text, the older exchanges do not all fit, so the oldest are omitted. expect(s.requests).toHaveLength(1) expect(userTexts(s.requests[0])[0]).toContain("older exchanges omitted") expect(userTexts(s.requests[0])[0]).not.toContain("Request 0:") - expect(userTexts(s.requests[0])[0]).not.toContain("Request 1:") - // The latest exchange is the retained tail, so shrinking drops only exchanges before it. - expect(userTexts(s.requests[0])[0]).toContain("Request 2:") - expect(userTexts(s.requests[0])[0]).not.toContain("Request 3:") + // The newest exchanges are the retained tail, so shrinking drops only exchanges before them. + expect(userTexts(s.requests[0])[0]).not.toContain("Request 15:") expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({ status: "completed", summary: "## Objective\n- Recovered",