From bc175f675ca199f964555e9e75b1368391bc2cd4 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:13:07 -0500 Subject: [PATCH] fix(core): size compaction requests from the provider's last count (#59441) Co-authored-by: Kit Langton --- packages/core/src/session/compaction.ts | 66 +++++-- packages/core/test/session-compaction.test.ts | 185 ++++++++++++++++++ 2 files changed, 235 insertions(+), 16 deletions(-) diff --git a/packages/core/src/session/compaction.ts b/packages/core/src/session/compaction.ts index 91a4e0a19b7..917d08819b2 100644 --- a/packages/core/src/session/compaction.ts +++ b/packages/core/src/session/compaction.ts @@ -467,14 +467,20 @@ export const layer = Layer.effect( const headings = SUMMARY_TEMPLATE.split("\n").filter((line) => line.startsWith("##")) const filled = (text: string) => text.split("\n").some((line) => headings.includes(line.trim())) const prepared = yield* prepare(context, messages, budget) + // Size the unchanged request with the number that decided to compact: the last response's count plus an + // estimate of what came after it. Characters overstate a conversation whose old reasoning the provider drops. + // A hook can still make this request larger than that count; the provider then rejects it and it is resent as + // text. An overflow already proved the count low, so it keeps the character estimate. + const measured = trigger.reason === "overflow" ? undefined : measuredPrompt({ ...context, messages }) // Hooks saw the request without the summary prompt, so it is appended here. A reply that ignores the // template gets one reminder before it counts as a failure. - const send = (request: LLMRequest) => + const send = (request: LLMRequest, size?: Estimate) => Effect.gen(function* () { const prompted = fitOutput( LLMRequest.update(request, { messages: [...request.messages, Message.user(prompt)] }), context.model.limit, + size && { ...size, estimated: size.estimated + Token.estimate(prompt) }, ) const reply = yield* stream(context, prompted, prepared.options, background) if (filled(reply.text)) return reply @@ -482,6 +488,7 @@ export const layer = Layer.effect( const nudged = fitOutput( LLMRequest.update(prompted, { messages: [...prompted.messages, Message.user(NUDGE)] }), context.model.limit, + size && { ...size, estimated: size.estimated + Token.estimate(prompt) + Token.estimate(NUDGE) }, ) const retry = yield* stream(context, nudged, prepared.options, background) if (filled(retry.text)) return retry @@ -501,13 +508,16 @@ export const layer = Layer.effect( // which still reuses the cache for that shorter prefix, and as text otherwise. Dropping exchanges to make room // for the tail would leave them in neither the summary nor the tail. Where 15% of the window is under // `RESERVE_MIN`, the ceiling equals `summaryRoom`, so automatic compaction always takes this fallback. - const fit = trigger.reason === "overflow" ? budget : Math.max(budget, summaryRoom(context.model.limit)) + const fit = + trigger.reason === "overflow" + ? budget + : Math.max(budget, summaryRoom(context.model.limit, prepared.request.generation?.maxTokens)) const result = yield* deliver( trigger, prepared, budget - overhead, send, - { fit: fit - overhead, history: withoutTail(prepared.request, keepID) }, + { fit: fit - overhead, history: withoutTail(prepared.request, keepID), measured }, background, ) if (!keepID) return result @@ -569,7 +579,8 @@ export const layer = Layer.effect( /** * Sends the request as-is if it is estimated to fit `shrink.fit`, else `shrink.history` as-is if it fits `target`, - * and otherwise `shrink.history` as text aimed at `target` (see `flattenAndDropOldest`). + * and otherwise `shrink.history` as text aimed at `target` (see `flattenAndDropOldest`). `shrink.measured`, when + * given, sizes the unchanged request until the provider rejects it; after that it is only sent as text. * Limits and estimates can be wrong, so until the provider rejects something as too long, a request that cannot * be made to fit is sent whole: unchanged, or as text once its media has to go. A "too long" rejection means the * estimate ran low, so the next attempts aim at 85%, 70%, 50%, then 35% of the first rejected request's estimate, @@ -583,8 +594,11 @@ export const layer = Layer.effect( trigger: Trigger, prepared: Prepared, target: number, - send: (request: LLMRequest) => Effect.Effect, - shrink: { readonly fit: number; readonly history: LLMRequest } = { fit: target, history: prepared.request }, + send: (request: LLMRequest, size?: Estimate) => Effect.Effect, + shrink: { readonly fit: number; readonly history: LLMRequest; readonly measured?: Estimate } = { + fit: target, + history: prepared.request, + }, background?: Background, ): Effect.fn.Return { const context = trigger.context @@ -597,9 +611,10 @@ export const layer = Layer.effect( let fit = shrink.fit while (true) { + const measured = rejections === 0 ? shrink.measured : undefined const unchanged = asText ? undefined - : estimateRequest(prepared.request) <= fit + : (measured ? total(measured) : estimateRequest(prepared.request)) <= fit ? prepared.request : estimateRequest(shrink.history) <= target ? shrink.history @@ -622,7 +637,8 @@ export const layer = Layer.effect( }) } - const attempt = yield* Effect.result(send(request)) + const size = request === prepared.request ? measured : undefined + const attempt = yield* Effect.result(send(request, size)) if (Result.isSuccess(attempt)) return attempt.success const cause = attempt.failure if (!(cause instanceof AIError)) return yield* Effect.fail(cause) @@ -637,7 +653,9 @@ export const layer = Layer.effect( if (tooLarge || isContextOverflowFailure(cause)) { const step = SHRINK_STEPS[rejections] if (step === undefined) return yield* Effect.fail({ error }) - rejected ??= estimateRequest(request) + rejected ??= size ? total(size) : estimateRequest(request) + // The characters can still say the rejected request fits, so it only goes out again as text. + if (size) asText = true target = Math.floor(rejected * step) fit = target rejections++ @@ -1122,9 +1140,12 @@ const calculateCeiling = (limit: SessionContext.Loaded["model"]["limit"], buffer return window - Math.max(Math.floor(window * RESERVE_SHARE), window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0) } -/** The largest summary request that leaves `RESERVE_MIN` of the window for the summary. Zero for an unknown window. */ -const summaryRoom = (limit: SessionContext.Loaded["model"]["limit"]) => - Math.max(0, (limit.input || limit.context) - RESERVE_MIN) +/** + * The largest summary request that leaves `RESERVE_MIN` of the window for the summary, or less when the request + * cannot generate that much. Zero for an unknown window. + */ +const summaryRoom = (limit: SessionContext.Loaded["model"]["limit"], output = RESERVE_MIN) => + Math.max(0, (limit.input || limit.context) - Math.min(RESERVE_MIN, output)) /** * Another provider's count may describe a native window this model cannot replay. History then holds the original @@ -1137,6 +1158,19 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L message.tokens !== undefined && message.tokens.input + message.tokens.cache.read + message.tokens.cache.write > 0 +/** The prompt size, only while the model, variant and agent that were measured still match; they shape the request. */ +const measuredPrompt = (context: SessionContext.Loaded) => { + const anchor = context.messages.findLast((message) => hasMeasuredPrompt(message, context.model.ref)) + if ( + anchor?.type !== "assistant" || + anchor.model.id !== context.model.ref.id || + anchor.model.variant !== context.model.ref.variant || + anchor.agent !== context.agent.id + ) + return undefined + return estimatePrompt(context) +} + /** * `prepare` sizes the output limit for a request of `budget` tokens, but `deliver` can send a larger one when it * cannot shrink the conversation further. Lower the limit to the room the request being sent leaves in the window, @@ -1146,15 +1180,15 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L * window, so that margin alone would cut a summary near its target to the minimum; there it still gets up to * `SUMMARY_OUTPUT_FLOOR` of the room left without the margin. */ -const fitOutput = (request: LLMRequest, limit: SessionContext.Loaded["model"]["limit"]) => { +const fitOutput = (request: LLMRequest, limit: SessionContext.Loaded["model"]["limit"], size?: Estimate) => { const current = request.generation?.maxTokens if (current === undefined) return request - const estimated = estimateRequest(request) + const input = size ?? { measured: 0, estimated: estimateRequest(request) } const maxTokens = Math.min( current, Math.max( - SessionModelRequest.outputLimit(limit, { measured: 0, estimated }), - Math.min(SUMMARY_OUTPUT_FLOOR, SessionModelRequest.outputLimit(limit, { measured: estimated, estimated: 0 })), + SessionModelRequest.outputLimit(limit, input), + Math.min(SUMMARY_OUTPUT_FLOOR, SessionModelRequest.outputLimit(limit, { measured: total(input), estimated: 0 })), ), ) if (maxTokens === current) return request diff --git a/packages/core/test/session-compaction.test.ts b/packages/core/test/session-compaction.test.ts index 0dd9666d4b1..beaf61beb1d 100644 --- a/packages/core/test/session-compaction.test.ts +++ b/packages/core/test/session-compaction.test.ts @@ -4,6 +4,7 @@ import { LLMClient, LLMEvent, LanguageModel, + Message, ToolDefinition, type LLMRequest, } from "@opencode/ai/effect" @@ -22,6 +23,7 @@ import { SessionHistory } from "@opencode/core/session/history" import { SessionMessage } from "@opencode/core/session/message" import { SessionModelRequest } from "@opencode/core/session/model-request" import { PluginHooks } from "@opencode/core/plugin/hooks" +import { Token } from "@opencode/core/util/token" import { SessionProjector } from "@opencode/core/session/projector" import { SessionRunnerModel } from "@opencode/core/session/runner/model" import { toLLMMessages } from "@opencode/core/session/runner/to-llm-message" @@ -46,6 +48,7 @@ import { testEffect } from "./lib/effect" let requests: LLMRequest[] = [] /** Runs before a request's response streams; tests hold it back to keep a background summary in flight. */ let beforeResponse: Effect.Effect = Effect.void +let reject: ((request: LLMRequest) => boolean) | undefined const model = LanguageModel.make({ id: "summary-model", provider: "test", @@ -64,6 +67,9 @@ const cost = [ const client = Layer.mock(LLMClient.Service)({ stream: (request: LLMRequest) => { requests.push(request) + if (reject?.(request)) { + return Stream.make(LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })) + } return Stream.fromEffect(beforeResponse).pipe( Stream.drain, Stream.concat( @@ -1001,6 +1007,185 @@ const enablePrecompute = Effect.gen(function* () { return compaction }) +const summaryWindow = resolved.limit.context +const summaryBuffer = summaryWindow / 8 +const summaryCeiling = summaryWindow - summaryBuffer +const summaryInput = summaryCeiling * 0.9 // Past precompute's trigger, below the compaction ceiling. +const tokenText = (tokens: number) => "text".repeat(Math.ceil(tokens / Token.estimate("text"))) +const oldReasoning = (input = summaryInput) => ({ + ...measured("Earlier answer", input), + content: [{ type: "reasoning" as const, text: tokenText(summaryWindow * 3) }], +}) +const summaryFixture = Effect.fnUntraced(function* (messages: readonly SessionMessage.Info[]) { + requests = [] + const context = { + ...loaded(yield* insertSession(Session.ID.create()), messages), + tools: { + definitions: [ToolDefinition.make({ name: "read", description: "Read files", inputSchema: { type: "object" } })], + execute: () => Effect.die("Compaction must not execute tools"), + }, + } + const compaction = yield* SessionCompaction.Service + yield* compaction.transform((editor) => editor.configure({ buffer: summaryBuffer, precompute: true })) + return { + context, + compaction, + run: (input = context) => + compaction.compact({ reason: "manual", context: input, inputID: SessionMessage.ID.create() }), + } +}) + +for (const background of [false, true]) { + it.effect(`summary cache prefix uses measured input with old reasoning (background=${background})`, () => + Effect.gen(function* () { + const messages = [ + textUser("Earlier question"), + oldReasoning(), + textUser("Latest question"), + measured("Latest answer", summaryInput), + ] + const fixture = yield* summaryFixture(messages) + const builder = yield* SessionModelRequest.Service + const normal = yield* builder.primary({ + ...fixture.context, + agent: fixture.context.agent.id, + ...SessionModelRequest.baseTranscript({ + ...fixture.context, + agent: fixture.context.agent.info, + messages: background ? messages.slice(0, -1) : messages, + }), + }) + if (background) yield* fixture.compaction.precompute(Effect.succeed(fixture.context)) + expect(yield* fixture.run()).toEqual({ status: "completed" }) + expect(requests).toHaveLength(1) + expect(requests[0]).toMatchObject({ + system: normal.request.system, + tools: normal.request.tools, + messages: [...normal.request.messages, Message.user(SessionCompaction.buildPrompt(false))], + generation: { maxTokens: summaryBuffer }, + }) + }), + ) +} + +it.effect("summary cache prefix still flattens genuinely oversized measured input", () => + Effect.gen(function* () { + const fixture = yield* summaryFixture([ + textUser(tokenText(summaryCeiling)), + measured("Latest answer", summaryWindow + summaryBuffer), + ]) + expect(yield* fixture.run()).toEqual({ status: "completed" }) + expect(requests).toHaveLength(1) + expect(requests[0].tools).toEqual([]) + expect(requests[0].messages).toHaveLength(2) + expect(JSON.stringify(requests[0].messages[0])).toContain("characters omitted") + }), +) + +it.effect("summary measured recovery shrinks from the rejected measured size", () => + Effect.gen(function* () { + const text = tokenText(summaryCeiling / 2) + const fixture = yield* summaryFixture([ + textUser(text), + oldReasoning(summaryCeiling / 2), + textUser(text), + measured("Latest answer", summaryCeiling), + ]) + // Reject until the original exchange is actually shortened, without duplicating a token estimator in the mock. + reject = (request) => + request.messages.some((message) => + message.content.some((part) => part.type === "text" && part.text.includes(text)), + ) + yield* Effect.addFinalizer(() => + Effect.sync(() => { + reject = undefined + }), + ) + expect(yield* fixture.run()).toEqual({ status: "completed" }) + // The retained exchange fits the 85% and 70% targets; 50% of the measured input forces truncation. + expect(requests).toHaveLength(4) + expect(JSON.stringify(requests.at(-1)?.messages)).toContain("characters omitted") + }), +) + +it.effect("summary cache prefix survives hooks that shape every request alike", () => + Effect.gen(function* () { + const fixture = yield* summaryFixture([ + textUser("Earlier question"), + oldReasoning(), + textUser("Latest question"), + measured("Latest answer", summaryInput), + ]) + // Built-in tool plugins register one hook for both normal and compaction requests. + const hooks = yield* PluginHooks.Service + const describe = (event: { tools: Record }) => + Effect.sync(() => { + if (event.tools.read) event.tools.read.description = "Read files from the project" + }) + yield* hooks.register("session", "context", describe) + yield* hooks.register("session", "compaction", describe) + expect(yield* fixture.run()).toEqual({ status: "completed" }) + expect(requests).toHaveLength(1) + expect(requests[0].tools.map((tool) => tool.description)).toEqual(["Read files from the project"]) + }), +) + +it.effect("summary resends as text when a hook makes the request larger than the measured count", () => + Effect.gen(function* () { + const fixture = yield* summaryFixture([ + textUser("Earlier question"), + oldReasoning(), + textUser("Latest question"), + measured("Latest answer", summaryInput), + ]) + const hooks = yield* PluginHooks.Service + yield* hooks.register("session", "compaction", (event) => + Effect.sync(() => { + event.system.push({ type: "text", text: "Plugin guidance" }) + }), + ) + // The provider rejects the unchanged request, which the measured count said would fit. + reject = (request) => request.tools.length > 0 + yield* Effect.addFinalizer(() => + Effect.sync(() => { + reject = undefined + }), + ) + expect(yield* fixture.run()).toEqual({ status: "completed" }) + expect(requests).toHaveLength(2) + expect(requests[0].tools).toEqual(fixture.context.tools.definitions) + expect(requests[1].tools).toEqual([]) + }), +) + +it.effect("summary measured recovery reuses usage only for the matching agent", () => + Effect.gen(function* () { + const fixture = yield* summaryFixture([textUser("Earlier question"), oldReasoning()]) + const agent = Agent.ID.make("other") + const context = { + ...fixture.context, + agent: { + id: agent, + info: { ...Agent.Info.default(agent), system: tokenText(summaryWindow - summaryBuffer / 5) }, + }, + } + expect(yield* fixture.run(context)).toEqual({ status: "completed" }) + expect(requests).toHaveLength(1) + expect(requests[0].generation?.maxTokens).toBeLessThanOrEqual(summaryBuffer / 5) + // Switching back restores the measured prefix; discarding usage for every agent is not sufficient. + expect(yield* fixture.run()).toEqual({ status: "completed" }) + expect(requests).toHaveLength(2) + expect(requests[1]).toMatchObject({ + tools: fixture.context.tools.definitions, + messages: [ + ...toLLMMessages(fixture.context.messages, resolved.ref), + Message.user(SessionCompaction.buildPrompt(false)), + ], + generation: { maxTokens: summaryBuffer }, + }) + }), +) + it.effect("background summaries stay off unless enabled", () => Effect.gen(function* () { requests = []