From fd6589239e6cc99155ab6a793a3539c28cd0415b Mon Sep 17 00:00:00 2001 From: Filip <34747899+neriousy@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:02:01 +0200 Subject: [PATCH] fix(core): summarize the whole conversation during compaction (#57091) --- packages/core/src/session/compaction.ts | 108 +++++++++++------ packages/core/test/session-compaction.test.ts | 111 ++++++++++++++++-- packages/core/test/session-runner.test.ts | 11 +- services/www/src/docs/content/compaction.mdx | 18 +-- 4 files changed, 189 insertions(+), 59 deletions(-) diff --git a/packages/core/src/session/compaction.ts b/packages/core/src/session/compaction.ts index 3cb52790295..3fb4902279e 100644 --- a/packages/core/src/session/compaction.ts +++ b/packages/core/src/session/compaction.ts @@ -40,7 +40,7 @@ import { SessionUsage } from "./usage.js" export type Settings = { auto: boolean - /** Tokens kept free below the model's limits before compacting. Unset keeps 10% free. */ + /** Tokens kept free below the model's limits before compacting. Unset keeps 15%, and at least 16k, free. */ buffer?: number /** Tokens of recent conversation kept verbatim beside the summary. */ keep: number @@ -95,6 +95,12 @@ const NOTHING_TO_COMPACT: Failure = { error: { type: "compaction.unavailable", m const SHRINK_STEPS = [0.7, 0.5, 0.35] // The least of the window kept free for the last reply before compaction and for the summary itself. const RESERVE_MIN = 16_000 +/** + * Share of the window kept free before automatic compaction. Above `RESERVE_MIN`, the margin between it and + * `summaryRoom` absorbs a large step past the trigger, so the whole conversation still goes out unchanged and reuses + * the cache. Single steps grew by up to ~22k tokens at p99 in real sessions. + */ +const RESERVE_SHARE = 0.15 /** A common window size, assumed for the compaction request when the model's window is unknown. */ const UNKNOWN_WINDOW = 200_000 const TOOL_OUTPUT_MAX_CHARS = 1_250 @@ -252,11 +258,12 @@ export const layer = Layer.effect( /** * first: [E1 … E8][E9 E10] → [S1 + "E9 E10"] - * again: [S1 + "E9 E10"][E11 … E18][E19] → [S2 + "E19"] S2 covers S1, E9 E10, and E11–E18 + * again: [S1 + "E9 E10"][E11 … E18][E19] → [S2 + "E19"] S2 covers the whole current conversation * - * The model is sent the older part, then the summary prompt: + * The model is sent the whole conversation, including the tail kept beside the summary, then the summary prompt. + * Keeping the normal request's prefix intact lets the summary reuse its prompt cache: * - * [system][S1 + "E9 E10"][E11 … E18][prompt] plus one nudge if the reply skips the template + * [system][S1 + "E9 E10"][E11 … E18][E19][prompt] plus one nudge if the reply skips the template * * Later requests open with the checkpoint as one user message: * @@ -275,14 +282,14 @@ export const layer = Layer.effect( keep: number, ): Effect.fn.Return { const context = trigger.context - const split = splitConversation(context.messages, keep) - if (!split) return yield* Effect.fail(NOTHING_TO_COMPACT) + const recent = recentConversation(context.messages, keep) + if (recent === undefined) return yield* Effect.fail(NOTHING_TO_COMPACT) const previous = previousCompaction(context.messages) const prompt = buildPrompt(previous !== undefined, previous?.summary.includes(LEGACY_HEADING) ?? false) const headings = SUMMARY_TEMPLATE.split("\n").filter((line) => line.startsWith("##")) const filled = (text: string) => text.split("\n").some((line) => headings.includes(line.trim())) - const prepared = yield* prepare(context, split.older, budget) + const prepared = yield* prepare(context, context.messages, budget) // Hooks saw the request without the summary prompt, so it is appended here. A reply that ignores the // template gets one reminder before it counts as a failure. @@ -293,14 +300,14 @@ export const layer = Layer.effect( context.model.limit, ) const reply = yield* stream(context, prompted, prepared.options) - if (filled(reply.text)) return { ...reply, recent: split.recent } + if (filled(reply.text)) return { ...reply, recent: recent.text } const nudged = fitOutput( LLMRequest.update(prompted, { messages: [...prompted.messages, Message.user(NUDGE)] }), context.model.limit, ) const retry = yield* stream(context, nudged, prepared.options) - if (filled(retry.text)) return { ...retry, recent: split.recent } + if (filled(retry.text)) return { ...retry, recent: recent.text } return yield* Effect.fail({ error: { type: "compaction.failed", @@ -312,7 +319,16 @@ export const layer = Layer.effect( }) const overhead = Token.estimate(prompt) + Token.estimate(NUDGE) - return yield* deliver(trigger, prepared, split.recent, budget - overhead, send) + // Automatic compaction starts past the ceiling, so the whole conversation is sent unchanged while the window can + // still fit the summary. Below that, the history before the tail is sent instead, as before: unchanged when it fits, + // which still reuses the cache for that shorter prefix, and as text otherwise. Dropping exchanges to make room + // for the tail would leave them in neither the summary nor the tail. Where 15% of the window is under + // `RESERVE_MIN`, the ceiling equals `summaryRoom`, so automatic compaction always takes this fallback. + const fit = trigger.reason === "overflow" ? budget : Math.max(budget, summaryRoom(context.model.limit)) + return yield* deliver(trigger, prepared, recent.text, budget - overhead, send, { + fit: fit - overhead, + history: withoutTail(prepared.request, recent.from), + }) }) /** @@ -378,7 +394,8 @@ export const layer = Layer.effect( }) /** - * Sends the request as-is if it is estimated to fit `target`, and as text otherwise (see `flattenAndDropOldest`). + * Sends the request as-is if it is estimated to fit `shrink.fit`, else `shrink.history` as-is if it fits `target`, + * and otherwise `shrink.history` as text aimed at `target` (see `flattenAndDropOldest`). * Limits and estimates can be wrong, so until the provider rejects something as too long, a request that cannot * be made to fit is sent whole: unchanged, or as text once its media has to go. A "too long" rejection means the * estimate ran low, so the next attempts aim at 70%, 50%, then 35% of the first rejected request's estimate, and a @@ -393,6 +410,7 @@ export const layer = Layer.effect( recent: string, target: number, send: (request: LLMRequest) => Effect.Effect, + shrink: { readonly fit: number; readonly history: LLMRequest } = { fit: target, history: prepared.request }, ): Effect.fn.Return { const context = trigger.context // The runner opened the manual compaction's message when it delivered the `/compact` item. @@ -405,17 +423,24 @@ export const layer = Layer.effect( let rejections = 0 let rejected: number | undefined let asText = false + let fit = shrink.fit while (true) { - const fits = !asText && estimateRequest(prepared.request) <= target - const request = fits - ? prepared.request - : (flattenAndDropOldest(prepared.request, context, target) ?? - (rejections > 0 - ? undefined - : asText - ? flattenAndDropOldest(prepared.request, context, Number.POSITIVE_INFINITY) - : prepared.request)) + const unchanged = asText + ? undefined + : estimateRequest(prepared.request) <= fit + ? prepared.request + : estimateRequest(shrink.history) <= target + ? shrink.history + : undefined + const request = + unchanged ?? + flattenAndDropOldest(shrink.history, context, target) ?? + (rejections > 0 + ? undefined + : asText + ? flattenAndDropOldest(shrink.history, context, Number.POSITIVE_INFINITY) + : shrink.history) if (!request) { return yield* Effect.fail({ error: { @@ -433,7 +458,7 @@ export const layer = Layer.effect( const error = toSessionError(cause) const tooLarge = cause.reason._tag === "InvalidRequest" && cause.reason.classification === "payload-too-large" - if (tooLarge && request === prepared.request) { + if (tooLarge && (request === prepared.request || request === shrink.history)) { asText = true continue } @@ -443,6 +468,7 @@ export const layer = Layer.effect( if (step === undefined) return yield* Effect.fail({ error }) rejected ??= estimateRequest(request) target = Math.floor(rejected * step) + fit = target rejections++ continue } @@ -702,21 +728,25 @@ const transcript = (context: SessionContext.Loaded, messages: ReadonlyArray, keep: number) => { - const entries = messages.flatMap((message, index) => { +const recentConversation = (messages: ReadonlyArray, keep: number) => { + const entries = messages.flatMap((message) => { const text = messageToText(message) - return text ? [{ message, text, index }] : [] + return text ? [{ message, text }] : [] }) if (entries.length === 0) return undefined const recent = entries.slice(recentStart(entries, keep, previousCompaction(messages))) - return { - older: messages.slice(0, recent[0]?.index ?? messages.length), - recent: recent.map((entry) => entry.text).join("\n\n"), - } + return { text: recent.map((entry) => entry.text).join("\n\n"), from: recent[0]?.message.id } +} + +/** The request without the retained tail, which starts at the message with ID `from`. */ +const withoutTail = (request: LLMRequest, from: SessionMessage.ID | undefined) => { + const index = from === undefined ? -1 : request.messages.findIndex((message) => message.id === from) + if (index < 0) return request + return LLMRequest.update(request, { messages: request.messages.slice(0, index) }) } const recentStart = ( @@ -734,12 +764,12 @@ const recentStart = ( const userBoundary = entries.findLastIndex((entry, index) => index <= dropped && entry.message.type === "user") if (userBoundary > 0) return userBoundary - // Everything fits. Keep only the latest exchange so there is an older part left to summarize. + // Everything fits. Keep only the latest exchange rather than duplicating the whole conversation beside the summary. const latestUser = entries.findLastIndex((entry) => entry.message.type === "user") if (latestUser > 0) return latestUser - // One exchange, nothing older. Summarize it all and keep nothing, unless a previous summary already - // kept recent text, in which case keep everything and summarize only the summary before it. + // One exchange, nothing older. Keep nothing, unless a previous checkpoint kept recent text: the new summary replaces + // that text, so this exchange stays verbatim in its place. return previous?.recent ? 0 : entries.length } @@ -904,9 +934,9 @@ export const estimatePrompt = (context: SessionContext.Loaded) => { } /** - * The largest request the model takes while leaving room for its reply: 10% of the window, or `RESERVE_MIN` when that - * is more. The summary request is capped at the same size, so its output limit is whatever the reserve leaves. A window - * too small to give up `RESERVE_MIN` keeps 10%. + * The largest request the model takes while leaving room for its reply: `RESERVE_SHARE` of the window, or + * `RESERVE_MIN` when that is more. A summary request that must be shrunk is aimed at the same size. A window too small + * to give up `RESERVE_MIN` keeps `RESERVE_SHARE`. */ const calculateCeiling = (limit: SessionContext.Loaded["model"]["limit"], buffer: number | undefined) => { // Unknown limits are reported as 0. An unknown input limit falls back to the context window; with no window at @@ -914,9 +944,13 @@ const calculateCeiling = (limit: SessionContext.Loaded["model"]["limit"], buffer const window = limit.input || limit.context if (window <= 0) return Number.POSITIVE_INFINITY if (buffer !== undefined) return window - buffer - return window - Math.max(Math.floor(window * 0.1), window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0) + return window - Math.max(Math.floor(window * RESERVE_SHARE), window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0) } +/** The largest summary request that leaves `RESERVE_MIN` of the window for the summary. Zero for an unknown window. */ +const summaryRoom = (limit: SessionContext.Loaded["model"]["limit"]) => + Math.max(0, (limit.input || limit.context) - RESERVE_MIN) + /** * Another provider's count may describe a native window this model cannot replay. History then holds the original * messages that window stood for, which the count leaves out, so only the current provider's counts are trusted. diff --git a/packages/core/test/session-compaction.test.ts b/packages/core/test/session-compaction.test.ts index 1361a0bb7f4..24d21a6b585 100644 --- a/packages/core/test/session-compaction.test.ts +++ b/packages/core/test/session-compaction.test.ts @@ -205,19 +205,19 @@ it.effect("auto compaction estimates current content against the buffered prompt const due = (context: SessionContext.Loaded) => compaction.compact({ reason: "auto", context }).pipe(Effect.map((outcome) => outcome.status !== "skipped")) - // 90% of the input limit, which takes precedence over the context window. + // 85% of the input limit, which takes precedence over the context window. const inputLimited = { context: 400_000, input: 272_000, output: 128_000 } - expect(yield* due(input(244_799, inputLimited))).toBe(false) - expect(yield* due(input(244_800, inputLimited))).toBe(true) + expect(yield* due(input(231_199, inputLimited))).toBe(false) + expect(yield* due(input(231_200, inputLimited))).toBe(true) const native = (tokens: number, limit: { context: number; input?: number; output: number } = inputLimited) => { const selected = input(tokens, limit) return { ...selected, model: { ...selected.model, compaction: { type: "native" as const } } } } - expect(yield* due(native(244_799))).toBe(false) - expect(yield* due(native(244_800))).toBe(true) + expect(yield* due(native(231_199))).toBe(false) + expect(yield* due(native(231_200))).toBe(true) expect(yield* due(native(1_000_000, { context: 0, input: undefined, output: 0 }))).toBe(false) - // The summary's 16k output limit is more than 10% of a 100k window, so it sets the ceiling. + // The summary's 16k output limit is more than 15% of a 100k window, so it sets the ceiling. const contextLimited = { context: 100_000, output: 10_000 } expect(yield* due(input(83_999, contextLimited))).toBe(false) expect(yield* due(input(84_000, contextLimited))).toBe(true) @@ -460,7 +460,7 @@ it.effect("manual compaction summarizes short context instead of no-op", () => "x-opencode-session": parentID, "x-opencode-client": "opencode", }) - expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 20_000 })) + expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 30_000 })) expect(JSON.stringify(requests[0]?.messages)).toContain("Manual compaction should include this short conversation.") expect(JSON.stringify(requests[0]?.messages)).toContain("Use Effect services and generators.") expect(JSON.stringify(requests[0]?.messages)).toContain("User shell pwd completed: /project") @@ -498,6 +498,97 @@ it.effect("manual compaction summarizes short context instead of no-op", () => }), ) +it.effect("summary updates include the retained tail in compaction hooks and the model request", () => + Effect.gen(function* () { + requests = [] + const session = yield* insertSession(Session.ID.make("ses_compaction_full_update")) + const messages = [ + SessionMessage.CompactionCompleted.make({ + id: SessionMessage.ID.create(), + type: "compaction", + status: "completed", + reason: "auto", + summary: "## Objective\n- Previous objective", + recent: "[User]: Previous exact request", + time: { created: DateTime.makeUnsafe(0) }, + }), + SessionMessage.User.make({ + id: SessionMessage.ID.create(), + type: "user", + text: "The latest request supersedes the previous objective.", + time: { created: DateTime.makeUnsafe(1) }, + }), + ] + const context = loaded(session, messages) + const expected = SessionModelRequest.baseTranscript({ ...context, agent: context.agent.info }).messages + const hooks = yield* PluginHooks.Service + yield* hooks.register("session", "compaction", (event) => + Effect.sync(() => { + expect(event.messages).toEqual(expected) + }), + ) + + expect(yield* compactManually(session, messages)).toEqual({ status: "completed" }) + expect(requests).toHaveLength(1) + expect(requests[0].messages.slice(0, -1)).toEqual(expected) + expect(requests[0].messages.at(-1)?.content).toEqual([{ type: "text", text: SessionCompaction.buildPrompt(true) }]) + const store = yield* SessionStore.Service + expect(yield* store.context(session.id)).toMatchObject([ + { type: "compaction", recent: "[User]: The latest request supersedes the previous objective." }, + ]) + }), +) + +it.effect("summary compaction past the ceiling keeps the conversation unchanged while the window fits", () => + Effect.gen(function* () { + requests = [] + const session = yield* insertSession(Session.ID.make("ses_compaction_near_window")) + // About 182k estimated tokens: past the 170k ceiling, within the 184k left after reserving room for the summary. + const messages = ["first", "second"].map((label) => + SessionMessage.User.make({ + id: SessionMessage.ID.create(), + type: "user", + text: `${label} ${"x".repeat(364_000)}`, + time: { created: DateTime.makeUnsafe(0) }, + }), + ) + + expect(yield* compactManually(session, messages)).toEqual({ status: "completed" }) + expect(requests).toHaveLength(1) + expect(requests[0].messages).toHaveLength(3) + expect(requests[0].messages[0].content).toEqual([{ type: "text", text: messages[0].text }]) + expect(requests[0].generation?.maxTokens).toBeLessThan(20_000) + }), +) + +it.effect( + "summary compaction sends the history before the tail unchanged when the whole conversation does not fit", + () => + Effect.gen(function* () { + requests = [] + const session = yield* insertSession(Session.ID.make("ses_compaction_older_fallback")) + // About 190k estimated tokens in total, past the 184k left for the summary. The 160k before the retained + // 30k latest exchange fits the 170k ceiling. + const messages = [ + ["older", 640_000], + ["latest", 120_000], + ].map(([label, size]) => + SessionMessage.User.make({ + id: SessionMessage.ID.create(), + type: "user", + text: `${label} ${"x".repeat(Number(size))}`, + time: { created: DateTime.makeUnsafe(0) }, + }), + ) + + expect(yield* compactManually(session, messages)).toEqual({ status: "completed" }) + expect(requests).toHaveLength(1) + expect(requests[0].messages).toHaveLength(2) + expect(requests[0].messages[0].content).toEqual([{ type: "text", text: messages[0].text }]) + expect(requests[0].messages[1].content).toEqual([{ type: "text", text: SessionCompaction.buildPrompt(false) }]) + }), +) + it.effect("compaction lowers the output limit when it sends a request it could not shrink", () => Effect.gen(function* () { requests = [] @@ -522,19 +613,19 @@ it.effect("compaction keeps the output limit for a request that fits its target" Effect.gen(function* () { requests = [] const session = yield* insertSession(Session.ID.make("ses_fitting_compaction")) - // About 170k estimated tokens, under the 180k target, leaving 30k of the 200k window. + // About 160k estimated tokens, under the 170k target, leaving 40k of the 200k window. const messages = [ SessionMessage.User.make({ id: SessionMessage.ID.create(), type: "user", - text: "x".repeat(680_000), + text: "x".repeat(640_000), time: { created: DateTime.makeUnsafe(0) }, }), ] expect(yield* compactManually(session, messages)).toEqual({ status: "completed" }) expect(requests).toHaveLength(1) - expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 20_000 })) + expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 30_000 })) }), ) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 431e5148184..4c8f566ea01 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -2557,7 +2557,7 @@ describe("SessionRunnerLLM", () => { role: "user", content: [{ type: "text", text: SessionCompaction.buildPrompt(false) }], }) - expect(userTexts(compact)).not.toContain("Retained recent request") + expect(userTexts(compact).includes("Retained recent request")).toBe(reason === "auto") for (const field of [ "model", "system", @@ -3008,7 +3008,9 @@ describe("SessionRunnerLLM", () => { expect(userTexts(s.requests[0])[0]).toContain("older exchanges omitted") expect(userTexts(s.requests[0])[0]).not.toContain("Request 0:") expect(userTexts(s.requests[0])[0]).not.toContain("Request 1:") + // The latest exchange is the retained tail, so shrinking drops only exchanges before it. expect(userTexts(s.requests[0])[0]).toContain("Request 2:") + expect(userTexts(s.requests[0])[0]).not.toContain("Request 3:") expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({ status: "completed", summary: "## Objective\n- Recovered", @@ -3102,6 +3104,7 @@ describe("SessionRunnerLLM", () => { yield* s.runPrompt("Recent exact request ".repeat(180)) expect(s.requests).toHaveLength(2) + expect(JSON.stringify(s.requests[0].messages)).toContain("Recent exact request ".repeat(180)) expect(userTexts(s.requests[0]).at(-1)).toContain("## Objective") expect(userTexts(s.requests[1])).toHaveLength(1) expect(userTexts(s.requests[1])[0]).toContain("\n## Objective\n- Preserve the task\n") @@ -3483,13 +3486,13 @@ describe("SessionRunnerLLM", () => { expect(s.requests).toHaveLength(2) expect(userTexts(s.requests[0]).at(-1)).toContain("## Objective") - // The summary may use the whole 20k reserve of a 200k window. - expect(s.requests[0]?.generation?.maxTokens).toBe(20_000) + // The summary may use the whole 30k reserve of a 200k window. + expect(s.requests[0]?.generation?.maxTokens).toBe(30_000) expect(s.requests[1]?.generation?.maxTokens).toBe(32_000) }) scenario("keeps the summary its room on a small window", function* (s) { - // 90% of 64k would leave 6.4k for the summary, so the 16k reserve sets the ceiling at 48k instead. + // 85% of 64k would leave 9.6k for the summary, so the 16k reserve sets the ceiling at 48k instead. s.currentModel = smallWindowModel yield* s.llm.push(TestLLM.textWithUsage("Earlier answer", "text-small-first", 59_000)) yield* s.runPrompt("Earlier question") diff --git a/services/www/src/docs/content/compaction.mdx b/services/www/src/docs/content/compaction.mdx index a22c87e5080..e76025e7edd 100644 --- a/services/www/src/docs/content/compaction.mdx +++ b/services/www/src/docs/content/compaction.mdx @@ -7,18 +7,20 @@ summary, so the session can keep going past the model's context limit. ## How it works -When a session approaches the model's context limit, OpenCode summarizes -everything except the most recent conversation, about 15,000 tokens by default. -Once the summary is ready, it is placed in front of that recent conversation and -the session continues from there. +When a session approaches the model's context limit, OpenCode summarizes the +entire current conversation. It also keeps the most recent conversation, about +15,000 tokens by default, beside the summary so exact recent details remain +available. ```text before [ system prompt ][ older conversation ............ ][ recent 15k ][ pending work ] -after [ system prompt ][ summary ][ recent 15k ][ pending work ] +request [ system prompt ][ older conversation ............ ][ recent 15k ][ summary instructions ] +after [ system prompt ][ summary of older + recent ][ recent 15k ][ pending work ] ``` -The model sees the summary as past conversation. Later compactions update the -same summary rather than starting over. +Sending the full conversation lets the summary request reuse the normal request's +prompt cache when the provider supports it. Later compactions update the existing +summary using the conversation since that checkpoint. ## Start @@ -82,7 +84,7 @@ Add `compaction` to any [OpenCode configuration file](/config): | --- | ---: | --- | | `auto` | `true` | Compact automatically near the context limit, and recover once when a provider rejects a request as too long. Manual compaction always works. | | `keep.tokens` | `15000` | Approximate recent conversation kept beside the summary. Larger values preserve more detail but leave less room for new work. | -| `buffer` | 10% of the limit | Tokens to keep free below the model's limit. Larger values start automatic compaction earlier. | +| `buffer` | 15% of the limit, at least 16k | Tokens to keep free below the model's limit. Larger values start automatic compaction earlier. | `keep.tokens` and `buffer` accept non-negative integers.