From a0c6df8fdedbb09eac46594c54298846a3b27bc0 Mon Sep 17 00:00:00 2001 From: Filip <34747899+neriousy@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:28:39 +0100 Subject: [PATCH] fix(core): start background compaction a fixed lead before the threshold (#60049) --- packages/core/src/session/compaction.ts | 67 ++++++--- packages/core/test/session-compaction.test.ts | 134 ++++++++++++++++-- services/www/src/docs/content/compaction.mdx | 16 ++- 3 files changed, 181 insertions(+), 36 deletions(-) diff --git a/packages/core/src/session/compaction.ts b/packages/core/src/session/compaction.ts index 41ccb1bd087..6751a7de6af 100644 --- a/packages/core/src/session/compaction.ts +++ b/packages/core/src/session/compaction.ts @@ -18,7 +18,7 @@ import type { StreamOptions } from "@opencode/ai/route" import type { SessionCompactionResult } from "@opencode/plugin/effect/session" import type { SessionError } from "@opencode/schema/session-error" import { makeLocationNode } from "@opencode/util/effect/app-node" -import { Context, Effect, Exit, Fiber, Layer, Result, Scope, Stream } from "effect" +import { Clock, Context, DateTime, Effect, Exit, Fiber, Layer, Result, Scope, Stream } from "effect" import { Agent } from "../agent.js" import { Bus } from "../bus.js" import { Database } from "../database/database.js" @@ -150,16 +150,23 @@ const TRANSCRIPT_LEVELS = [0, OUTPUTS_CLIPPED, OUTPUTS_REMOVED, USER_ONLY, OMITT const TRANSCRIPT_CLIP = 2_000 const IMAGE_TOKEN_ESTIMATE = 1_500 const PDF_TOKEN_ESTIMATE = 2_000 -/** Share of the compaction ceiling at which a background summary starts. */ -const PRECOMPUTE_RATIO = 0.85 -/** Measured prompt tokens below which a background summary is not worth its cost. */ -const PRECOMPUTE_MIN_TOKENS = 64_000 /** - * The most text a swapped-in summary keeps verbatim beside it: twice `keep`, at least 30k, at most a quarter of the - * budget. A longer tail would leave the context near the threshold, so compaction summarizes from scratch instead. + * Tokens before the ceiling at which a background summary starts. Writing one takes 30–60 s, and in that time the + * conversation grows by a few thousand tokens at the median but by 25k or more one time in twenty, when large file + * reads arrive together. How far ahead to start depends on that growth, not on the window, so the lead is fixed. */ -const PRECOMPUTED_TAIL_MIN = 30_000 -const PRECOMPUTED_TAIL_SHARE = 0.25 +const PRECOMPUTE_LEAD = 60_000 +/** Prompt caches expire this long after their last use unless the model sets `cache.ttl`. */ +const CACHE_LIFETIME = 5 * 60_000 +/** A background summary starts only while this much of the latest response's cache lifetime is left. */ +const CACHE_MARGIN = 30_000 +/** + * The most text a swapped-in summary keeps verbatim beside it, and the tail at which a held summary is replaced, as a + * share of the budget. Tails are usually far shorter; the limit is there for bursts of large tool inputs or pasted + * text. At 40%, a swapped-in context stays under about two thirds of the budget even with Claude's tokenizer, so the + * next cycle keeps room; a longer tail makes compaction summarize from scratch instead. + */ +const PRECOMPUTED_TAIL_SHARE = 0.4 /** Failed background summaries after which no new one starts until the next compaction. */ const PRECOMPUTE_MAX_FAILURES = 2 @@ -329,11 +336,14 @@ export const layer = Layer.effect( }) /** - * Once the context passes `PRECOMPUTE_RATIO` of the ceiling, summarizes everything before the latest response in - * the background. That is the prefix the provider just processed, so the summary request reads it from the prompt - * cache. The summary is held, not applied: swapping it in now would rewrite the prefix and throw away the cache - * for every step until compaction is actually due. One summary is held per Session until compaction takes it, or - * until it fails or falls too far behind, when a newer response replaces it. + * Once the context is within `PRECOMPUTE_LEAD` of the ceiling, summarizes everything before the latest response + * in the background. That is the prefix the provider just processed, so the summary request reads it from the + * prompt cache. The summary is held, not applied: swapping it in now would rewrite the prefix and throw away the + * cache for every step until compaction is actually due. One summary is held per Session until compaction takes + * it, or until it fails or falls too far behind, when a newer response replaces it. + * + * On a small window the lead is at most half the room the cycle started with, so the summary still covers more + * than the previous one. */ const precompute = Effect.fn("SessionCompaction.precompute")(function* ( load: Effect.Effect, @@ -347,12 +357,13 @@ export const layer = Layer.effect( if (!Number.isFinite(ceiling)) return const estimate = given ?? estimatePrompt(context) if (due(context, ceiling, estimate)) return - if (total(estimate) < ceiling * PRECOMPUTE_RATIO) return + const first = context.messages.find((message) => hasMeasuredPrompt(message, context.model.ref)) + const cycleStart = first?.type === "assistant" && first.tokens ? promptTokens(first.tokens) : 0 + if (total(estimate) < ceiling - Math.min(PRECOMPUTE_LEAD, (ceiling - cycleStart) / 2)) return const anchorIndex = context.messages.findLastIndex((message) => hasMeasuredPrompt(message, context.model.ref)) const anchor = context.messages[anchorIndex] if (anchorIndex <= 0 || anchor?.type !== "assistant" || !anchor.tokens) return - if (anchor.tokens.input + anchor.tokens.cache.read + anchor.tokens.cache.write < PRECOMPUTE_MIN_TOKENS) return const sessionID = context.session.id const existing = precomputed.get(sessionID) @@ -369,6 +380,10 @@ export const layer = Layer.effect( } // After a model switch nothing is cached for the new model yet; wait until its first response has written it. if (anchor.model.id !== context.model.ref.id || anchor.model.variant !== context.model.ref.variant) return + // The same once the latest response is older than the cache lifetime, as when the user comes back from a + // break: the summary would write the whole prefix again, at the same time as the next request. + const age = (yield* Clock.currentTimeMillis) - DateTime.toEpochMillis(anchor.time.created) + if (age > cacheLifetime(context.model.cache?.ttl) - CACHE_MARGIN) return const model = yield* selectModel(context, settings.buffer, false) const usage: SessionUsage.Recorded[] = [] @@ -381,7 +396,7 @@ export const layer = Layer.effect( keepID: anchor.id, model: context.model.ref, initial: Instructions.join(context.initial), - tailLimit: tailLimit(settings.keep, model.cap), + tailLimit: tailLimit(model.cap), failures, usage, fiber, @@ -406,7 +421,7 @@ export const layer = Layer.effect( trigger.reason !== "overflow" && context.model.compaction?.type !== "native" && index >= 0 && - tailTokens(context, index) <= tailLimit(settings.keep, budget) + tailTokens(context, index) <= tailLimit(budget) if (!usable) { yield* Fiber.interrupt(held.fiber) return undefined @@ -961,8 +976,18 @@ const heldIndex = (held: Precomputed, context: SessionContext.Loaded) => { const tailTokens = (context: SessionContext.Loaded, index: number) => Token.estimate(context.messages.slice(index).map(SessionRecentText.messageToText).filter(Boolean).join("\n\n")) -const tailLimit = (keep: number, budget: number) => - Math.min(Math.max(keep * 2, PRECOMPUTED_TAIL_MIN), Math.floor(budget * PRECOMPUTED_TAIL_SHARE)) +const tailLimit = (budget: number) => Math.floor(budget * PRECOMPUTED_TAIL_SHARE) + +const promptTokens = (tokens: { input: number; cache: { read: number; write: number } }) => + tokens.input + tokens.cache.read + tokens.cache.write + +const DURATION_UNITS = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 } + +/** Milliseconds a prompt cache entry lives, from a `cache.ttl` such as "5m" or "1h". */ +const cacheLifetime = (ttl: string | undefined) => { + const match = ttl?.match(/^(\d+)(ms|s|m|h|d)$/) + return match ? Number(match[1]) * DURATION_UNITS[match[2] as keyof typeof DURATION_UNITS] : CACHE_LIFETIME +} /** History loads from the latest completed compaction, so a previous one is always the first message. */ const previousCompaction = (messages: ReadonlyArray) => { @@ -1166,7 +1191,7 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L message.model.providerID === model.providerID && !message.error && message.tokens !== undefined && - message.tokens.input + message.tokens.cache.read + message.tokens.cache.write > 0 + promptTokens(message.tokens) > 0 /** The prompt size, only while the model, variant and agent that were measured still match; they shape the request. */ const measuredPrompt = (context: SessionContext.Loaded) => { diff --git a/packages/core/test/session-compaction.test.ts b/packages/core/test/session-compaction.test.ts index b9d7aca6290..14b1ab94a88 100644 --- a/packages/core/test/session-compaction.test.ts +++ b/packages/core/test/session-compaction.test.ts @@ -44,6 +44,7 @@ import { Money } from "@opencode/schema/money" import { Skill } from "@opencode/schema/skill" import { Shell } from "@opencode/schema/shell" import { DateTime, Deferred, Effect, Fiber, Layer, Schema, Stream } from "effect" +import { TestClock } from "effect/testing" import { asc, eq, inArray, sql } from "drizzle-orm" import { testEffect } from "./lib/effect" @@ -1000,7 +1001,7 @@ it.effect("forked session compaction reuses the fork root prompt cache key", () }), ) -const measured = (text: string, input: number) => +const measured = (text: string, input: number, created = 0) => Schema.decodeUnknownSync(SessionMessage.Assistant)({ id: SessionMessage.ID.create(), type: "assistant", @@ -1008,12 +1009,12 @@ const measured = (text: string, input: number) => model: { id: "summary-model", providerID: "test" }, content: [{ type: "text", text }], tokens: { input, output: 100, reasoning: 0, cache: { read: 0, write: 0 } }, - time: { created: 0, completed: 1 }, + time: { created, completed: created + 1 }, }) /** - * A 200k window compacts at 170k and starts a background summary at 85% of that, 144.5k. `near` ends past that - * point; `due` continues it past the threshold. + * A 200k window compacts at 170k. The first response measured 80k, so a background summary starts half that room, + * 45k, before the threshold, at 125k. `near` ends past that point; `due` continues it past the threshold. */ const precomputeHistory = () => { const near = [textUser("First question"), measured("First answer", 80_000), textUser("Second question")] @@ -1061,7 +1062,8 @@ for (const background of [false, true]) { Effect.gen(function* () { const messages = [ textUser("Earlier question"), - oldReasoning(), + // The cycle starts low, so the latest response is past the point where a background summary starts. + oldReasoning(summaryInput / 4), textUser("Latest question"), measured("Latest answer", summaryInput), ] @@ -1329,8 +1331,9 @@ it.effect("a background summary that fell too far behind is replaced from a newe const compaction = yield* enablePrecompute const session = yield* insertSession(Session.ID.make("ses_precompute_behind")) const history = precomputeHistory() - // About 35k tokens arrived after the summarized prefix, past the 30k that may stay verbatim beside it. - const behind = [...history.near, textUser("x".repeat(140_000)), measured("Burst handled", 160_000)] + // About 75k tokens arrived after the summarized prefix, past the 68k (40% of the 170k budget) that may stay + // verbatim beside it. + const behind = [...history.near, textUser("x".repeat(300_000)), measured("Burst handled", 160_000)] yield* compaction.precompute(Effect.succeed(loaded(session, history.near))) yield* compaction.precompute(Effect.succeed(loaded(session, behind))) @@ -1393,8 +1396,8 @@ it.effect("compaction summarizes as usual when the background summary no longer return JSON.stringify(requests.at(-1)?.messages) }) - // About 35k tokens arrived after the summarized prefix, past the 30k that may stay verbatim beside it. - const burst = [...history.near, textUser("x".repeat(140_000)), measured("Burst handled", 182_000)] + // About 75k tokens arrived after the summarized prefix, past the 68k that may stay verbatim beside it. + const burst = [...history.near, textUser("x".repeat(300_000)), measured("Burst handled", 182_000)] expect(yield* compactAfterPrecompute(burst)).toContain("Burst handled") // A different model would replay a summary written for the previous one. @@ -1405,3 +1408,116 @@ it.effect("compaction summarizes as usual when the background summary no longer expect(yield* compactAfterPrecompute(history.due, switched)).toContain("Third answer") }), ) + +it.effect("a background summary keeps a tail longer than twice keep while it stays under 40% of the budget", () => + Effect.gen(function* () { + requests = [] + const compaction = yield* enablePrecompute + const session = yield* insertSession(Session.ID.make("ses_precompute_long_tail")) + const history = precomputeHistory() + // About 35k tokens arrived after the summarized prefix: more than twice keep, less than 68k. + const burst = [...history.near, textUser("x".repeat(140_000)), measured("Burst handled", 160_000)] + const due = [...burst, textUser("Next question"), measured("Next answer", 182_000)] + + yield* compaction.precompute(Effect.succeed(loaded(session, history.near))) + yield* compaction.precompute(Effect.succeed(loaded(session, burst))) + expect(yield* compaction.compact({ reason: "auto", context: loaded(session, due) })).toEqual({ + status: "completed", + }) + + // The first summary was kept and swapped in: one request, ending before the response it was started from. + expect(requests).toHaveLength(1) + const summarized = JSON.stringify(requests[0].messages) + expect(summarized).toContain("Second question") + expect(summarized).not.toContain("Second answer") + }), +) + +it.effect("a background summary starts a fixed lead before the threshold, not at a share of it", () => + Effect.gen(function* () { + requests = [] + const compaction = yield* enablePrecompute + const session = yield* insertSession(Session.ID.make("ses_precompute_lead")) + // A 1M window compacts at 850k, so a background summary starts 60k before that, at 790k. + const large = (messages: readonly SessionMessage.Info[]) => ({ + ...loaded(session, messages), + model: { ...resolved, limit: { context: 1_000_000, output: 128_000 } }, + }) + const early = [textUser("First question"), measured("First answer", 100_000), textUser("Second question")] + const before = [...early, measured("Second answer", 760_000)] + const within = [...before, textUser("Third question"), measured("Third answer", 800_000)] + const due = [...within, textUser("Fourth question"), measured("Fourth answer", 860_000)] + + // 760k is past 85% of the threshold, but still more than the lead away from it. + yield* compaction.precompute(Effect.succeed(large(before))) + yield* compaction.precompute(Effect.succeed(large(within))) + expect(yield* compaction.compact({ reason: "auto", context: large(due) })).toEqual({ status: "completed" }) + + expect(requests).toHaveLength(1) + const summarized = JSON.stringify(requests[0].messages) + expect(summarized).toContain("Third question") + expect(summarized).not.toContain("Third answer") + }), +) + +it.effect("a background summary starts on a window whose threshold is under 64k", () => + Effect.gen(function* () { + requests = [] + const compaction = yield* enablePrecompute + const session = yield* insertSession(Session.ID.make("ses_precompute_small")) + // A 64k window compacts at 48k. The first response measured 20k, so half that room, 14k, sets the start at 34k. + const small = (messages: readonly SessionMessage.Info[]) => ({ + ...loaded(session, messages), + model: { ...resolved, limit: { context: 64_000, output: 16_000 } }, + }) + const early = [textUser("First question"), measured("First answer", 20_000), textUser("Second question")] + const before = [...early, measured("Second answer", 30_000)] + const within = [...before, textUser("Third question"), measured("Third answer", 40_000)] + const due = [...within, textUser("Fourth question"), measured("Fourth answer", 50_000)] + + yield* compaction.precompute(Effect.succeed(small(before))) + yield* compaction.precompute(Effect.succeed(small(within))) + expect(yield* compaction.compact({ reason: "auto", context: small(due) })).toEqual({ status: "completed" }) + + expect(requests).toHaveLength(1) + const summarized = JSON.stringify(requests[0].messages) + expect(summarized).toContain("Third question") + expect(summarized).not.toContain("Third answer") + }), +) + +it.effect("a background summary waits for a response whose prompt is still cached", () => + Effect.gen(function* () { + const compaction = yield* enablePrecompute + const history = precomputeHistory() + const later = (created: number) => [ + ...history.near, + textUser("Third question"), + measured("Third answer", 160_000, created), + ] + const summarizedAfterBreak = (sessionID: string, ttl?: "1h") => + Effect.gen(function* () { + requests = [] + const session = yield* insertSession(Session.ID.make(sessionID)) + const context = (messages: readonly SessionMessage.Info[]) => ({ + ...loaded(session, messages), + model: { ...resolved, ...(ttl ? { cache: { ttl } } : {}) }, + }) + const now = DateTime.toEpochMillis(yield* DateTime.now) + // The latest response was written ten minutes ago; then a new one arrives. + yield* compaction.precompute(Effect.succeed(context(history.near))) + const resumed = later(now) + yield* compaction.precompute(Effect.succeed(context(resumed))) + const due = [...resumed, textUser("Fourth question"), measured("Fourth answer", 182_000, now)] + expect(yield* compaction.compact({ reason: "auto", context: context(due) })).toEqual({ status: "completed" }) + expect(requests).toHaveLength(1) + return JSON.stringify(requests[0].messages) + }) + + yield* TestClock.adjust("10 minutes") + // Past the default five-minute cache, the summary starts from the next response instead. + expect(yield* summarizedAfterBreak("ses_precompute_cold")).toContain("Third question") + // A model that keeps its cache for an hour summarizes the older response right away. + expect(yield* summarizedAfterBreak("ses_precompute_hour", "1h")).not.toContain("Third question") + }), +) diff --git a/services/www/src/docs/content/compaction.mdx b/services/www/src/docs/content/compaction.mdx index 8e2a06c50af..47a373834cd 100644 --- a/services/www/src/docs/content/compaction.mdx +++ b/services/www/src/docs/content/compaction.mdx @@ -137,16 +137,20 @@ writes the summary ahead of time and swaps it in when compaction is due: ``` ```text -85% of the threshold summary of the conversation so far starts in the background - the session keeps going; the summary is held, not applied -threshold the held summary is swapped in; everything since it is kept as is +60k tokens before the threshold summary of the conversation so far starts in the background + the session keeps going; the summary is held, not applied +threshold the held summary is swapped in; everything since it is kept as is ``` +On small context windows, the summary starts at most halfway between the first response after the last compaction +and the threshold. + | Rule | Behavior | | --- | --- | -| Cost | The summary request reuses the prompt cache. A summary that is never used still costs tokens. | -| Falls back | Compaction summarizes as usual if too much conversation arrived since the held summary, or if the model, instructions, or start of the conversation changed. | -| Skipped | Not used for native compaction, in sessions under about 64k tokens, right after a model switch, or when a provider rejects a request as too long. | +| Cost | The summary request reuses the prompt cache. A summary that is never used, for example because the session ends before the threshold, still costs tokens. | +| Waits | If the summary is still being written when compaction is due, the session waits for it to finish. | +| Falls back | Compaction summarizes as usual if the conversation since the held summary grew past about 40% of the threshold, or if the model, instructions, or start of the conversation changed. | +| Skipped | Not used for native compaction, right after a model switch, or when a provider rejects a request as too long. After a break longer than the prompt cache lifetime, it starts from the next response instead. | | Storage | Held summaries live in memory and are lost on restart. | ## Providers