fix(core): start background compaction a fixed lead before the threshold (#60049)

This commit is contained in:
Filip
2026-11-05 00:28:39 +01:00
committed by GitHub
Unverified
parent 134d1b411c
commit a0c6df8fde
3 changed files with 181 additions and 36 deletions
+46 -21
View File
@@ -18,7 +18,7 @@ import type { StreamOptions } from "@opencode/ai/route"
import type { SessionCompactionResult } from "@opencode/plugin/effect/session"
import type { SessionError } from "@opencode/schema/session-error"
import { makeLocationNode } from "@opencode/util/effect/app-node"
import { Context, Effect, Exit, Fiber, Layer, Result, Scope, Stream } from "effect"
import { Clock, Context, DateTime, Effect, Exit, Fiber, Layer, Result, Scope, Stream } from "effect"
import { Agent } from "../agent.js"
import { Bus } from "../bus.js"
import { Database } from "../database/database.js"
@@ -150,16 +150,23 @@ const TRANSCRIPT_LEVELS = [0, OUTPUTS_CLIPPED, OUTPUTS_REMOVED, USER_ONLY, OMITT
const TRANSCRIPT_CLIP = 2_000
const IMAGE_TOKEN_ESTIMATE = 1_500
const PDF_TOKEN_ESTIMATE = 2_000
/** Share of the compaction ceiling at which a background summary starts. */
const PRECOMPUTE_RATIO = 0.85
/** Measured prompt tokens below which a background summary is not worth its cost. */
const PRECOMPUTE_MIN_TOKENS = 64_000
/**
* The most text a swapped-in summary keeps verbatim beside it: twice `keep`, at least 30k, at most a quarter of the
* budget. A longer tail would leave the context near the threshold, so compaction summarizes from scratch instead.
* Tokens before the ceiling at which a background summary starts. Writing one takes 30–60 s, and in that time the
* conversation grows by a few thousand tokens at the median but by 25k or more one time in twenty, when large file
* reads arrive together. How far ahead to start depends on that growth, not on the window, so the lead is fixed.
*/
const PRECOMPUTED_TAIL_MIN = 30_000
const PRECOMPUTED_TAIL_SHARE = 0.25
const PRECOMPUTE_LEAD = 60_000
/** Prompt caches expire this long after their last use unless the model sets `cache.ttl`. */
const CACHE_LIFETIME = 5 * 60_000
/** A background summary starts only while this much of the latest response's cache lifetime is left. */
const CACHE_MARGIN = 30_000
/**
* The most text a swapped-in summary keeps verbatim beside it, and the tail at which a held summary is replaced, as a
* share of the budget. Tails are usually far shorter; the limit is there for bursts of large tool inputs or pasted
* text. At 40%, a swapped-in context stays under about two thirds of the budget even with Claude's tokenizer, so the
* next cycle keeps room; a longer tail makes compaction summarize from scratch instead.
*/
const PRECOMPUTED_TAIL_SHARE = 0.4
/** Failed background summaries after which no new one starts until the next compaction. */
const PRECOMPUTE_MAX_FAILURES = 2
@@ -329,11 +336,14 @@ export const layer = Layer.effect(
})
/**
* Once the context passes `PRECOMPUTE_RATIO` of the ceiling, summarizes everything before the latest response in
* the background. That is the prefix the provider just processed, so the summary request reads it from the prompt
* cache. The summary is held, not applied: swapping it in now would rewrite the prefix and throw away the cache
* for every step until compaction is actually due. One summary is held per Session until compaction takes it, or
* until it fails or falls too far behind, when a newer response replaces it.
* Once the context is within `PRECOMPUTE_LEAD` of the ceiling, summarizes everything before the latest response
* in the background. That is the prefix the provider just processed, so the summary request reads it from the
* prompt cache. The summary is held, not applied: swapping it in now would rewrite the prefix and throw away the
* cache for every step until compaction is actually due. One summary is held per Session until compaction takes
* it, or until it fails or falls too far behind, when a newer response replaces it.
*
* On a small window the lead is at most half the room the cycle started with, so the summary still covers more
* than the previous one.
*/
const precompute = Effect.fn("SessionCompaction.precompute")(function* <E>(
load: Effect.Effect<SessionContext.Loaded, E>,
@@ -347,12 +357,13 @@ export const layer = Layer.effect(
if (!Number.isFinite(ceiling)) return
const estimate = given ?? estimatePrompt(context)
if (due(context, ceiling, estimate)) return
if (total(estimate) < ceiling * PRECOMPUTE_RATIO) return
const first = context.messages.find((message) => hasMeasuredPrompt(message, context.model.ref))
const cycleStart = first?.type === "assistant" && first.tokens ? promptTokens(first.tokens) : 0
if (total(estimate) < ceiling - Math.min(PRECOMPUTE_LEAD, (ceiling - cycleStart) / 2)) return
const anchorIndex = context.messages.findLastIndex((message) => hasMeasuredPrompt(message, context.model.ref))
const anchor = context.messages[anchorIndex]
if (anchorIndex <= 0 || anchor?.type !== "assistant" || !anchor.tokens) return
if (anchor.tokens.input + anchor.tokens.cache.read + anchor.tokens.cache.write < PRECOMPUTE_MIN_TOKENS) return
const sessionID = context.session.id
const existing = precomputed.get(sessionID)
@@ -369,6 +380,10 @@ export const layer = Layer.effect(
}
// After a model switch nothing is cached for the new model yet; wait until its first response has written it.
if (anchor.model.id !== context.model.ref.id || anchor.model.variant !== context.model.ref.variant) return
// The same once the latest response is older than the cache lifetime, as when the user comes back from a
// break: the summary would write the whole prefix again, at the same time as the next request.
const age = (yield* Clock.currentTimeMillis) - DateTime.toEpochMillis(anchor.time.created)
if (age > cacheLifetime(context.model.cache?.ttl) - CACHE_MARGIN) return
const model = yield* selectModel(context, settings.buffer, false)
const usage: SessionUsage.Recorded[] = []
@@ -381,7 +396,7 @@ export const layer = Layer.effect(
keepID: anchor.id,
model: context.model.ref,
initial: Instructions.join(context.initial),
tailLimit: tailLimit(settings.keep, model.cap),
tailLimit: tailLimit(model.cap),
failures,
usage,
fiber,
@@ -406,7 +421,7 @@ export const layer = Layer.effect(
trigger.reason !== "overflow" &&
context.model.compaction?.type !== "native" &&
index >= 0 &&
tailTokens(context, index) <= tailLimit(settings.keep, budget)
tailTokens(context, index) <= tailLimit(budget)
if (!usable) {
yield* Fiber.interrupt(held.fiber)
return undefined
@@ -961,8 +976,18 @@ const heldIndex = (held: Precomputed, context: SessionContext.Loaded) => {
const tailTokens = (context: SessionContext.Loaded, index: number) =>
Token.estimate(context.messages.slice(index).map(SessionRecentText.messageToText).filter(Boolean).join("\n\n"))
const tailLimit = (keep: number, budget: number) =>
Math.min(Math.max(keep * 2, PRECOMPUTED_TAIL_MIN), Math.floor(budget * PRECOMPUTED_TAIL_SHARE))
const tailLimit = (budget: number) => Math.floor(budget * PRECOMPUTED_TAIL_SHARE)
const promptTokens = (tokens: { input: number; cache: { read: number; write: number } }) =>
tokens.input + tokens.cache.read + tokens.cache.write
const DURATION_UNITS = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 }
/** Milliseconds a prompt cache entry lives, from a `cache.ttl` such as "5m" or "1h". */
const cacheLifetime = (ttl: string | undefined) => {
const match = ttl?.match(/^(\d+)(ms|s|m|h|d)$/)
return match ? Number(match[1]) * DURATION_UNITS[match[2] as keyof typeof DURATION_UNITS] : CACHE_LIFETIME
}
/** History loads from the latest completed compaction, so a previous one is always the first message. */
const previousCompaction = (messages: ReadonlyArray<SessionMessage.Info>) => {
@@ -1166,7 +1191,7 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L
message.model.providerID === model.providerID &&
!message.error &&
message.tokens !== undefined &&
message.tokens.input + message.tokens.cache.read + message.tokens.cache.write > 0
promptTokens(message.tokens) > 0
/** The prompt size, only while the model, variant and agent that were measured still match; they shape the request. */
const measuredPrompt = (context: SessionContext.Loaded) => {
+125 -9
View File
@@ -44,6 +44,7 @@ import { Money } from "@opencode/schema/money"
import { Skill } from "@opencode/schema/skill"
import { Shell } from "@opencode/schema/shell"
import { DateTime, Deferred, Effect, Fiber, Layer, Schema, Stream } from "effect"
import { TestClock } from "effect/testing"
import { asc, eq, inArray, sql } from "drizzle-orm"
import { testEffect } from "./lib/effect"
@@ -1000,7 +1001,7 @@ it.effect("forked session compaction reuses the fork root prompt cache key", ()
}),
)
const measured = (text: string, input: number) =>
const measured = (text: string, input: number, created = 0) =>
Schema.decodeUnknownSync(SessionMessage.Assistant)({
id: SessionMessage.ID.create(),
type: "assistant",
@@ -1008,12 +1009,12 @@ const measured = (text: string, input: number) =>
model: { id: "summary-model", providerID: "test" },
content: [{ type: "text", text }],
tokens: { input, output: 100, reasoning: 0, cache: { read: 0, write: 0 } },
time: { created: 0, completed: 1 },
time: { created, completed: created + 1 },
})
/**
* A 200k window compacts at 170k and starts a background summary at 85% of that, 144.5k. `near` ends past that
* point; `due` continues it past the threshold.
* A 200k window compacts at 170k. The first response measured 80k, so a background summary starts half that room,
* 45k, before the threshold, at 125k. `near` ends past that point; `due` continues it past the threshold.
*/
const precomputeHistory = () => {
const near = [textUser("First question"), measured("First answer", 80_000), textUser("Second question")]
@@ -1061,7 +1062,8 @@ for (const background of [false, true]) {
Effect.gen(function* () {
const messages = [
textUser("Earlier question"),
oldReasoning(),
// The cycle starts low, so the latest response is past the point where a background summary starts.
oldReasoning(summaryInput / 4),
textUser("Latest question"),
measured("Latest answer", summaryInput),
]
@@ -1329,8 +1331,9 @@ it.effect("a background summary that fell too far behind is replaced from a newe
const compaction = yield* enablePrecompute
const session = yield* insertSession(Session.ID.make("ses_precompute_behind"))
const history = precomputeHistory()
// About 35k tokens arrived after the summarized prefix, past the 30k that may stay verbatim beside it.
const behind = [...history.near, textUser("x".repeat(140_000)), measured("Burst handled", 160_000)]
// About 75k tokens arrived after the summarized prefix, past the 68k (40% of the 170k budget) that may stay
// verbatim beside it.
const behind = [...history.near, textUser("x".repeat(300_000)), measured("Burst handled", 160_000)]
yield* compaction.precompute(Effect.succeed(loaded(session, history.near)))
yield* compaction.precompute(Effect.succeed(loaded(session, behind)))
@@ -1393,8 +1396,8 @@ it.effect("compaction summarizes as usual when the background summary no longer
return JSON.stringify(requests.at(-1)?.messages)
})
// About 35k tokens arrived after the summarized prefix, past the 30k that may stay verbatim beside it.
const burst = [...history.near, textUser("x".repeat(140_000)), measured("Burst handled", 182_000)]
// About 75k tokens arrived after the summarized prefix, past the 68k that may stay verbatim beside it.
const burst = [...history.near, textUser("x".repeat(300_000)), measured("Burst handled", 182_000)]
expect(yield* compactAfterPrecompute(burst)).toContain("Burst handled")
// A different model would replay a summary written for the previous one.
@@ -1405,3 +1408,116 @@ it.effect("compaction summarizes as usual when the background summary no longer
expect(yield* compactAfterPrecompute(history.due, switched)).toContain("Third answer")
}),
)
it.effect("a background summary keeps a tail longer than twice keep while it stays under 40% of the budget", () =>
Effect.gen(function* () {
requests = []
const compaction = yield* enablePrecompute
const session = yield* insertSession(Session.ID.make("ses_precompute_long_tail"))
const history = precomputeHistory()
// About 35k tokens arrived after the summarized prefix: more than twice keep, less than 68k.
const burst = [...history.near, textUser("x".repeat(140_000)), measured("Burst handled", 160_000)]
const due = [...burst, textUser("Next question"), measured("Next answer", 182_000)]
yield* compaction.precompute(Effect.succeed(loaded(session, history.near)))
yield* compaction.precompute(Effect.succeed(loaded(session, burst)))
expect(yield* compaction.compact({ reason: "auto", context: loaded(session, due) })).toEqual({
status: "completed",
})
// The first summary was kept and swapped in: one request, ending before the response it was started from.
expect(requests).toHaveLength(1)
const summarized = JSON.stringify(requests[0].messages)
expect(summarized).toContain("Second question")
expect(summarized).not.toContain("Second answer")
}),
)
it.effect("a background summary starts a fixed lead before the threshold, not at a share of it", () =>
Effect.gen(function* () {
requests = []
const compaction = yield* enablePrecompute
const session = yield* insertSession(Session.ID.make("ses_precompute_lead"))
// A 1M window compacts at 850k, so a background summary starts 60k before that, at 790k.
const large = (messages: readonly SessionMessage.Info[]) => ({
...loaded(session, messages),
model: { ...resolved, limit: { context: 1_000_000, output: 128_000 } },
})
const early = [textUser("First question"), measured("First answer", 100_000), textUser("Second question")]
const before = [...early, measured("Second answer", 760_000)]
const within = [...before, textUser("Third question"), measured("Third answer", 800_000)]
const due = [...within, textUser("Fourth question"), measured("Fourth answer", 860_000)]
// 760k is past 85% of the threshold, but still more than the lead away from it.
yield* compaction.precompute(Effect.succeed(large(before)))
yield* compaction.precompute(Effect.succeed(large(within)))
expect(yield* compaction.compact({ reason: "auto", context: large(due) })).toEqual({ status: "completed" })
expect(requests).toHaveLength(1)
const summarized = JSON.stringify(requests[0].messages)
expect(summarized).toContain("Third question")
expect(summarized).not.toContain("Third answer")
}),
)
it.effect("a background summary starts on a window whose threshold is under 64k", () =>
Effect.gen(function* () {
requests = []
const compaction = yield* enablePrecompute
const session = yield* insertSession(Session.ID.make("ses_precompute_small"))
// A 64k window compacts at 48k. The first response measured 20k, so half that room, 14k, sets the start at 34k.
const small = (messages: readonly SessionMessage.Info[]) => ({
...loaded(session, messages),
model: { ...resolved, limit: { context: 64_000, output: 16_000 } },
})
const early = [textUser("First question"), measured("First answer", 20_000), textUser("Second question")]
const before = [...early, measured("Second answer", 30_000)]
const within = [...before, textUser("Third question"), measured("Third answer", 40_000)]
const due = [...within, textUser("Fourth question"), measured("Fourth answer", 50_000)]
yield* compaction.precompute(Effect.succeed(small(before)))
yield* compaction.precompute(Effect.succeed(small(within)))
expect(yield* compaction.compact({ reason: "auto", context: small(due) })).toEqual({ status: "completed" })
expect(requests).toHaveLength(1)
const summarized = JSON.stringify(requests[0].messages)
expect(summarized).toContain("Third question")
expect(summarized).not.toContain("Third answer")
}),
)
it.effect("a background summary waits for a response whose prompt is still cached", () =>
Effect.gen(function* () {
const compaction = yield* enablePrecompute
const history = precomputeHistory()
const later = (created: number) => [
...history.near,
textUser("Third question"),
measured("Third answer", 160_000, created),
]
const summarizedAfterBreak = (sessionID: string, ttl?: "1h") =>
Effect.gen(function* () {
requests = []
const session = yield* insertSession(Session.ID.make(sessionID))
const context = (messages: readonly SessionMessage.Info[]) => ({
...loaded(session, messages),
model: { ...resolved, ...(ttl ? { cache: { ttl } } : {}) },
})
const now = DateTime.toEpochMillis(yield* DateTime.now)
// The latest response was written ten minutes ago; then a new one arrives.
yield* compaction.precompute(Effect.succeed(context(history.near)))
const resumed = later(now)
yield* compaction.precompute(Effect.succeed(context(resumed)))
const due = [...resumed, textUser("Fourth question"), measured("Fourth answer", 182_000, now)]
expect(yield* compaction.compact({ reason: "auto", context: context(due) })).toEqual({ status: "completed" })
expect(requests).toHaveLength(1)
return JSON.stringify(requests[0].messages)
})
yield* TestClock.adjust("10 minutes")
// Past the default five-minute cache, the summary starts from the next response instead.
expect(yield* summarizedAfterBreak("ses_precompute_cold")).toContain("Third question")
// A model that keeps its cache for an hour summarizes the older response right away.
expect(yield* summarizedAfterBreak("ses_precompute_hour", "1h")).not.toContain("Third question")
}),
)
+10 -6
View File
@@ -137,16 +137,20 @@ writes the summary ahead of time and swaps it in when compaction is due:
```
```text
85% of the threshold summary of the conversation so far starts in the background
the session keeps going; the summary is held, not applied
threshold the held summary is swapped in; everything since it is kept as is
60k tokens before the threshold summary of the conversation so far starts in the background
the session keeps going; the summary is held, not applied
threshold the held summary is swapped in; everything since it is kept as is
```
On small context windows, the summary starts at most halfway between the first response after the last compaction
and the threshold.
| Rule | Behavior |
| --- | --- |
| Cost | The summary request reuses the prompt cache. A summary that is never used still costs tokens. |
| Falls back | Compaction summarizes as usual if too much conversation arrived since the held summary, or if the model, instructions, or start of the conversation changed. |
| Skipped | Not used for native compaction, in sessions under about 64k tokens, right after a model switch, or when a provider rejects a request as too long. |
| Cost | The summary request reuses the prompt cache. A summary that is never used, for example because the session ends before the threshold, still costs tokens. |
| Waits | If the summary is still being written when compaction is due, the session waits for it to finish. |
| Falls back | Compaction summarizes as usual if the conversation since the held summary grew past about 40% of the threshold, or if the model, instructions, or start of the conversation changed. |
| Skipped | Not used for native compaction, right after a model switch, or when a provider rejects a request as too long. After a break longer than the prompt cache lifetime, it starts from the next response instead. |
| Storage | Held summaries live in memory and are lost on restart. |
## Providers