mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
fix(core): start background compaction a fixed lead before the threshold (#60049)
This commit is contained in:
@@ -18,7 +18,7 @@ import type { StreamOptions } from "@opencode/ai/route"
|
||||
import type { SessionCompactionResult } from "@opencode/plugin/effect/session"
|
||||
import type { SessionError } from "@opencode/schema/session-error"
|
||||
import { makeLocationNode } from "@opencode/util/effect/app-node"
|
||||
import { Context, Effect, Exit, Fiber, Layer, Result, Scope, Stream } from "effect"
|
||||
import { Clock, Context, DateTime, Effect, Exit, Fiber, Layer, Result, Scope, Stream } from "effect"
|
||||
import { Agent } from "../agent.js"
|
||||
import { Bus } from "../bus.js"
|
||||
import { Database } from "../database/database.js"
|
||||
@@ -150,16 +150,23 @@ const TRANSCRIPT_LEVELS = [0, OUTPUTS_CLIPPED, OUTPUTS_REMOVED, USER_ONLY, OMITT
|
||||
const TRANSCRIPT_CLIP = 2_000
|
||||
const IMAGE_TOKEN_ESTIMATE = 1_500
|
||||
const PDF_TOKEN_ESTIMATE = 2_000
|
||||
/** Share of the compaction ceiling at which a background summary starts. */
|
||||
const PRECOMPUTE_RATIO = 0.85
|
||||
/** Measured prompt tokens below which a background summary is not worth its cost. */
|
||||
const PRECOMPUTE_MIN_TOKENS = 64_000
|
||||
/**
|
||||
* The most text a swapped-in summary keeps verbatim beside it: twice `keep`, at least 30k, at most a quarter of the
|
||||
* budget. A longer tail would leave the context near the threshold, so compaction summarizes from scratch instead.
|
||||
* Tokens before the ceiling at which a background summary starts. Writing one takes 30–60 s, and in that time the
|
||||
* conversation grows by a few thousand tokens at the median but by 25k or more one time in twenty, when large file
|
||||
* reads arrive together. How far ahead to start depends on that growth, not on the window, so the lead is fixed.
|
||||
*/
|
||||
const PRECOMPUTED_TAIL_MIN = 30_000
|
||||
const PRECOMPUTED_TAIL_SHARE = 0.25
|
||||
const PRECOMPUTE_LEAD = 60_000
|
||||
/** Prompt caches expire this long after their last use unless the model sets `cache.ttl`. */
|
||||
const CACHE_LIFETIME = 5 * 60_000
|
||||
/** A background summary starts only while this much of the latest response's cache lifetime is left. */
|
||||
const CACHE_MARGIN = 30_000
|
||||
/**
|
||||
* The most text a swapped-in summary keeps verbatim beside it, and the tail at which a held summary is replaced, as a
|
||||
* share of the budget. Tails are usually far shorter; the limit is there for bursts of large tool inputs or pasted
|
||||
* text. At 40%, a swapped-in context stays under about two thirds of the budget even with Claude's tokenizer, so the
|
||||
* next cycle keeps room; a longer tail makes compaction summarize from scratch instead.
|
||||
*/
|
||||
const PRECOMPUTED_TAIL_SHARE = 0.4
|
||||
/** Failed background summaries after which no new one starts until the next compaction. */
|
||||
const PRECOMPUTE_MAX_FAILURES = 2
|
||||
|
||||
@@ -329,11 +336,14 @@ export const layer = Layer.effect(
|
||||
})
|
||||
|
||||
/**
|
||||
* Once the context passes `PRECOMPUTE_RATIO` of the ceiling, summarizes everything before the latest response in
|
||||
* the background. That is the prefix the provider just processed, so the summary request reads it from the prompt
|
||||
* cache. The summary is held, not applied: swapping it in now would rewrite the prefix and throw away the cache
|
||||
* for every step until compaction is actually due. One summary is held per Session until compaction takes it, or
|
||||
* until it fails or falls too far behind, when a newer response replaces it.
|
||||
* Once the context is within `PRECOMPUTE_LEAD` of the ceiling, summarizes everything before the latest response
|
||||
* in the background. That is the prefix the provider just processed, so the summary request reads it from the
|
||||
* prompt cache. The summary is held, not applied: swapping it in now would rewrite the prefix and throw away the
|
||||
* cache for every step until compaction is actually due. One summary is held per Session until compaction takes
|
||||
* it, or until it fails or falls too far behind, when a newer response replaces it.
|
||||
*
|
||||
* On a small window the lead is at most half the room the cycle started with, so the summary still covers more
|
||||
* than the previous one.
|
||||
*/
|
||||
const precompute = Effect.fn("SessionCompaction.precompute")(function* <E>(
|
||||
load: Effect.Effect<SessionContext.Loaded, E>,
|
||||
@@ -347,12 +357,13 @@ export const layer = Layer.effect(
|
||||
if (!Number.isFinite(ceiling)) return
|
||||
const estimate = given ?? estimatePrompt(context)
|
||||
if (due(context, ceiling, estimate)) return
|
||||
if (total(estimate) < ceiling * PRECOMPUTE_RATIO) return
|
||||
const first = context.messages.find((message) => hasMeasuredPrompt(message, context.model.ref))
|
||||
const cycleStart = first?.type === "assistant" && first.tokens ? promptTokens(first.tokens) : 0
|
||||
if (total(estimate) < ceiling - Math.min(PRECOMPUTE_LEAD, (ceiling - cycleStart) / 2)) return
|
||||
|
||||
const anchorIndex = context.messages.findLastIndex((message) => hasMeasuredPrompt(message, context.model.ref))
|
||||
const anchor = context.messages[anchorIndex]
|
||||
if (anchorIndex <= 0 || anchor?.type !== "assistant" || !anchor.tokens) return
|
||||
if (anchor.tokens.input + anchor.tokens.cache.read + anchor.tokens.cache.write < PRECOMPUTE_MIN_TOKENS) return
|
||||
|
||||
const sessionID = context.session.id
|
||||
const existing = precomputed.get(sessionID)
|
||||
@@ -369,6 +380,10 @@ export const layer = Layer.effect(
|
||||
}
|
||||
// After a model switch nothing is cached for the new model yet; wait until its first response has written it.
|
||||
if (anchor.model.id !== context.model.ref.id || anchor.model.variant !== context.model.ref.variant) return
|
||||
// The same once the latest response is older than the cache lifetime, as when the user comes back from a
|
||||
// break: the summary would write the whole prefix again, at the same time as the next request.
|
||||
const age = (yield* Clock.currentTimeMillis) - DateTime.toEpochMillis(anchor.time.created)
|
||||
if (age > cacheLifetime(context.model.cache?.ttl) - CACHE_MARGIN) return
|
||||
|
||||
const model = yield* selectModel(context, settings.buffer, false)
|
||||
const usage: SessionUsage.Recorded[] = []
|
||||
@@ -381,7 +396,7 @@ export const layer = Layer.effect(
|
||||
keepID: anchor.id,
|
||||
model: context.model.ref,
|
||||
initial: Instructions.join(context.initial),
|
||||
tailLimit: tailLimit(settings.keep, model.cap),
|
||||
tailLimit: tailLimit(model.cap),
|
||||
failures,
|
||||
usage,
|
||||
fiber,
|
||||
@@ -406,7 +421,7 @@ export const layer = Layer.effect(
|
||||
trigger.reason !== "overflow" &&
|
||||
context.model.compaction?.type !== "native" &&
|
||||
index >= 0 &&
|
||||
tailTokens(context, index) <= tailLimit(settings.keep, budget)
|
||||
tailTokens(context, index) <= tailLimit(budget)
|
||||
if (!usable) {
|
||||
yield* Fiber.interrupt(held.fiber)
|
||||
return undefined
|
||||
@@ -961,8 +976,18 @@ const heldIndex = (held: Precomputed, context: SessionContext.Loaded) => {
|
||||
const tailTokens = (context: SessionContext.Loaded, index: number) =>
|
||||
Token.estimate(context.messages.slice(index).map(SessionRecentText.messageToText).filter(Boolean).join("\n\n"))
|
||||
|
||||
const tailLimit = (keep: number, budget: number) =>
|
||||
Math.min(Math.max(keep * 2, PRECOMPUTED_TAIL_MIN), Math.floor(budget * PRECOMPUTED_TAIL_SHARE))
|
||||
const tailLimit = (budget: number) => Math.floor(budget * PRECOMPUTED_TAIL_SHARE)
|
||||
|
||||
const promptTokens = (tokens: { input: number; cache: { read: number; write: number } }) =>
|
||||
tokens.input + tokens.cache.read + tokens.cache.write
|
||||
|
||||
const DURATION_UNITS = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 }
|
||||
|
||||
/** Milliseconds a prompt cache entry lives, from a `cache.ttl` such as "5m" or "1h". */
|
||||
const cacheLifetime = (ttl: string | undefined) => {
|
||||
const match = ttl?.match(/^(\d+)(ms|s|m|h|d)$/)
|
||||
return match ? Number(match[1]) * DURATION_UNITS[match[2] as keyof typeof DURATION_UNITS] : CACHE_LIFETIME
|
||||
}
|
||||
|
||||
/** History loads from the latest completed compaction, so a previous one is always the first message. */
|
||||
const previousCompaction = (messages: ReadonlyArray<SessionMessage.Info>) => {
|
||||
@@ -1166,7 +1191,7 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L
|
||||
message.model.providerID === model.providerID &&
|
||||
!message.error &&
|
||||
message.tokens !== undefined &&
|
||||
message.tokens.input + message.tokens.cache.read + message.tokens.cache.write > 0
|
||||
promptTokens(message.tokens) > 0
|
||||
|
||||
/** The prompt size, only while the model, variant and agent that were measured still match; they shape the request. */
|
||||
const measuredPrompt = (context: SessionContext.Loaded) => {
|
||||
|
||||
@@ -44,6 +44,7 @@ import { Money } from "@opencode/schema/money"
|
||||
import { Skill } from "@opencode/schema/skill"
|
||||
import { Shell } from "@opencode/schema/shell"
|
||||
import { DateTime, Deferred, Effect, Fiber, Layer, Schema, Stream } from "effect"
|
||||
import { TestClock } from "effect/testing"
|
||||
import { asc, eq, inArray, sql } from "drizzle-orm"
|
||||
import { testEffect } from "./lib/effect"
|
||||
|
||||
@@ -1000,7 +1001,7 @@ it.effect("forked session compaction reuses the fork root prompt cache key", ()
|
||||
}),
|
||||
)
|
||||
|
||||
const measured = (text: string, input: number) =>
|
||||
const measured = (text: string, input: number, created = 0) =>
|
||||
Schema.decodeUnknownSync(SessionMessage.Assistant)({
|
||||
id: SessionMessage.ID.create(),
|
||||
type: "assistant",
|
||||
@@ -1008,12 +1009,12 @@ const measured = (text: string, input: number) =>
|
||||
model: { id: "summary-model", providerID: "test" },
|
||||
content: [{ type: "text", text }],
|
||||
tokens: { input, output: 100, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
time: { created: 0, completed: 1 },
|
||||
time: { created, completed: created + 1 },
|
||||
})
|
||||
|
||||
/**
|
||||
* A 200k window compacts at 170k and starts a background summary at 85% of that, 144.5k. `near` ends past that
|
||||
* point; `due` continues it past the threshold.
|
||||
* A 200k window compacts at 170k. The first response measured 80k, so a background summary starts half that room,
|
||||
* 45k, before the threshold, at 125k. `near` ends past that point; `due` continues it past the threshold.
|
||||
*/
|
||||
const precomputeHistory = () => {
|
||||
const near = [textUser("First question"), measured("First answer", 80_000), textUser("Second question")]
|
||||
@@ -1061,7 +1062,8 @@ for (const background of [false, true]) {
|
||||
Effect.gen(function* () {
|
||||
const messages = [
|
||||
textUser("Earlier question"),
|
||||
oldReasoning(),
|
||||
// The cycle starts low, so the latest response is past the point where a background summary starts.
|
||||
oldReasoning(summaryInput / 4),
|
||||
textUser("Latest question"),
|
||||
measured("Latest answer", summaryInput),
|
||||
]
|
||||
@@ -1329,8 +1331,9 @@ it.effect("a background summary that fell too far behind is replaced from a newe
|
||||
const compaction = yield* enablePrecompute
|
||||
const session = yield* insertSession(Session.ID.make("ses_precompute_behind"))
|
||||
const history = precomputeHistory()
|
||||
// About 35k tokens arrived after the summarized prefix, past the 30k that may stay verbatim beside it.
|
||||
const behind = [...history.near, textUser("x".repeat(140_000)), measured("Burst handled", 160_000)]
|
||||
// About 75k tokens arrived after the summarized prefix, past the 68k (40% of the 170k budget) that may stay
|
||||
// verbatim beside it.
|
||||
const behind = [...history.near, textUser("x".repeat(300_000)), measured("Burst handled", 160_000)]
|
||||
|
||||
yield* compaction.precompute(Effect.succeed(loaded(session, history.near)))
|
||||
yield* compaction.precompute(Effect.succeed(loaded(session, behind)))
|
||||
@@ -1393,8 +1396,8 @@ it.effect("compaction summarizes as usual when the background summary no longer
|
||||
return JSON.stringify(requests.at(-1)?.messages)
|
||||
})
|
||||
|
||||
// About 35k tokens arrived after the summarized prefix, past the 30k that may stay verbatim beside it.
|
||||
const burst = [...history.near, textUser("x".repeat(140_000)), measured("Burst handled", 182_000)]
|
||||
// About 75k tokens arrived after the summarized prefix, past the 68k that may stay verbatim beside it.
|
||||
const burst = [...history.near, textUser("x".repeat(300_000)), measured("Burst handled", 182_000)]
|
||||
expect(yield* compactAfterPrecompute(burst)).toContain("Burst handled")
|
||||
|
||||
// A different model would replay a summary written for the previous one.
|
||||
@@ -1405,3 +1408,116 @@ it.effect("compaction summarizes as usual when the background summary no longer
|
||||
expect(yield* compactAfterPrecompute(history.due, switched)).toContain("Third answer")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("a background summary keeps a tail longer than twice keep while it stays under 40% of the budget", () =>
|
||||
Effect.gen(function* () {
|
||||
requests = []
|
||||
const compaction = yield* enablePrecompute
|
||||
const session = yield* insertSession(Session.ID.make("ses_precompute_long_tail"))
|
||||
const history = precomputeHistory()
|
||||
// About 35k tokens arrived after the summarized prefix: more than twice keep, less than 68k.
|
||||
const burst = [...history.near, textUser("x".repeat(140_000)), measured("Burst handled", 160_000)]
|
||||
const due = [...burst, textUser("Next question"), measured("Next answer", 182_000)]
|
||||
|
||||
yield* compaction.precompute(Effect.succeed(loaded(session, history.near)))
|
||||
yield* compaction.precompute(Effect.succeed(loaded(session, burst)))
|
||||
expect(yield* compaction.compact({ reason: "auto", context: loaded(session, due) })).toEqual({
|
||||
status: "completed",
|
||||
})
|
||||
|
||||
// The first summary was kept and swapped in: one request, ending before the response it was started from.
|
||||
expect(requests).toHaveLength(1)
|
||||
const summarized = JSON.stringify(requests[0].messages)
|
||||
expect(summarized).toContain("Second question")
|
||||
expect(summarized).not.toContain("Second answer")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("a background summary starts a fixed lead before the threshold, not at a share of it", () =>
|
||||
Effect.gen(function* () {
|
||||
requests = []
|
||||
const compaction = yield* enablePrecompute
|
||||
const session = yield* insertSession(Session.ID.make("ses_precompute_lead"))
|
||||
// A 1M window compacts at 850k, so a background summary starts 60k before that, at 790k.
|
||||
const large = (messages: readonly SessionMessage.Info[]) => ({
|
||||
...loaded(session, messages),
|
||||
model: { ...resolved, limit: { context: 1_000_000, output: 128_000 } },
|
||||
})
|
||||
const early = [textUser("First question"), measured("First answer", 100_000), textUser("Second question")]
|
||||
const before = [...early, measured("Second answer", 760_000)]
|
||||
const within = [...before, textUser("Third question"), measured("Third answer", 800_000)]
|
||||
const due = [...within, textUser("Fourth question"), measured("Fourth answer", 860_000)]
|
||||
|
||||
// 760k is past 85% of the threshold, but still more than the lead away from it.
|
||||
yield* compaction.precompute(Effect.succeed(large(before)))
|
||||
yield* compaction.precompute(Effect.succeed(large(within)))
|
||||
expect(yield* compaction.compact({ reason: "auto", context: large(due) })).toEqual({ status: "completed" })
|
||||
|
||||
expect(requests).toHaveLength(1)
|
||||
const summarized = JSON.stringify(requests[0].messages)
|
||||
expect(summarized).toContain("Third question")
|
||||
expect(summarized).not.toContain("Third answer")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("a background summary starts on a window whose threshold is under 64k", () =>
|
||||
Effect.gen(function* () {
|
||||
requests = []
|
||||
const compaction = yield* enablePrecompute
|
||||
const session = yield* insertSession(Session.ID.make("ses_precompute_small"))
|
||||
// A 64k window compacts at 48k. The first response measured 20k, so half that room, 14k, sets the start at 34k.
|
||||
const small = (messages: readonly SessionMessage.Info[]) => ({
|
||||
...loaded(session, messages),
|
||||
model: { ...resolved, limit: { context: 64_000, output: 16_000 } },
|
||||
})
|
||||
const early = [textUser("First question"), measured("First answer", 20_000), textUser("Second question")]
|
||||
const before = [...early, measured("Second answer", 30_000)]
|
||||
const within = [...before, textUser("Third question"), measured("Third answer", 40_000)]
|
||||
const due = [...within, textUser("Fourth question"), measured("Fourth answer", 50_000)]
|
||||
|
||||
yield* compaction.precompute(Effect.succeed(small(before)))
|
||||
yield* compaction.precompute(Effect.succeed(small(within)))
|
||||
expect(yield* compaction.compact({ reason: "auto", context: small(due) })).toEqual({ status: "completed" })
|
||||
|
||||
expect(requests).toHaveLength(1)
|
||||
const summarized = JSON.stringify(requests[0].messages)
|
||||
expect(summarized).toContain("Third question")
|
||||
expect(summarized).not.toContain("Third answer")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("a background summary waits for a response whose prompt is still cached", () =>
|
||||
Effect.gen(function* () {
|
||||
const compaction = yield* enablePrecompute
|
||||
const history = precomputeHistory()
|
||||
const later = (created: number) => [
|
||||
...history.near,
|
||||
textUser("Third question"),
|
||||
measured("Third answer", 160_000, created),
|
||||
]
|
||||
const summarizedAfterBreak = (sessionID: string, ttl?: "1h") =>
|
||||
Effect.gen(function* () {
|
||||
requests = []
|
||||
const session = yield* insertSession(Session.ID.make(sessionID))
|
||||
const context = (messages: readonly SessionMessage.Info[]) => ({
|
||||
...loaded(session, messages),
|
||||
model: { ...resolved, ...(ttl ? { cache: { ttl } } : {}) },
|
||||
})
|
||||
const now = DateTime.toEpochMillis(yield* DateTime.now)
|
||||
// The latest response was written ten minutes ago; then a new one arrives.
|
||||
yield* compaction.precompute(Effect.succeed(context(history.near)))
|
||||
const resumed = later(now)
|
||||
yield* compaction.precompute(Effect.succeed(context(resumed)))
|
||||
const due = [...resumed, textUser("Fourth question"), measured("Fourth answer", 182_000, now)]
|
||||
expect(yield* compaction.compact({ reason: "auto", context: context(due) })).toEqual({ status: "completed" })
|
||||
expect(requests).toHaveLength(1)
|
||||
return JSON.stringify(requests[0].messages)
|
||||
})
|
||||
|
||||
yield* TestClock.adjust("10 minutes")
|
||||
// Past the default five-minute cache, the summary starts from the next response instead.
|
||||
expect(yield* summarizedAfterBreak("ses_precompute_cold")).toContain("Third question")
|
||||
// A model that keeps its cache for an hour summarizes the older response right away.
|
||||
expect(yield* summarizedAfterBreak("ses_precompute_hour", "1h")).not.toContain("Third question")
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -137,16 +137,20 @@ writes the summary ahead of time and swaps it in when compaction is due:
|
||||
```
|
||||
|
||||
```text
|
||||
85% of the threshold summary of the conversation so far starts in the background
|
||||
the session keeps going; the summary is held, not applied
|
||||
threshold the held summary is swapped in; everything since it is kept as is
|
||||
60k tokens before the threshold summary of the conversation so far starts in the background
|
||||
the session keeps going; the summary is held, not applied
|
||||
threshold the held summary is swapped in; everything since it is kept as is
|
||||
```
|
||||
|
||||
On small context windows, the summary starts at most halfway between the first response after the last compaction
|
||||
and the threshold.
|
||||
|
||||
| Rule | Behavior |
|
||||
| --- | --- |
|
||||
| Cost | The summary request reuses the prompt cache. A summary that is never used still costs tokens. |
|
||||
| Falls back | Compaction summarizes as usual if too much conversation arrived since the held summary, or if the model, instructions, or start of the conversation changed. |
|
||||
| Skipped | Not used for native compaction, in sessions under about 64k tokens, right after a model switch, or when a provider rejects a request as too long. |
|
||||
| Cost | The summary request reuses the prompt cache. A summary that is never used, for example because the session ends before the threshold, still costs tokens. |
|
||||
| Waits | If the summary is still being written when compaction is due, the session waits for it to finish. |
|
||||
| Falls back | Compaction summarizes as usual if the conversation since the held summary grew past about 40% of the threshold, or if the model, instructions, or start of the conversation changed. |
|
||||
| Skipped | Not used for native compaction, right after a model switch, or when a provider rejects a request as too long. After a break longer than the prompt cache lifetime, it starts from the next response instead. |
|
||||
| Storage | Held summaries live in memory and are lost on restart. |
|
||||
|
||||
## Providers
|
||||
|
||||
Reference in New Issue
Block a user