fix(core): size compaction requests from the provider's last count (#59441)

Co-authored-by: Kit Langton <kit.langton@gmail.com>

packages/core/src/session/compaction.ts

@@ -467,14 +467,20 @@ export const layer = Layer.effect(
       const headings = SUMMARY_TEMPLATE.split("\n").filter((line) => line.startsWith("##"))
       const filled = (text: string) => text.split("\n").some((line) => headings.includes(line.trim()))
       const prepared = yield* prepare(context, messages, budget)
+      // Size the unchanged request with the number that decided to compact: the last response's count plus an
+      // estimate of what came after it. Characters overstate a conversation whose old reasoning the provider drops.
+      // A hook can still make this request larger than that count; the provider then rejects it and it is resent as
+      // text. An overflow already proved the count low, so it keeps the character estimate.
+      const measured = trigger.reason === "overflow" ? undefined : measuredPrompt({ ...context, messages })
 
       // Hooks saw the request without the summary prompt, so it is appended here. A reply that ignores the
       // template gets one reminder before it counts as a failure.
-      const send = (request: LLMRequest) =>
+      const send = (request: LLMRequest, size?: Estimate) =>
         Effect.gen(function* () {
           const prompted = fitOutput(
             LLMRequest.update(request, { messages: [...request.messages, Message.user(prompt)] }),
             context.model.limit,
+            size && { ...size, estimated: size.estimated + Token.estimate(prompt) },
           )
           const reply = yield* stream(context, prompted, prepared.options, background)
           if (filled(reply.text)) return reply
@@ -482,6 +488,7 @@ export const layer = Layer.effect(
           const nudged = fitOutput(
             LLMRequest.update(prompted, { messages: [...prompted.messages, Message.user(NUDGE)] }),
             context.model.limit,
+            size && { ...size, estimated: size.estimated + Token.estimate(prompt) + Token.estimate(NUDGE) },
           )
           const retry = yield* stream(context, nudged, prepared.options, background)
           if (filled(retry.text)) return retry
@@ -501,13 +508,16 @@ export const layer = Layer.effect(
       // which still reuses the cache for that shorter prefix, and as text otherwise. Dropping exchanges to make room
       // for the tail would leave them in neither the summary nor the tail. Where 15% of the window is under
       // `RESERVE_MIN`, the ceiling equals `summaryRoom`, so automatic compaction always takes this fallback.
-      const fit = trigger.reason === "overflow" ? budget : Math.max(budget, summaryRoom(context.model.limit))
+      const fit =
+        trigger.reason === "overflow"
+          ? budget
+          : Math.max(budget, summaryRoom(context.model.limit, prepared.request.generation?.maxTokens))
       const result = yield* deliver(
         trigger,
         prepared,
         budget - overhead,
         send,
-        { fit: fit - overhead, history: withoutTail(prepared.request, keepID) },
+        { fit: fit - overhead, history: withoutTail(prepared.request, keepID), measured },
         background,
       )
       if (!keepID) return result
@@ -569,7 +579,8 @@ export const layer = Layer.effect(
 
     /**
      * Sends the request as-is if it is estimated to fit `shrink.fit`, else `shrink.history` as-is if it fits `target`,
-     * and otherwise `shrink.history` as text aimed at `target` (see `flattenAndDropOldest`).
+     * and otherwise `shrink.history` as text aimed at `target` (see `flattenAndDropOldest`). `shrink.measured`, when
+     * given, sizes the unchanged request until the provider rejects it; after that it is only sent as text.
      * Limits and estimates can be wrong, so until the provider rejects something as too long, a request that cannot
      * be made to fit is sent whole: unchanged, or as text once its media has to go. A "too long" rejection means the
      * estimate ran low, so the next attempts aim at 85%, 70%, 50%, then 35% of the first rejected request's estimate,
@@ -583,8 +594,11 @@ export const layer = Layer.effect(
       trigger: Trigger,
       prepared: Prepared,
       target: number,
-      send: (request: LLMRequest) => Effect.Effect<Result, AIError | Failure>,
-      shrink: { readonly fit: number; readonly history: LLMRequest } = { fit: target, history: prepared.request },
+      send: (request: LLMRequest, size?: Estimate) => Effect.Effect<Result, AIError | Failure>,
+      shrink: { readonly fit: number; readonly history: LLMRequest; readonly measured?: Estimate } = {
+        fit: target,
+        history: prepared.request,
+      },
       background?: Background,
     ): Effect.fn.Return<Result, Failure> {
       const context = trigger.context
@@ -597,9 +611,10 @@ export const layer = Layer.effect(
       let fit = shrink.fit
 
       while (true) {
+        const measured = rejections === 0 ? shrink.measured : undefined
         const unchanged = asText
           ? undefined
-          : estimateRequest(prepared.request) <= fit
+          : (measured ? total(measured) : estimateRequest(prepared.request)) <= fit
             ? prepared.request
             : estimateRequest(shrink.history) <= target
               ? shrink.history
@@ -622,7 +637,8 @@ export const layer = Layer.effect(
           })
         }
 
-        const attempt = yield* Effect.result(send(request))
+        const size = request === prepared.request ? measured : undefined
+        const attempt = yield* Effect.result(send(request, size))
         if (Result.isSuccess(attempt)) return attempt.success
         const cause = attempt.failure
         if (!(cause instanceof AIError)) return yield* Effect.fail(cause)
@@ -637,7 +653,9 @@ export const layer = Layer.effect(
         if (tooLarge || isContextOverflowFailure(cause)) {
           const step = SHRINK_STEPS[rejections]
           if (step === undefined) return yield* Effect.fail<Failure>({ error })
-          rejected ??= estimateRequest(request)
+          rejected ??= size ? total(size) : estimateRequest(request)
+          // The characters can still say the rejected request fits, so it only goes out again as text.
+          if (size) asText = true
           target = Math.floor(rejected * step)
           fit = target
           rejections++
@@ -1122,9 +1140,12 @@ const calculateCeiling = (limit: SessionContext.Loaded["model"]["limit"], buffer
   return window - Math.max(Math.floor(window * RESERVE_SHARE), window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0)
 }
 
-/** The largest summary request that leaves `RESERVE_MIN` of the window for the summary. Zero for an unknown window. */
-const summaryRoom = (limit: SessionContext.Loaded["model"]["limit"]) =>
-  Math.max(0, (limit.input || limit.context) - RESERVE_MIN)
+/**
+ * The largest summary request that leaves `RESERVE_MIN` of the window for the summary, or less when the request
+ * cannot generate that much. Zero for an unknown window.
+ */
+const summaryRoom = (limit: SessionContext.Loaded["model"]["limit"], output = RESERVE_MIN) =>
+  Math.max(0, (limit.input || limit.context) - Math.min(RESERVE_MIN, output))
 
 /**
  * Another provider's count may describe a native window this model cannot replay. History then holds the original
@@ -1137,6 +1158,19 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L
   message.tokens !== undefined &&
   message.tokens.input + message.tokens.cache.read + message.tokens.cache.write > 0
 
+/** The prompt size, only while the model, variant and agent that were measured still match; they shape the request. */
+const measuredPrompt = (context: SessionContext.Loaded) => {
+  const anchor = context.messages.findLast((message) => hasMeasuredPrompt(message, context.model.ref))
+  if (
+    anchor?.type !== "assistant" ||
+    anchor.model.id !== context.model.ref.id ||
+    anchor.model.variant !== context.model.ref.variant ||
+    anchor.agent !== context.agent.id
+  )
+    return undefined
+  return estimatePrompt(context)
+}
+
 /**
  * `prepare` sizes the output limit for a request of `budget` tokens, but `deliver` can send a larger one when it
  * cannot shrink the conversation further. Lower the limit to the room the request being sent leaves in the window,
@@ -1146,15 +1180,15 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L
  * window, so that margin alone would cut a summary near its target to the minimum; there it still gets up to
  * `SUMMARY_OUTPUT_FLOOR` of the room left without the margin.
  */
-const fitOutput = (request: LLMRequest, limit: SessionContext.Loaded["model"]["limit"]) => {
+const fitOutput = (request: LLMRequest, limit: SessionContext.Loaded["model"]["limit"], size?: Estimate) => {
   const current = request.generation?.maxTokens
   if (current === undefined) return request
-  const estimated = estimateRequest(request)
+  const input = size ?? { measured: 0, estimated: estimateRequest(request) }
   const maxTokens = Math.min(
     current,
     Math.max(
-      SessionModelRequest.outputLimit(limit, { measured: 0, estimated }),
-      Math.min(SUMMARY_OUTPUT_FLOOR, SessionModelRequest.outputLimit(limit, { measured: estimated, estimated: 0 })),
+      SessionModelRequest.outputLimit(limit, input),
+      Math.min(SUMMARY_OUTPUT_FLOOR, SessionModelRequest.outputLimit(limit, { measured: total(input), estimated: 0 })),
     ),
   )
   if (maxTokens === current) return request

packages/core/test/session-compaction.test.ts

@@ -4,6 +4,7 @@ import {
   LLMClient,
   LLMEvent,
   LanguageModel,
+  Message,
   ToolDefinition,
   type LLMRequest,
 } from "@opencode/ai/effect"
@@ -22,6 +23,7 @@ import { SessionHistory } from "@opencode/core/session/history"
 import { SessionMessage } from "@opencode/core/session/message"
 import { SessionModelRequest } from "@opencode/core/session/model-request"
 import { PluginHooks } from "@opencode/core/plugin/hooks"
+import { Token } from "@opencode/core/util/token"
 import { SessionProjector } from "@opencode/core/session/projector"
 import { SessionRunnerModel } from "@opencode/core/session/runner/model"
 import { toLLMMessages } from "@opencode/core/session/runner/to-llm-message"
@@ -46,6 +48,7 @@ import { testEffect } from "./lib/effect"
 let requests: LLMRequest[] = []
 /** Runs before a request's response streams; tests hold it back to keep a background summary in flight. */
 let beforeResponse: Effect.Effect<void> = Effect.void
+let reject: ((request: LLMRequest) => boolean) | undefined
 const model = LanguageModel.make({
   id: "summary-model",
   provider: "test",
@@ -64,6 +67,9 @@ const cost = [
 const client = Layer.mock(LLMClient.Service)({
   stream: (request: LLMRequest) => {
     requests.push(request)
+    if (reject?.(request)) {
+      return Stream.make(LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }))
+    }
     return Stream.fromEffect(beforeResponse).pipe(
       Stream.drain,
       Stream.concat(
@@ -1001,6 +1007,185 @@ const enablePrecompute = Effect.gen(function* () {
   return compaction
 })
 
+const summaryWindow = resolved.limit.context
+const summaryBuffer = summaryWindow / 8
+const summaryCeiling = summaryWindow - summaryBuffer
+const summaryInput = summaryCeiling * 0.9 // Past precompute's trigger, below the compaction ceiling.
+const tokenText = (tokens: number) => "text".repeat(Math.ceil(tokens / Token.estimate("text")))
+const oldReasoning = (input = summaryInput) => ({
+  ...measured("Earlier answer", input),
+  content: [{ type: "reasoning" as const, text: tokenText(summaryWindow * 3) }],
+})
+const summaryFixture = Effect.fnUntraced(function* (messages: readonly SessionMessage.Info[]) {
+  requests = []
+  const context = {
+    ...loaded(yield* insertSession(Session.ID.create()), messages),
+    tools: {
+      definitions: [ToolDefinition.make({ name: "read", description: "Read files", inputSchema: { type: "object" } })],
+      execute: () => Effect.die("Compaction must not execute tools"),
+    },
+  }
+  const compaction = yield* SessionCompaction.Service
+  yield* compaction.transform((editor) => editor.configure({ buffer: summaryBuffer, precompute: true }))
+  return {
+    context,
+    compaction,
+    run: (input = context) =>
+      compaction.compact({ reason: "manual", context: input, inputID: SessionMessage.ID.create() }),
+  }
+})
+
+for (const background of [false, true]) {
+  it.effect(`summary cache prefix uses measured input with old reasoning (background=${background})`, () =>
+    Effect.gen(function* () {
+      const messages = [
+        textUser("Earlier question"),
+        oldReasoning(),
+        textUser("Latest question"),
+        measured("Latest answer", summaryInput),
+      ]
+      const fixture = yield* summaryFixture(messages)
+      const builder = yield* SessionModelRequest.Service
+      const normal = yield* builder.primary({
+        ...fixture.context,
+        agent: fixture.context.agent.id,
+        ...SessionModelRequest.baseTranscript({
+          ...fixture.context,
+          agent: fixture.context.agent.info,
+          messages: background ? messages.slice(0, -1) : messages,
+        }),
+      })
+      if (background) yield* fixture.compaction.precompute(Effect.succeed(fixture.context))
+      expect(yield* fixture.run()).toEqual({ status: "completed" })
+      expect(requests).toHaveLength(1)
+      expect(requests[0]).toMatchObject({
+        system: normal.request.system,
+        tools: normal.request.tools,
+        messages: [...normal.request.messages, Message.user(SessionCompaction.buildPrompt(false))],
+        generation: { maxTokens: summaryBuffer },
+      })
+    }),
+  )
+}
+
+it.effect("summary cache prefix still flattens genuinely oversized measured input", () =>
+  Effect.gen(function* () {
+    const fixture = yield* summaryFixture([
+      textUser(tokenText(summaryCeiling)),
+      measured("Latest answer", summaryWindow + summaryBuffer),
+    ])
+    expect(yield* fixture.run()).toEqual({ status: "completed" })
+    expect(requests).toHaveLength(1)
+    expect(requests[0].tools).toEqual([])
+    expect(requests[0].messages).toHaveLength(2)
+    expect(JSON.stringify(requests[0].messages[0])).toContain("characters omitted")
+  }),
+)
+
+it.effect("summary measured recovery shrinks from the rejected measured size", () =>
+  Effect.gen(function* () {
+    const text = tokenText(summaryCeiling / 2)
+    const fixture = yield* summaryFixture([
+      textUser(text),
+      oldReasoning(summaryCeiling / 2),
+      textUser(text),
+      measured("Latest answer", summaryCeiling),
+    ])
+    // Reject until the original exchange is actually shortened, without duplicating a token estimator in the mock.
+    reject = (request) =>
+      request.messages.some((message) =>
+        message.content.some((part) => part.type === "text" && part.text.includes(text)),
+      )
+    yield* Effect.addFinalizer(() =>
+      Effect.sync(() => {
+        reject = undefined
+      }),
+    )
+    expect(yield* fixture.run()).toEqual({ status: "completed" })
+    // The retained exchange fits the 85% and 70% targets; 50% of the measured input forces truncation.
+    expect(requests).toHaveLength(4)
+    expect(JSON.stringify(requests.at(-1)?.messages)).toContain("characters omitted")
+  }),
+)
+
+it.effect("summary cache prefix survives hooks that shape every request alike", () =>
+  Effect.gen(function* () {
+    const fixture = yield* summaryFixture([
+      textUser("Earlier question"),
+      oldReasoning(),
+      textUser("Latest question"),
+      measured("Latest answer", summaryInput),
+    ])
+    // Built-in tool plugins register one hook for both normal and compaction requests.
+    const hooks = yield* PluginHooks.Service
+    const describe = (event: { tools: Record<string, { description: string }> }) =>
+      Effect.sync(() => {
+        if (event.tools.read) event.tools.read.description = "Read files from the project"
+      })
+    yield* hooks.register("session", "context", describe)
+    yield* hooks.register("session", "compaction", describe)
+    expect(yield* fixture.run()).toEqual({ status: "completed" })
+    expect(requests).toHaveLength(1)
+    expect(requests[0].tools.map((tool) => tool.description)).toEqual(["Read files from the project"])
+  }),
+)
+
+it.effect("summary resends as text when a hook makes the request larger than the measured count", () =>
+  Effect.gen(function* () {
+    const fixture = yield* summaryFixture([
+      textUser("Earlier question"),
+      oldReasoning(),
+      textUser("Latest question"),
+      measured("Latest answer", summaryInput),
+    ])
+    const hooks = yield* PluginHooks.Service
+    yield* hooks.register("session", "compaction", (event) =>
+      Effect.sync(() => {
+        event.system.push({ type: "text", text: "Plugin guidance" })
+      }),
+    )
+    // The provider rejects the unchanged request, which the measured count said would fit.
+    reject = (request) => request.tools.length > 0
+    yield* Effect.addFinalizer(() =>
+      Effect.sync(() => {
+        reject = undefined
+      }),
+    )
+    expect(yield* fixture.run()).toEqual({ status: "completed" })
+    expect(requests).toHaveLength(2)
+    expect(requests[0].tools).toEqual(fixture.context.tools.definitions)
+    expect(requests[1].tools).toEqual([])
+  }),
+)
+
+it.effect("summary measured recovery reuses usage only for the matching agent", () =>
+  Effect.gen(function* () {
+    const fixture = yield* summaryFixture([textUser("Earlier question"), oldReasoning()])
+    const agent = Agent.ID.make("other")
+    const context = {
+      ...fixture.context,
+      agent: {
+        id: agent,
+        info: { ...Agent.Info.default(agent), system: tokenText(summaryWindow - summaryBuffer / 5) },
+      },
+    }
+    expect(yield* fixture.run(context)).toEqual({ status: "completed" })
+    expect(requests).toHaveLength(1)
+    expect(requests[0].generation?.maxTokens).toBeLessThanOrEqual(summaryBuffer / 5)
+    // Switching back restores the measured prefix; discarding usage for every agent is not sufficient.
+    expect(yield* fixture.run()).toEqual({ status: "completed" })
+    expect(requests).toHaveLength(2)
+    expect(requests[1]).toMatchObject({
+      tools: fixture.context.tools.definitions,
+      messages: [
+        ...toLLMMessages(fixture.context.messages, resolved.ref),
+        Message.user(SessionCompaction.buildPrompt(false)),
+      ],
+      generation: { maxTokens: summaryBuffer },
+    })
+  }),
+)
+
 it.effect("background summaries stay off unless enabled", () =>
   Effect.gen(function* () {
     requests = []