fix(core): fit compaction output limit to the request actually sent (#55613)

This commit is contained in:
Aiden Cline
2026-10-15 01:57:46 -05:00
committed by GitHub
Unverified
parent 7ced591c3c
commit ae8d59f8b5
3 changed files with 68 additions and 3 deletions
+27 -2
View File
@@ -288,11 +288,17 @@ export const layer = Layer.effect(
// template gets one reminder before it counts as a failure.
const send = (request: LLMRequest) =>
Effect.gen(function* () {
const prompted = LLMRequest.update(request, { messages: [...request.messages, Message.user(prompt)] })
const prompted = fitOutput(
LLMRequest.update(request, { messages: [...request.messages, Message.user(prompt)] }),
context.model.limit,
)
const reply = yield* stream(context, prompted, prepared.options)
if (filled(reply.text)) return { ...reply, recent: split.recent }
const nudged = LLMRequest.update(prompted, { messages: [...prompted.messages, Message.user(NUDGE)] })
const nudged = fitOutput(
LLMRequest.update(prompted, { messages: [...prompted.messages, Message.user(NUDGE)] }),
context.model.limit,
)
const retry = yield* stream(context, nudged, prepared.options)
if (filled(retry.text)) return { ...retry, recent: split.recent }
return yield* Effect.fail<Failure>({
@@ -922,6 +928,25 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L
message.tokens !== undefined &&
message.tokens.input + message.tokens.cache.read + message.tokens.cache.write > 0
/**
* `prepare` sizes the output limit for a request of `budget` tokens, but `deliver` can send a larger one when it
* cannot shrink the conversation further. Lower the limit to the room the request being sent leaves in the window,
* so the input and the requested output never exceed the window by our own estimate.
*
* Like `budget` in `prepare`, the estimate counts as measured, without the margin primary requests add for estimates:
* `deliver` aims at about 90% of the window, so that margin would cut every summary near its target to the minimum.
*/
const fitOutput = (request: LLMRequest, limit: SessionContext.Loaded["model"]["limit"]) => {
const current = request.generation?.maxTokens
if (current === undefined) return request
const maxTokens = Math.min(
current,
SessionModelRequest.outputLimit(limit, "compaction", { measured: estimateRequest(request), estimated: 0 }),
)
if (maxTokens === current) return request
return LLMRequest.update(request, { generation: { ...request.generation, maxTokens } })
}
const estimateRequest = (request: Pick<LLMRequest, "system" | "tools" | "messages">) =>
request.system.reduce((sum, part) => sum + Token.estimate(part.text), 0) +
request.tools.reduce((sum, tool) => sum + estimateTool(tool), 0) +
+1 -1
View File
@@ -85,7 +85,7 @@ export interface Input {
}
/** The default output limit: the catalog limit, fitted to the room the prompt leaves in the context window. */
const outputLimit = (
export const outputLimit = (
limit: Model.Info["limit"],
kind: "primary" | "compaction",
inputTokens?: Input["inputTokens"],
@@ -498,6 +498,46 @@ it.effect("manual compaction summarizes short context instead of no-op", () =>
}),
)
it.effect("compaction lowers the output limit when it sends a request it could not shrink", () =>
Effect.gen(function* () {
requests = []
const session = yield* insertSession(Session.ID.make("ses_unshrinkable_compaction"))
// One exchange estimated at 250k tokens, past the 200k window, so no older exchange can be dropped.
const messages = [
SessionMessage.User.make({
id: SessionMessage.ID.create(),
type: "user",
text: "x".repeat(1_000_000),
time: { created: DateTime.makeUnsafe(0) },
}),
]
expect(yield* compactManually(session, messages)).toEqual({ status: "completed" })
expect(requests).toHaveLength(1)
expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 1_024 }))
}),
)
it.effect("compaction keeps the output limit for a request that fits its target", () =>
Effect.gen(function* () {
requests = []
const session = yield* insertSession(Session.ID.make("ses_fitting_compaction"))
// About 170k estimated tokens, under the 180k target, leaving 30k of the 200k window.
const messages = [
SessionMessage.User.make({
id: SessionMessage.ID.create(),
type: "user",
text: "x".repeat(680_000),
time: { created: DateTime.makeUnsafe(0) },
}),
]
expect(yield* compactManually(session, messages)).toEqual({ status: "completed" })
expect(requests).toHaveLength(1)
expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 20_000 }))
}),
)
it.effect("compaction hooks can supply the summary instead of the model", () =>
Effect.gen(function* () {
requests = []