mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
fix(core): fit compaction output limit to the request actually sent (#55613)
This commit is contained in:
committed by
GitHub
Unverified
parent
7ced591c3c
commit
ae8d59f8b5
@@ -288,11 +288,17 @@ export const layer = Layer.effect(
|
||||
// template gets one reminder before it counts as a failure.
|
||||
const send = (request: LLMRequest) =>
|
||||
Effect.gen(function* () {
|
||||
const prompted = LLMRequest.update(request, { messages: [...request.messages, Message.user(prompt)] })
|
||||
const prompted = fitOutput(
|
||||
LLMRequest.update(request, { messages: [...request.messages, Message.user(prompt)] }),
|
||||
context.model.limit,
|
||||
)
|
||||
const reply = yield* stream(context, prompted, prepared.options)
|
||||
if (filled(reply.text)) return { ...reply, recent: split.recent }
|
||||
|
||||
const nudged = LLMRequest.update(prompted, { messages: [...prompted.messages, Message.user(NUDGE)] })
|
||||
const nudged = fitOutput(
|
||||
LLMRequest.update(prompted, { messages: [...prompted.messages, Message.user(NUDGE)] }),
|
||||
context.model.limit,
|
||||
)
|
||||
const retry = yield* stream(context, nudged, prepared.options)
|
||||
if (filled(retry.text)) return { ...retry, recent: split.recent }
|
||||
return yield* Effect.fail<Failure>({
|
||||
@@ -922,6 +928,25 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L
|
||||
message.tokens !== undefined &&
|
||||
message.tokens.input + message.tokens.cache.read + message.tokens.cache.write > 0
|
||||
|
||||
/**
|
||||
* `prepare` sizes the output limit for a request of `budget` tokens, but `deliver` can send a larger one when it
|
||||
* cannot shrink the conversation further. Lower the limit to the room the request being sent leaves in the window,
|
||||
* so the input and the requested output never exceed the window by our own estimate.
|
||||
*
|
||||
* Like `budget` in `prepare`, the estimate counts as measured, without the margin primary requests add for estimates:
|
||||
* `deliver` aims at about 90% of the window, so that margin would cut every summary near its target to the minimum.
|
||||
*/
|
||||
const fitOutput = (request: LLMRequest, limit: SessionContext.Loaded["model"]["limit"]) => {
|
||||
const current = request.generation?.maxTokens
|
||||
if (current === undefined) return request
|
||||
const maxTokens = Math.min(
|
||||
current,
|
||||
SessionModelRequest.outputLimit(limit, "compaction", { measured: estimateRequest(request), estimated: 0 }),
|
||||
)
|
||||
if (maxTokens === current) return request
|
||||
return LLMRequest.update(request, { generation: { ...request.generation, maxTokens } })
|
||||
}
|
||||
|
||||
const estimateRequest = (request: Pick<LLMRequest, "system" | "tools" | "messages">) =>
|
||||
request.system.reduce((sum, part) => sum + Token.estimate(part.text), 0) +
|
||||
request.tools.reduce((sum, tool) => sum + estimateTool(tool), 0) +
|
||||
|
||||
@@ -85,7 +85,7 @@ export interface Input {
|
||||
}
|
||||
|
||||
/** The default output limit: the catalog limit, fitted to the room the prompt leaves in the context window. */
|
||||
const outputLimit = (
|
||||
export const outputLimit = (
|
||||
limit: Model.Info["limit"],
|
||||
kind: "primary" | "compaction",
|
||||
inputTokens?: Input["inputTokens"],
|
||||
|
||||
@@ -498,6 +498,46 @@ it.effect("manual compaction summarizes short context instead of no-op", () =>
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("compaction lowers the output limit when it sends a request it could not shrink", () =>
|
||||
Effect.gen(function* () {
|
||||
requests = []
|
||||
const session = yield* insertSession(Session.ID.make("ses_unshrinkable_compaction"))
|
||||
// One exchange estimated at 250k tokens, past the 200k window, so no older exchange can be dropped.
|
||||
const messages = [
|
||||
SessionMessage.User.make({
|
||||
id: SessionMessage.ID.create(),
|
||||
type: "user",
|
||||
text: "x".repeat(1_000_000),
|
||||
time: { created: DateTime.makeUnsafe(0) },
|
||||
}),
|
||||
]
|
||||
|
||||
expect(yield* compactManually(session, messages)).toEqual({ status: "completed" })
|
||||
expect(requests).toHaveLength(1)
|
||||
expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 1_024 }))
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("compaction keeps the output limit for a request that fits its target", () =>
|
||||
Effect.gen(function* () {
|
||||
requests = []
|
||||
const session = yield* insertSession(Session.ID.make("ses_fitting_compaction"))
|
||||
// About 170k estimated tokens, under the 180k target, leaving 30k of the 200k window.
|
||||
const messages = [
|
||||
SessionMessage.User.make({
|
||||
id: SessionMessage.ID.create(),
|
||||
type: "user",
|
||||
text: "x".repeat(680_000),
|
||||
time: { created: DateTime.makeUnsafe(0) },
|
||||
}),
|
||||
]
|
||||
|
||||
expect(yield* compactManually(session, messages)).toEqual({ status: "completed" })
|
||||
expect(requests).toHaveLength(1)
|
||||
expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 20_000 }))
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("compaction hooks can supply the summary instead of the model", () =>
|
||||
Effect.gen(function* () {
|
||||
requests = []
|
||||
|
||||
Reference in New Issue
Block a user