mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-20 18:57:17 +03:00
fix(ai): fit thinking budgets to the output limit (#51157)
This commit is contained in:
committed by
GitHub
Unverified
parent
14aaf91e65
commit
c903774556
@@ -70,7 +70,11 @@ export const protocol = Protocol.make({
|
||||
return {
|
||||
...(yield* OpenAIChat.protocol.body.from(req)),
|
||||
enable_thinking: opts.enableThinking,
|
||||
thinking_budget: opts.thinkingBudget,
|
||||
// Alibaba also rejects an explicit budget that is not below `max_completion_tokens`.
|
||||
thinking_budget:
|
||||
opts.thinkingBudget === undefined
|
||||
? undefined
|
||||
: ProviderShared.fitThinkingBudget(opts.thinkingBudget, req.generation?.maxTokens),
|
||||
preserve_thinking: opts.preserveThinking,
|
||||
clear_thinking: opts.clearThinking,
|
||||
thinking: opts.thinking,
|
||||
|
||||
@@ -26,18 +26,21 @@ export const protocol = Protocol.make({
|
||||
from: Effect.fn("AlibabaMessages.fromRequest")(function* (req) {
|
||||
const opts = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Options))(req.providerOptions ?? {})
|
||||
// Model Studio accepts enabled thinking without Anthropic's mandatory token budget.
|
||||
const body = yield* AnthropicMessages.protocol.body.from(
|
||||
LLMRequest.update(req, {
|
||||
providerOptions: { ...req.providerOptions, thinking: undefined },
|
||||
}),
|
||||
)
|
||||
const budget = opts.thinking?.budgetTokens ?? opts.thinking?.budget_tokens
|
||||
return {
|
||||
...(yield* AnthropicMessages.protocol.body.from(
|
||||
LLMRequest.update(req, {
|
||||
providerOptions: { ...req.providerOptions, thinking: undefined },
|
||||
}),
|
||||
)),
|
||||
...body,
|
||||
thinking:
|
||||
opts.thinking === undefined
|
||||
? undefined
|
||||
: {
|
||||
type: opts.thinking.type,
|
||||
budget_tokens: opts.thinking.budgetTokens ?? opts.thinking.budget_tokens,
|
||||
budget_tokens:
|
||||
budget === undefined ? undefined : ProviderShared.fitThinkingBudget(budget, body.max_tokens),
|
||||
},
|
||||
}
|
||||
}),
|
||||
|
||||
@@ -38,6 +38,7 @@ const ADAPTER = "anthropic-messages"
|
||||
export const DEFAULT_BASE_URL = "https://api.anthropic.com/v1"
|
||||
export const PATH = "/messages"
|
||||
export const DEFAULT_MAX_TOKENS = 32_000
|
||||
const MIN_THINKING_BUDGET = 1_024
|
||||
const DEFAULT_EFFORT = "high"
|
||||
|
||||
const SSE_EVENTS = new Set([
|
||||
@@ -1027,6 +1028,15 @@ const applyThinkingBindingDefault = (model: LLMRequest["model"], thinking: Anthr
|
||||
}
|
||||
}
|
||||
|
||||
// Anthropic also requires an explicit thinking budget below `max_tokens` and at or above its minimum.
|
||||
const fitThinking = (thinking: AnthropicThinking | undefined, maxTokens: number) =>
|
||||
thinking?.type === "enabled"
|
||||
? {
|
||||
...thinking,
|
||||
budget_tokens: ProviderShared.fitThinkingBudget(thinking.budget_tokens, maxTokens, MIN_THINKING_BUDGET),
|
||||
}
|
||||
: thinking
|
||||
|
||||
const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (request: LLMRequest) {
|
||||
const options = yield* decodeOptions(request.providerOptions ?? {})
|
||||
const management = options.contextManagement
|
||||
@@ -1064,6 +1074,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
|
||||
}
|
||||
const output_config =
|
||||
updates.effort === undefined && format === undefined ? undefined : { effort: updates.effort, format }
|
||||
const maxTokens = generation?.maxTokens ?? DEFAULT_MAX_TOKENS
|
||||
const body = {
|
||||
model: request.model.id,
|
||||
system,
|
||||
@@ -1071,12 +1082,12 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
|
||||
tools,
|
||||
tool_choice: toolChoice,
|
||||
stream: true as const,
|
||||
max_tokens: generation?.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
max_tokens: maxTokens,
|
||||
temperature: generation?.temperature,
|
||||
top_p: generation?.topP,
|
||||
top_k: generation?.topK,
|
||||
stop_sequences: generation?.stop,
|
||||
thinking: applyThinkingBindingDefault(request.model, options.thinking),
|
||||
thinking: applyThinkingBindingDefault(request.model, fitThinking(options.thinking, maxTokens)),
|
||||
output_config,
|
||||
// top-level passthrough per SDK MessageCreateParamsBase:4638,4643,4649,4654,4670
|
||||
cache_control: options.cache_control ?? options.cacheControl,
|
||||
|
||||
@@ -29,6 +29,8 @@ import { ToolSchemaProjection } from "./utils/tool-schema.js"
|
||||
const ADAPTER = "gemini"
|
||||
// Google documents this sentinel for replaying Gemini 3 function calls after their original signature was lost.
|
||||
const SKIP_THOUGHT_SIGNATURE_VALIDATOR = "skip_thought_signature_validator"
|
||||
// Gemini 2.5 rejects a budget under the model's minimum: 512 on Flash-Lite, the highest, and 128 on Pro.
|
||||
const MIN_THINKING_BUDGET = 512
|
||||
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
||||
// Gemini 3 rejects replayed function calls without a thought signature. Google's SDKs avoid that in normal chats by
|
||||
@@ -452,10 +454,22 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
|
||||
presencePenalty: generation?.presencePenalty,
|
||||
seed: generation?.seed,
|
||||
stopSequences: generation?.stop,
|
||||
// Gemini accepts a budget above `maxOutputTokens`, but thinking then leaves the answer empty.
|
||||
thinkingConfig:
|
||||
options.thinkingConfig === undefined
|
||||
? undefined
|
||||
: { ...options.thinkingConfig, includeThoughts: options.thinkingConfig.includeThoughts ?? true },
|
||||
: {
|
||||
...options.thinkingConfig,
|
||||
includeThoughts: options.thinkingConfig.includeThoughts ?? true,
|
||||
thinkingBudget:
|
||||
options.thinkingConfig.thinkingBudget === undefined
|
||||
? undefined
|
||||
: ProviderShared.fitThinkingBudget(
|
||||
options.thinkingConfig.thinkingBudget,
|
||||
generation?.maxTokens,
|
||||
MIN_THINKING_BUDGET,
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -110,6 +110,14 @@ export const sumTokens = (...values: ReadonlyArray<number | undefined>): number
|
||||
return values.reduce((acc: number, value) => acc + (value ?? 0), 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Caps an explicit thinking budget at half the output limit. Thinking counts against the output limit, so a budget
|
||||
* near it leaves the answer, a tool call, or a summary without room. Smaller budgets, special values such as `-1` and
|
||||
* `0`, and requests without an output limit pass through unchanged.
|
||||
*/
|
||||
export const fitThinkingBudget = (budget: number, maxTokens: number | undefined, minimum = 1) =>
|
||||
maxTokens === undefined || budget <= maxTokens / 2 ? budget : Math.max(minimum, Math.floor(maxTokens / 2))
|
||||
|
||||
export const eventError = (route: string, message: string, body?: string, cause?: unknown) =>
|
||||
new AIError({
|
||||
reason: new InvalidProviderOutputError({ route, message, body, cause }),
|
||||
|
||||
@@ -216,6 +216,33 @@ it.effect("Alibaba keeps native reasoning controls and future efforts on their s
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("Alibaba fits explicit thinking budgets to half the output limit", () =>
|
||||
Effect.gen(function* () {
|
||||
const provider = Alibaba.configure({ region: "ap-southeast-1", apiKey: "fixture" })
|
||||
const chat = (maxTokens?: number) =>
|
||||
compileRequest(
|
||||
LLM.request({
|
||||
model: provider.chat("qwen3.7-plus"),
|
||||
prompt: "hi",
|
||||
...(maxTokens === undefined ? {} : { generation: { maxTokens } }),
|
||||
providerOptions: { enableThinking: true, thinkingBudget: 131_071 },
|
||||
}),
|
||||
).pipe(Effect.map((prepared) => prepared.body.thinking_budget))
|
||||
const messages = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: provider.messages("qwen3.7-plus"),
|
||||
prompt: "hi",
|
||||
generation: { maxTokens: 32_000 },
|
||||
providerOptions: { thinking: { type: "enabled", budgetTokens: 131_071 } },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(yield* chat(32_000)).toBe(16_000)
|
||||
expect(yield* chat()).toBe(131_071)
|
||||
expect(messages.body.thinking).toEqual({ type: "enabled", budget_tokens: 16_000 })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("Alibaba validates malformed options before execution", () =>
|
||||
Effect.gen(function* () {
|
||||
const provider = Alibaba.configure({ region: "ap-southeast-1", apiKey: "fixture" })
|
||||
|
||||
@@ -148,11 +148,13 @@ describe("Anthropic Messages route", () => {
|
||||
Effect.gen(function* () {
|
||||
const enabled = yield* compileRequest(
|
||||
LLMRequest.update(request, {
|
||||
generation: { maxTokens: 4_096 },
|
||||
providerOptions: { thinking: { type: "enabled", budgetTokens: 1_024 } },
|
||||
}),
|
||||
)
|
||||
const legacy = yield* compileRequest(
|
||||
LLMRequest.update(request, {
|
||||
generation: { maxTokens: 4_096 },
|
||||
providerOptions: { thinking: { type: "enabled", budget_tokens: 2_048 } },
|
||||
}),
|
||||
)
|
||||
@@ -168,6 +170,22 @@ describe("Anthropic Messages route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fits the thinking budget to half the output limit", () =>
|
||||
Effect.gen(function* () {
|
||||
const thinking = (maxTokens: number) =>
|
||||
compileRequest(
|
||||
LLMRequest.update(request, {
|
||||
generation: { maxTokens },
|
||||
providerOptions: { thinking: { type: "enabled", budgetTokens: 31_999 } },
|
||||
}),
|
||||
).pipe(Effect.map((prepared) => prepared.body.thinking))
|
||||
|
||||
expect(yield* thinking(64_000)).toEqual({ type: "enabled", budget_tokens: 31_999 })
|
||||
expect(yield* thinking(20_000)).toEqual({ type: "enabled", budget_tokens: 10_000 })
|
||||
expect(yield* thinking(1_500)).toEqual({ type: "enabled", budget_tokens: 1_024 })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects enabled thinking without a budget", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* compileRequest(
|
||||
|
||||
@@ -90,6 +90,23 @@ describe("Gemini route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fits the thinking budget to half the output limit", () =>
|
||||
Effect.gen(function* () {
|
||||
const thinkingBudget = (budget: number, maxTokens = 32_000) =>
|
||||
compileRequest(
|
||||
LLMRequest.update(request, {
|
||||
generation: { maxTokens },
|
||||
providerOptions: { thinkingConfig: { thinkingBudget: budget } },
|
||||
}),
|
||||
).pipe(Effect.map((prepared) => prepared.body.generationConfig?.thinkingConfig?.thinkingBudget))
|
||||
|
||||
expect(yield* thinkingBudget(32_768)).toBe(16_000)
|
||||
expect(yield* thinkingBudget(8_000)).toBe(8_000)
|
||||
expect(yield* thinkingBudget(-1)).toBe(-1)
|
||||
expect(yield* thinkingBudget(8_192, 1_000)).toBe(512)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("forwards standard Gemini generation options", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
|
||||
Reference in New Issue
Block a user