fix(ai): fit thinking budgets to the output limit (#51157)

This commit is contained in:
Aiden Cline
2026-09-24 12:58:54 -05:00
committed by GitHub
Unverified
parent 14aaf91e65
commit c903774556
8 changed files with 112 additions and 10 deletions
+5 -1
View File
@@ -70,7 +70,11 @@ export const protocol = Protocol.make({
return {
...(yield* OpenAIChat.protocol.body.from(req)),
enable_thinking: opts.enableThinking,
thinking_budget: opts.thinkingBudget,
// Alibaba also rejects an explicit budget that is not below `max_completion_tokens`.
thinking_budget:
opts.thinkingBudget === undefined
? undefined
: ProviderShared.fitThinkingBudget(opts.thinkingBudget, req.generation?.maxTokens),
preserve_thinking: opts.preserveThinking,
clear_thinking: opts.clearThinking,
thinking: opts.thinking,
@@ -26,18 +26,21 @@ export const protocol = Protocol.make({
from: Effect.fn("AlibabaMessages.fromRequest")(function* (req) {
const opts = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Options))(req.providerOptions ?? {})
// Model Studio accepts enabled thinking without Anthropic's mandatory token budget.
const body = yield* AnthropicMessages.protocol.body.from(
LLMRequest.update(req, {
providerOptions: { ...req.providerOptions, thinking: undefined },
}),
)
const budget = opts.thinking?.budgetTokens ?? opts.thinking?.budget_tokens
return {
...(yield* AnthropicMessages.protocol.body.from(
LLMRequest.update(req, {
providerOptions: { ...req.providerOptions, thinking: undefined },
}),
)),
...body,
thinking:
opts.thinking === undefined
? undefined
: {
type: opts.thinking.type,
budget_tokens: opts.thinking.budgetTokens ?? opts.thinking.budget_tokens,
budget_tokens:
budget === undefined ? undefined : ProviderShared.fitThinkingBudget(budget, body.max_tokens),
},
}
}),
@@ -38,6 +38,7 @@ const ADAPTER = "anthropic-messages"
export const DEFAULT_BASE_URL = "https://api.anthropic.com/v1"
export const PATH = "/messages"
export const DEFAULT_MAX_TOKENS = 32_000
const MIN_THINKING_BUDGET = 1_024
const DEFAULT_EFFORT = "high"
const SSE_EVENTS = new Set([
@@ -1027,6 +1028,15 @@ const applyThinkingBindingDefault = (model: LLMRequest["model"], thinking: Anthr
}
}
// Anthropic also requires an explicit thinking budget below `max_tokens` and at or above its minimum.
const fitThinking = (thinking: AnthropicThinking | undefined, maxTokens: number) =>
thinking?.type === "enabled"
? {
...thinking,
budget_tokens: ProviderShared.fitThinkingBudget(thinking.budget_tokens, maxTokens, MIN_THINKING_BUDGET),
}
: thinking
const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (request: LLMRequest) {
const options = yield* decodeOptions(request.providerOptions ?? {})
const management = options.contextManagement
@@ -1064,6 +1074,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
}
const output_config =
updates.effort === undefined && format === undefined ? undefined : { effort: updates.effort, format }
const maxTokens = generation?.maxTokens ?? DEFAULT_MAX_TOKENS
const body = {
model: request.model.id,
system,
@@ -1071,12 +1082,12 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
tools,
tool_choice: toolChoice,
stream: true as const,
max_tokens: generation?.maxTokens ?? DEFAULT_MAX_TOKENS,
max_tokens: maxTokens,
temperature: generation?.temperature,
top_p: generation?.topP,
top_k: generation?.topK,
stop_sequences: generation?.stop,
thinking: applyThinkingBindingDefault(request.model, options.thinking),
thinking: applyThinkingBindingDefault(request.model, fitThinking(options.thinking, maxTokens)),
output_config,
// top-level passthrough per SDK MessageCreateParamsBase:4638,4643,4649,4654,4670
cache_control: options.cache_control ?? options.cacheControl,
+15 -1
View File
@@ -29,6 +29,8 @@ import { ToolSchemaProjection } from "./utils/tool-schema.js"
const ADAPTER = "gemini"
// Google documents this sentinel for replaying Gemini 3 function calls after their original signature was lost.
const SKIP_THOUGHT_SIGNATURE_VALIDATOR = "skip_thought_signature_validator"
// Gemini 2.5 rejects a budget under the model's minimum: 512 on Flash-Lite, the highest, and 128 on Pro.
const MIN_THINKING_BUDGET = 512
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
// Gemini 3 rejects replayed function calls without a thought signature. Google's SDKs avoid that in normal chats by
@@ -452,10 +454,22 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
presencePenalty: generation?.presencePenalty,
seed: generation?.seed,
stopSequences: generation?.stop,
// Gemini accepts a budget above `maxOutputTokens`, but thinking then leaves the answer empty.
thinkingConfig:
options.thinkingConfig === undefined
? undefined
: { ...options.thinkingConfig, includeThoughts: options.thinkingConfig.includeThoughts ?? true },
: {
...options.thinkingConfig,
includeThoughts: options.thinkingConfig.includeThoughts ?? true,
thinkingBudget:
options.thinkingConfig.thinkingBudget === undefined
? undefined
: ProviderShared.fitThinkingBudget(
options.thinkingConfig.thinkingBudget,
generation?.maxTokens,
MIN_THINKING_BUDGET,
),
},
}
return {
+8
View File
@@ -110,6 +110,14 @@ export const sumTokens = (...values: ReadonlyArray<number | undefined>): number
return values.reduce((acc: number, value) => acc + (value ?? 0), 0)
}
/**
* Caps an explicit thinking budget at half the output limit. Thinking counts against the output limit, so a budget
* near it leaves the answer, a tool call, or a summary without room. Smaller budgets, special values such as `-1` and
* `0`, and requests without an output limit pass through unchanged.
*/
export const fitThinkingBudget = (budget: number, maxTokens: number | undefined, minimum = 1) =>
maxTokens === undefined || budget <= maxTokens / 2 ? budget : Math.max(minimum, Math.floor(maxTokens / 2))
export const eventError = (route: string, message: string, body?: string, cause?: unknown) =>
new AIError({
reason: new InvalidProviderOutputError({ route, message, body, cause }),
+27
View File
@@ -216,6 +216,33 @@ it.effect("Alibaba keeps native reasoning controls and future efforts on their s
}),
)
it.effect("Alibaba fits explicit thinking budgets to half the output limit", () =>
Effect.gen(function* () {
const provider = Alibaba.configure({ region: "ap-southeast-1", apiKey: "fixture" })
const chat = (maxTokens?: number) =>
compileRequest(
LLM.request({
model: provider.chat("qwen3.7-plus"),
prompt: "hi",
...(maxTokens === undefined ? {} : { generation: { maxTokens } }),
providerOptions: { enableThinking: true, thinkingBudget: 131_071 },
}),
).pipe(Effect.map((prepared) => prepared.body.thinking_budget))
const messages = yield* compileRequest(
LLM.request({
model: provider.messages("qwen3.7-plus"),
prompt: "hi",
generation: { maxTokens: 32_000 },
providerOptions: { thinking: { type: "enabled", budgetTokens: 131_071 } },
}),
)
expect(yield* chat(32_000)).toBe(16_000)
expect(yield* chat()).toBe(131_071)
expect(messages.body.thinking).toEqual({ type: "enabled", budget_tokens: 16_000 })
}),
)
it.effect("Alibaba validates malformed options before execution", () =>
Effect.gen(function* () {
const provider = Alibaba.configure({ region: "ap-southeast-1", apiKey: "fixture" })
@@ -148,11 +148,13 @@ describe("Anthropic Messages route", () => {
Effect.gen(function* () {
const enabled = yield* compileRequest(
LLMRequest.update(request, {
generation: { maxTokens: 4_096 },
providerOptions: { thinking: { type: "enabled", budgetTokens: 1_024 } },
}),
)
const legacy = yield* compileRequest(
LLMRequest.update(request, {
generation: { maxTokens: 4_096 },
providerOptions: { thinking: { type: "enabled", budget_tokens: 2_048 } },
}),
)
@@ -168,6 +170,22 @@ describe("Anthropic Messages route", () => {
}),
)
it.effect("fits the thinking budget to half the output limit", () =>
Effect.gen(function* () {
const thinking = (maxTokens: number) =>
compileRequest(
LLMRequest.update(request, {
generation: { maxTokens },
providerOptions: { thinking: { type: "enabled", budgetTokens: 31_999 } },
}),
).pipe(Effect.map((prepared) => prepared.body.thinking))
expect(yield* thinking(64_000)).toEqual({ type: "enabled", budget_tokens: 31_999 })
expect(yield* thinking(20_000)).toEqual({ type: "enabled", budget_tokens: 10_000 })
expect(yield* thinking(1_500)).toEqual({ type: "enabled", budget_tokens: 1_024 })
}),
)
it.effect("rejects enabled thinking without a budget", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
+17
View File
@@ -90,6 +90,23 @@ describe("Gemini route", () => {
}),
)
it.effect("fits the thinking budget to half the output limit", () =>
Effect.gen(function* () {
const thinkingBudget = (budget: number, maxTokens = 32_000) =>
compileRequest(
LLMRequest.update(request, {
generation: { maxTokens },
providerOptions: { thinkingConfig: { thinkingBudget: budget } },
}),
).pipe(Effect.map((prepared) => prepared.body.generationConfig?.thinkingConfig?.thinkingBudget))
expect(yield* thinkingBudget(32_768)).toBe(16_000)
expect(yield* thinkingBudget(8_000)).toBe(8_000)
expect(yield* thinkingBudget(-1)).toBe(-1)
expect(yield* thinkingBudget(8_192, 1_000)).toBe(512)
}),
)
it.effect("forwards standard Gemini generation options", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(