fix(core): retry buffered generations with shared policy (#56497)

This commit is contained in:
Aiden Cline
2026-10-17 14:33:47 -05:00
committed by GitHub
Unverified
parent 8c173a8f38
commit 86c7f7c840
7 changed files with 227 additions and 52 deletions
+2
View File
@@ -7,6 +7,7 @@ import { makeLocationNode } from "@opencode/util/effect/app-node"
import { llmClient } from "./effect/app-node-platform.js"
import { ModelResolver } from "./model-resolver.js"
import { Model } from "./model.js"
import { ModelRetry } from "./model-retry.js"
export interface TextInput {
readonly prompt: string
@@ -71,6 +72,7 @@ export const layer = Layer.effect(
}),
)
.pipe(
ModelRetry.retry,
Effect.mapError(
(error: AIError) =>
new UnavailableError({
+73
View File
@@ -0,0 +1,73 @@
export * as ModelRetry from "./model-retry.js"
import { AIError, isRetryable } from "@opencode/ai"
import { Clock, Duration, Effect, Pull, Schedule } from "effect"
interface Input {
readonly cause: AIError
readonly retry: boolean
}
/** Bound provider-requested delays so a hostile or buggy retry-after cannot stall a call for hours. */
const RETRY_AFTER_MAX = Duration.toMillis("15 minutes")
const retryAfter = (input: Input) => {
if (input.cause.reason._tag === "RateLimit" || input.cause.reason._tag === "ProviderInternal")
return input.cause.reason.retryAfterMs === undefined
? undefined
: Math.min(input.cause.reason.retryAfterMs, RETRY_AFTER_MAX)
return undefined
}
// Exponential from 2s capped at 10s per gap, for 10 retries: 2, 4, 8, then 10 × 7, about 84s of
// waiting when every attempt fails (67–101s with jitter). `min` takes the faster schedule, so the
// cap applies per gap; `max` with `recurs` bounds the count.
const schedule = Schedule.max([
Schedule.min([Schedule.exponential("2 seconds"), Schedule.spaced("10 seconds")]),
Schedule.recurs(10),
]).pipe(
Schedule.jittered,
Schedule.setInputType<Input>(),
Schedule.modifyDelay(({ input, duration: delay }) => {
const minimum = retryAfter(input)
const duration = minimum === undefined ? delay : Duration.max(delay, Duration.millis(minimum))
return Effect.succeed(Duration.millis(Math.ceil(Duration.toMillis(duration))))
}),
)
// A timed-out attempt already waited minutes before failing, so the general allowance would let a
// dead provider hold a call for most of an hour. Cap those attempts well below it.
const MAX_TIMEOUT_RETRIES = 3
export const policy = Effect.gen(function* () {
const step = yield* Schedule.toStep(schedule)
let attempt = 1
let timeouts = 0
return (input: Input) =>
Effect.gen(function* () {
const now = yield* Clock.currentTimeMillis
const next = yield* step(now, input).pipe(Pull.catchDone(() => Effect.succeed(undefined)))
if (!next) return undefined
const [, duration] = next
attempt++
if (input.cause.reason._tag === "Transport" && input.cause.reason.code === "Timeout") timeouts++
return {
attempt,
delay: Math.ceil(Duration.toMillis(duration)),
retry: input.retry && timeouts <= MAX_TIMEOUT_RETRIES,
}
})
})
/** Retry a buffered model call from scratch; failed attempts never contribute to its returned output. */
export const retry = <A, R>(effect: Effect.Effect<A, AIError, R>) =>
Effect.gen(function* () {
const decide = yield* policy
while (true) {
const result = yield* Effect.result(effect)
if (result._tag === "Success") return result.success
const decision = yield* decide({ cause: result.failure, retry: isRetryable(result.failure) })
if (!decision?.retry) return yield* result.failure
yield* Effect.sleep(Duration.millis(decision.delay))
}
})
+2 -1
View File
@@ -12,6 +12,7 @@ import type { AgentNotFoundError } from "./error.js"
import { SessionHistory } from "./history.js"
import { SessionProviderContext } from "./provider-context.js"
import { SessionModelRequest } from "./model-request.js"
import { ModelRetry } from "../model-retry.js"
import type { SessionRunnerModel } from "./runner/model.js"
import type { SessionSchema } from "./schema.js"
@@ -66,7 +67,7 @@ export const generate = Effect.fn("SessionGenerate.generate")(function* (input:
providerID: model.ref.providerID,
modelID: model.ref.id,
})
const response = yield* llm.generate(prepared.request, prepared.options)
const response = yield* llm.generate(prepared.request, prepared.options).pipe(ModelRetry.retry)
yield* Effect.logInfo("session generation usage diagnostic", { usage: response.usage })
return response.text
}).pipe(instances.provide(input.session))
+10 -47
View File
@@ -4,8 +4,9 @@ import { AIError, isRetryable } from "@opencode/ai"
import { Agent } from "@opencode/schema/agent"
import { Model } from "@opencode/schema/model"
import { SessionError } from "@opencode/schema/session-error"
import { Clock, Duration, Effect, Pull, Schedule } from "effect"
import { Clock, Duration, Effect } from "effect"
import { Bus } from "../../bus.js"
import { ModelRetry } from "../../model-retry.js"
import type { PluginHooks } from "../../plugin/hooks.js"
import { SessionEvent } from "../event.js"
import { SessionMessage } from "../message.js"
@@ -28,66 +29,28 @@ export interface Decision {
readonly delay: number
}
/** Bound provider-requested delays so a hostile or buggy retry-after cannot stall a session for hours. */
const RETRY_AFTER_MAX = Duration.toMillis("15 minutes")
const retryAfter = (input: Input) => {
if (input.cause.reason._tag === "RateLimit" || input.cause.reason._tag === "ProviderInternal")
return input.cause.reason.retryAfterMs === undefined
? undefined
: Math.min(input.cause.reason.retryAfterMs, RETRY_AFTER_MAX)
return undefined
}
// Exponential from 2s capped at 10s per gap, for 10 retries: 2, 4, 8, then 10 × 7, about 84s of
// waiting when every attempt fails (67–101s with jitter). `min` takes the faster schedule, so the
// cap applies per gap; `max` with `recurs` bounds the count.
const schedule = Schedule.max([
Schedule.min([Schedule.exponential("2 seconds"), Schedule.spaced("10 seconds")]),
Schedule.recurs(10),
]).pipe(
Schedule.jittered,
Schedule.setInputType<Input>(),
Schedule.modifyDelay(({ input, duration: delay }) => {
const minimum = retryAfter(input)
const duration = minimum === undefined ? delay : Duration.max(delay, Duration.millis(minimum))
return Effect.succeed(Duration.millis(Math.ceil(Duration.toMillis(duration))))
}),
)
// A timed-out attempt already waited minutes before failing, so the general allowance would let a
// dead provider hold a step for most of an hour. Cap those attempts well below it.
const MAX_TIMEOUT_RETRIES = 3
const isTimeout = (error: AIError) => error.reason._tag === "Transport" && error.reason.code === "Timeout"
export const policy = (sessionID: SessionSchema.ID) =>
Effect.gen(function* () {
const step = yield* Schedule.toStep(schedule)
let attempt = 1
let timeouts = 0
const decide = yield* ModelRetry.policy
return (input: Input) =>
Effect.gen(function* () {
const now = yield* Clock.currentTimeMillis
const next = yield* step(now, input).pipe(Pull.catchDone(() => Effect.succeed(undefined)))
const next = yield* decide(input)
if (!next) return { retry: false as const }
const [, duration] = next
attempt++
if (isTimeout(input.cause)) timeouts++
const delay = Math.ceil(Duration.toMillis(duration))
const event: PluginHooks.Domains["session"]["retry"] = {
sessionID,
agent: input.agent,
model: input.model,
error: input.error,
attempt,
decision: input.retry && timeouts <= MAX_TIMEOUT_RETRIES ? { retry: true, delay } : { retry: false },
attempt: next.attempt,
decision: next.retry ? { retry: true, delay: next.delay } : { retry: false },
}
yield* input.hook(event)
if (!event.decision.retry) return event.decision
const normalized =
Number.isFinite(event.decision.delay) && event.decision.delay >= 0 ? Math.ceil(event.decision.delay) : delay
return { retry: true as const, attempt, delay: normalized }
Number.isFinite(event.decision.delay) && event.decision.delay >= 0
? Math.ceil(event.decision.delay)
: next.delay
return { retry: true as const, attempt: next.attempt, delay: normalized }
})
})
+101 -2
View File
@@ -1,5 +1,16 @@
import { expect } from "bun:test"
import { LanguageModel, LLMClient } from "@opencode/ai"
import {
AIError,
AuthenticationError,
InvalidRequestError,
LanguageModel,
LLMClient,
LLMEvent,
ProviderInternalError,
QuotaExceededError,
RateLimitError,
TransportError,
} from "@opencode/ai"
import { RequestExecutor } from "@opencode/ai/route"
import { OpenAIChat } from "@opencode/ai/protocols"
import { TestLLM } from "@opencode/ai/testing"
@@ -10,7 +21,8 @@ import { ModelResolver } from "@opencode/core/model-resolver"
import { ID, Info, Model, Ref } from "@opencode/core/model"
import { Provider } from "@opencode/core/provider"
import { Npm } from "@opencode/util/npm"
import { Effect, Layer } from "effect"
import { Effect, Exit, Fiber, Layer } from "effect"
import { TestClock } from "effect/testing"
import { HttpClient, HttpClientResponse } from "effect/http"
import { testEffect } from "./lib/effect"
@@ -19,6 +31,7 @@ const selected = Info.make({
package: Provider.aisdk("@ai-sdk/perplexity"),
})
const runtime = LanguageModel.make({ id: "gemini", provider: "test-provider", route: OpenAIChat.route })
const reference = Ref.make({ providerID: selected.providerID, id: selected.id })
const providers = Layer.mock(Provider.Service, {
get: () => Effect.undefined,
@@ -66,6 +79,7 @@ const client = TestLLM.testLayer({ fallback: TestLLM.text("OK", "generate") })
const resolver = ModelResolver.layer.pipe(Layer.provide(Layer.mergeAll(providers, models, integrations, npm, aisdk)))
const it = testEffect(Generate.layer.pipe(Layer.provide(Layer.merge(resolver, client))))
const resolverIt = testEffect(resolver)
const retryIt = testEffect(Generate.layer.pipe(Layer.provideMerge(Layer.merge(resolver, TestLLM.testLayer()))))
it.effect("loads dynamic AI SDK models", () =>
Effect.gen(function* () {
@@ -79,6 +93,91 @@ it.effect("loads dynamic AI SDK models", () =>
}),
)
for (const operation of ["request", "read"] as const) {
retryIt.effect(`regenerates the full stateless response after a ${operation} failure`, () =>
Effect.gen(function* () {
const generate = yield* Generate.Service
const llm = yield* TestLLM.Test
yield* llm.push(
TestLLM.failAfter(
new AIError({ reason: new TransportError({ message: "Disconnected", transport: "http", operation }) }),
...(operation === "read"
? [LLMEvent.textStart({ id: "failed" }), LLMEvent.textDelta({ id: "failed", text: "Discard this" })]
: []),
),
TestLLM.text("Fresh answer", "success"),
)
const result = yield* generate
.text({ prompt: "Answer privately", model: reference })
.pipe(Effect.forkChild({ startImmediately: true }))
yield* llm.wait(1)
yield* TestClock.adjust("3 seconds")
expect(yield* Fiber.join(result)).toBe("Fresh answer")
const requests = yield* llm.requests()
expect(requests).toHaveLength(2)
expect(requests[1]).toBe(requests[0])
}),
)
}
for (const reason of [
new AuthenticationError({ message: "Invalid credentials" }),
new QuotaExceededError({ message: "No quota" }),
new InvalidRequestError({ message: "Invalid prompt" }),
]) {
retryIt.effect(`does not retry stateless ${reason._tag} failures`, () =>
Effect.gen(function* () {
const generate = yield* Generate.Service
const llm = yield* TestLLM.Test
yield* llm.push(TestLLM.failAfter(new AIError({ reason })))
const result = yield* generate.text({ prompt: "Answer privately", model: reference }).pipe(Effect.flip)
expect(result).toBeInstanceOf(Generate.UnavailableError)
expect(result.message).toBe(reason.message)
expect(yield* llm.requests()).toHaveLength(1)
}),
)
}
retryIt.effect("honors provider backoff and cancels without another generation", () =>
Effect.gen(function* () {
const generate = yield* Generate.Service
const llm = yield* TestLLM.Test
yield* llm.always(
TestLLM.failAfter(new AIError({ reason: new RateLimitError({ message: "Wait", retryAfterMs: 5_000 }) })),
)
const result = yield* generate
.text({ prompt: "Answer privately", model: reference })
.pipe(Effect.forkChild({ startImmediately: true }))
yield* llm.wait(1)
yield* TestClock.adjust("4999 millis")
expect(yield* llm.requests()).toHaveLength(1)
yield* Fiber.interrupt(result)
yield* TestClock.adjust("1 minute")
expect(Exit.hasInterrupts(yield* Fiber.await(result))).toBe(true)
expect(yield* llm.requests()).toHaveLength(1)
}),
)
for (const [reason, attempts] of [
[new ProviderInternalError({ message: "Unavailable" }), 11],
[new TransportError({ message: "Timed out", transport: "http", operation: "request", code: "Timeout" }), 4],
] as const) {
retryIt.effect(`bounds stateless ${reason._tag} retries`, () =>
Effect.gen(function* () {
const generate = yield* Generate.Service
const llm = yield* TestLLM.Test
yield* llm.always(TestLLM.failAfter(new AIError({ reason })))
const result = yield* generate
.text({ prompt: "Answer privately", model: reference })
.pipe(Effect.flip, Effect.forkChild({ startImmediately: true }))
yield* llm.wait(1)
yield* TestClock.adjust("2 minutes")
expect(yield* Fiber.join(result)).toBeInstanceOf(Generate.UnavailableError)
expect(yield* llm.requests()).toHaveLength(attempts)
}),
)
}
resolverIt.effect("resolves dynamic models with their catalog metadata", () =>
Effect.gen(function* () {
const resolver = yield* ModelResolver.Service
+38 -1
View File
@@ -1,5 +1,6 @@
import { expect } from "bun:test"
import {
AIError,
LLMClient,
LLMEvent,
LLMResponse,
@@ -7,9 +8,11 @@ import {
Message,
SystemPart,
ToolDefinition,
TransportError,
type LLMRequest,
} from "@opencode/ai"
import { OpenAIChat } from "@opencode/ai/protocols"
import { TestLLM } from "@opencode/ai/testing"
import type { StreamOptions } from "@opencode/ai/route"
import { Agent } from "@opencode/core/agent"
import { Database } from "@opencode/core/database/database"
@@ -51,7 +54,8 @@ import { PluginHooks } from "@opencode/core/plugin/hooks"
import { PluginSupervisor } from "@opencode/core/plugin/supervisor"
import { Tool } from "@opencode/core/tool"
import { asc, eq } from "drizzle-orm"
import { Effect, Layer, Schema, Stream } from "effect"
import { Effect, Fiber, Layer, Schema, Stream } from "effect"
import { TestClock } from "effect/testing"
import { testEffect } from "./lib/effect"
const requests: LLMRequest[] = []
@@ -394,6 +398,39 @@ it.effect(
{ timeout: 15_000 },
)
it.effect("regenerates a failed partial answer without mutating Session history", () =>
Effect.gen(function* () {
instruction = "Initial context"
const { db, bus, instructions, session, instances } = yield* setup
yield* InstructionState.prepare(db, bus, instructions, sessionID)
const before = yield* durableState(db, sessionID)
const llm = yield* TestLLM.Test
yield* llm.push(
TestLLM.failAfter(
new AIError({
reason: new TransportError({ message: "Disconnected", transport: "http", operation: "read" }),
}),
LLMEvent.textStart({ id: "failed" }),
LLMEvent.textDelta({ id: "failed", text: "Discard this" }),
),
TestLLM.text("Fresh answer", "success"),
)
const result = yield* SessionGenerate.generate({ session, prompt: "Summarize privately" }).pipe(
Effect.provideService(Instance.Service, instances),
Effect.provideService(LLMClient.Service, llm),
Effect.forkChild({ startImmediately: true }),
)
yield* llm.wait(1)
yield* TestClock.adjust("3 seconds")
expect(yield* Fiber.join(result)).toBe("Fresh answer")
const requests = yield* llm.requests()
expect(requests).toHaveLength(2)
expect(requests[1]).toBe(requests[0])
expect(userTexts(requests[1])).toContain("Summarize privately")
expect(yield* durableState(db, sessionID)).toEqual(before)
}).pipe(Effect.provide(TestLLM.testLayer())),
)
it.effect(
"blocks unavailable initial instructions before generation",
() =>
@@ -41,7 +41,7 @@ describe("SessionRunnerRetry.policy", () => {
it.effect("keeps the general allowance for other transient failures", () =>
Effect.gen(function* () {
expect(yield* decisions(internal, 10)).toEqual(Array(10).fill(true))
expect(yield* decisions(internal, 11)).toEqual([...Array(10).fill(true), false])
}),
)
})