mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-20 01:08:31 +03:00
fix(core): retry buffered generations with shared policy (#56497)
This commit is contained in:
committed by
GitHub
Unverified
parent
8c173a8f38
commit
86c7f7c840
@@ -7,6 +7,7 @@ import { makeLocationNode } from "@opencode/util/effect/app-node"
|
||||
import { llmClient } from "./effect/app-node-platform.js"
|
||||
import { ModelResolver } from "./model-resolver.js"
|
||||
import { Model } from "./model.js"
|
||||
import { ModelRetry } from "./model-retry.js"
|
||||
|
||||
export interface TextInput {
|
||||
readonly prompt: string
|
||||
@@ -71,6 +72,7 @@ export const layer = Layer.effect(
|
||||
}),
|
||||
)
|
||||
.pipe(
|
||||
ModelRetry.retry,
|
||||
Effect.mapError(
|
||||
(error: AIError) =>
|
||||
new UnavailableError({
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
export * as ModelRetry from "./model-retry.js"
|
||||
|
||||
import { AIError, isRetryable } from "@opencode/ai"
|
||||
import { Clock, Duration, Effect, Pull, Schedule } from "effect"
|
||||
|
||||
interface Input {
|
||||
readonly cause: AIError
|
||||
readonly retry: boolean
|
||||
}
|
||||
|
||||
/** Bound provider-requested delays so a hostile or buggy retry-after cannot stall a call for hours. */
|
||||
const RETRY_AFTER_MAX = Duration.toMillis("15 minutes")
|
||||
|
||||
const retryAfter = (input: Input) => {
|
||||
if (input.cause.reason._tag === "RateLimit" || input.cause.reason._tag === "ProviderInternal")
|
||||
return input.cause.reason.retryAfterMs === undefined
|
||||
? undefined
|
||||
: Math.min(input.cause.reason.retryAfterMs, RETRY_AFTER_MAX)
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Exponential from 2s capped at 10s per gap, for 10 retries: 2, 4, 8, then 10 × 7, about 84s of
|
||||
// waiting when every attempt fails (67–101s with jitter). `min` takes the faster schedule, so the
|
||||
// cap applies per gap; `max` with `recurs` bounds the count.
|
||||
const schedule = Schedule.max([
|
||||
Schedule.min([Schedule.exponential("2 seconds"), Schedule.spaced("10 seconds")]),
|
||||
Schedule.recurs(10),
|
||||
]).pipe(
|
||||
Schedule.jittered,
|
||||
Schedule.setInputType<Input>(),
|
||||
Schedule.modifyDelay(({ input, duration: delay }) => {
|
||||
const minimum = retryAfter(input)
|
||||
const duration = minimum === undefined ? delay : Duration.max(delay, Duration.millis(minimum))
|
||||
return Effect.succeed(Duration.millis(Math.ceil(Duration.toMillis(duration))))
|
||||
}),
|
||||
)
|
||||
|
||||
// A timed-out attempt already waited minutes before failing, so the general allowance would let a
|
||||
// dead provider hold a call for most of an hour. Cap those attempts well below it.
|
||||
const MAX_TIMEOUT_RETRIES = 3
|
||||
|
||||
export const policy = Effect.gen(function* () {
|
||||
const step = yield* Schedule.toStep(schedule)
|
||||
let attempt = 1
|
||||
let timeouts = 0
|
||||
return (input: Input) =>
|
||||
Effect.gen(function* () {
|
||||
const now = yield* Clock.currentTimeMillis
|
||||
const next = yield* step(now, input).pipe(Pull.catchDone(() => Effect.succeed(undefined)))
|
||||
if (!next) return undefined
|
||||
const [, duration] = next
|
||||
attempt++
|
||||
if (input.cause.reason._tag === "Transport" && input.cause.reason.code === "Timeout") timeouts++
|
||||
return {
|
||||
attempt,
|
||||
delay: Math.ceil(Duration.toMillis(duration)),
|
||||
retry: input.retry && timeouts <= MAX_TIMEOUT_RETRIES,
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
/** Retry a buffered model call from scratch; failed attempts never contribute to its returned output. */
|
||||
export const retry = <A, R>(effect: Effect.Effect<A, AIError, R>) =>
|
||||
Effect.gen(function* () {
|
||||
const decide = yield* policy
|
||||
while (true) {
|
||||
const result = yield* Effect.result(effect)
|
||||
if (result._tag === "Success") return result.success
|
||||
const decision = yield* decide({ cause: result.failure, retry: isRetryable(result.failure) })
|
||||
if (!decision?.retry) return yield* result.failure
|
||||
yield* Effect.sleep(Duration.millis(decision.delay))
|
||||
}
|
||||
})
|
||||
@@ -12,6 +12,7 @@ import type { AgentNotFoundError } from "./error.js"
|
||||
import { SessionHistory } from "./history.js"
|
||||
import { SessionProviderContext } from "./provider-context.js"
|
||||
import { SessionModelRequest } from "./model-request.js"
|
||||
import { ModelRetry } from "../model-retry.js"
|
||||
import type { SessionRunnerModel } from "./runner/model.js"
|
||||
import type { SessionSchema } from "./schema.js"
|
||||
|
||||
@@ -66,7 +67,7 @@ export const generate = Effect.fn("SessionGenerate.generate")(function* (input:
|
||||
providerID: model.ref.providerID,
|
||||
modelID: model.ref.id,
|
||||
})
|
||||
const response = yield* llm.generate(prepared.request, prepared.options)
|
||||
const response = yield* llm.generate(prepared.request, prepared.options).pipe(ModelRetry.retry)
|
||||
yield* Effect.logInfo("session generation usage diagnostic", { usage: response.usage })
|
||||
return response.text
|
||||
}).pipe(instances.provide(input.session))
|
||||
|
||||
@@ -4,8 +4,9 @@ import { AIError, isRetryable } from "@opencode/ai"
|
||||
import { Agent } from "@opencode/schema/agent"
|
||||
import { Model } from "@opencode/schema/model"
|
||||
import { SessionError } from "@opencode/schema/session-error"
|
||||
import { Clock, Duration, Effect, Pull, Schedule } from "effect"
|
||||
import { Clock, Duration, Effect } from "effect"
|
||||
import { Bus } from "../../bus.js"
|
||||
import { ModelRetry } from "../../model-retry.js"
|
||||
import type { PluginHooks } from "../../plugin/hooks.js"
|
||||
import { SessionEvent } from "../event.js"
|
||||
import { SessionMessage } from "../message.js"
|
||||
@@ -28,66 +29,28 @@ export interface Decision {
|
||||
readonly delay: number
|
||||
}
|
||||
|
||||
/** Bound provider-requested delays so a hostile or buggy retry-after cannot stall a session for hours. */
|
||||
const RETRY_AFTER_MAX = Duration.toMillis("15 minutes")
|
||||
|
||||
const retryAfter = (input: Input) => {
|
||||
if (input.cause.reason._tag === "RateLimit" || input.cause.reason._tag === "ProviderInternal")
|
||||
return input.cause.reason.retryAfterMs === undefined
|
||||
? undefined
|
||||
: Math.min(input.cause.reason.retryAfterMs, RETRY_AFTER_MAX)
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Exponential from 2s capped at 10s per gap, for 10 retries: 2, 4, 8, then 10 × 7, about 84s of
|
||||
// waiting when every attempt fails (67–101s with jitter). `min` takes the faster schedule, so the
|
||||
// cap applies per gap; `max` with `recurs` bounds the count.
|
||||
const schedule = Schedule.max([
|
||||
Schedule.min([Schedule.exponential("2 seconds"), Schedule.spaced("10 seconds")]),
|
||||
Schedule.recurs(10),
|
||||
]).pipe(
|
||||
Schedule.jittered,
|
||||
Schedule.setInputType<Input>(),
|
||||
Schedule.modifyDelay(({ input, duration: delay }) => {
|
||||
const minimum = retryAfter(input)
|
||||
const duration = minimum === undefined ? delay : Duration.max(delay, Duration.millis(minimum))
|
||||
return Effect.succeed(Duration.millis(Math.ceil(Duration.toMillis(duration))))
|
||||
}),
|
||||
)
|
||||
|
||||
// A timed-out attempt already waited minutes before failing, so the general allowance would let a
|
||||
// dead provider hold a step for most of an hour. Cap those attempts well below it.
|
||||
const MAX_TIMEOUT_RETRIES = 3
|
||||
|
||||
const isTimeout = (error: AIError) => error.reason._tag === "Transport" && error.reason.code === "Timeout"
|
||||
|
||||
export const policy = (sessionID: SessionSchema.ID) =>
|
||||
Effect.gen(function* () {
|
||||
const step = yield* Schedule.toStep(schedule)
|
||||
let attempt = 1
|
||||
let timeouts = 0
|
||||
const decide = yield* ModelRetry.policy
|
||||
return (input: Input) =>
|
||||
Effect.gen(function* () {
|
||||
const now = yield* Clock.currentTimeMillis
|
||||
const next = yield* step(now, input).pipe(Pull.catchDone(() => Effect.succeed(undefined)))
|
||||
const next = yield* decide(input)
|
||||
if (!next) return { retry: false as const }
|
||||
const [, duration] = next
|
||||
attempt++
|
||||
if (isTimeout(input.cause)) timeouts++
|
||||
const delay = Math.ceil(Duration.toMillis(duration))
|
||||
const event: PluginHooks.Domains["session"]["retry"] = {
|
||||
sessionID,
|
||||
agent: input.agent,
|
||||
model: input.model,
|
||||
error: input.error,
|
||||
attempt,
|
||||
decision: input.retry && timeouts <= MAX_TIMEOUT_RETRIES ? { retry: true, delay } : { retry: false },
|
||||
attempt: next.attempt,
|
||||
decision: next.retry ? { retry: true, delay: next.delay } : { retry: false },
|
||||
}
|
||||
yield* input.hook(event)
|
||||
if (!event.decision.retry) return event.decision
|
||||
const normalized =
|
||||
Number.isFinite(event.decision.delay) && event.decision.delay >= 0 ? Math.ceil(event.decision.delay) : delay
|
||||
return { retry: true as const, attempt, delay: normalized }
|
||||
Number.isFinite(event.decision.delay) && event.decision.delay >= 0
|
||||
? Math.ceil(event.decision.delay)
|
||||
: next.delay
|
||||
return { retry: true as const, attempt: next.attempt, delay: normalized }
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,5 +1,16 @@
|
||||
import { expect } from "bun:test"
|
||||
import { LanguageModel, LLMClient } from "@opencode/ai"
|
||||
import {
|
||||
AIError,
|
||||
AuthenticationError,
|
||||
InvalidRequestError,
|
||||
LanguageModel,
|
||||
LLMClient,
|
||||
LLMEvent,
|
||||
ProviderInternalError,
|
||||
QuotaExceededError,
|
||||
RateLimitError,
|
||||
TransportError,
|
||||
} from "@opencode/ai"
|
||||
import { RequestExecutor } from "@opencode/ai/route"
|
||||
import { OpenAIChat } from "@opencode/ai/protocols"
|
||||
import { TestLLM } from "@opencode/ai/testing"
|
||||
@@ -10,7 +21,8 @@ import { ModelResolver } from "@opencode/core/model-resolver"
|
||||
import { ID, Info, Model, Ref } from "@opencode/core/model"
|
||||
import { Provider } from "@opencode/core/provider"
|
||||
import { Npm } from "@opencode/util/npm"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { Effect, Exit, Fiber, Layer } from "effect"
|
||||
import { TestClock } from "effect/testing"
|
||||
import { HttpClient, HttpClientResponse } from "effect/http"
|
||||
import { testEffect } from "./lib/effect"
|
||||
|
||||
@@ -19,6 +31,7 @@ const selected = Info.make({
|
||||
package: Provider.aisdk("@ai-sdk/perplexity"),
|
||||
})
|
||||
const runtime = LanguageModel.make({ id: "gemini", provider: "test-provider", route: OpenAIChat.route })
|
||||
const reference = Ref.make({ providerID: selected.providerID, id: selected.id })
|
||||
|
||||
const providers = Layer.mock(Provider.Service, {
|
||||
get: () => Effect.undefined,
|
||||
@@ -66,6 +79,7 @@ const client = TestLLM.testLayer({ fallback: TestLLM.text("OK", "generate") })
|
||||
const resolver = ModelResolver.layer.pipe(Layer.provide(Layer.mergeAll(providers, models, integrations, npm, aisdk)))
|
||||
const it = testEffect(Generate.layer.pipe(Layer.provide(Layer.merge(resolver, client))))
|
||||
const resolverIt = testEffect(resolver)
|
||||
const retryIt = testEffect(Generate.layer.pipe(Layer.provideMerge(Layer.merge(resolver, TestLLM.testLayer()))))
|
||||
|
||||
it.effect("loads dynamic AI SDK models", () =>
|
||||
Effect.gen(function* () {
|
||||
@@ -79,6 +93,91 @@ it.effect("loads dynamic AI SDK models", () =>
|
||||
}),
|
||||
)
|
||||
|
||||
for (const operation of ["request", "read"] as const) {
|
||||
retryIt.effect(`regenerates the full stateless response after a ${operation} failure`, () =>
|
||||
Effect.gen(function* () {
|
||||
const generate = yield* Generate.Service
|
||||
const llm = yield* TestLLM.Test
|
||||
yield* llm.push(
|
||||
TestLLM.failAfter(
|
||||
new AIError({ reason: new TransportError({ message: "Disconnected", transport: "http", operation }) }),
|
||||
...(operation === "read"
|
||||
? [LLMEvent.textStart({ id: "failed" }), LLMEvent.textDelta({ id: "failed", text: "Discard this" })]
|
||||
: []),
|
||||
),
|
||||
TestLLM.text("Fresh answer", "success"),
|
||||
)
|
||||
const result = yield* generate
|
||||
.text({ prompt: "Answer privately", model: reference })
|
||||
.pipe(Effect.forkChild({ startImmediately: true }))
|
||||
yield* llm.wait(1)
|
||||
yield* TestClock.adjust("3 seconds")
|
||||
expect(yield* Fiber.join(result)).toBe("Fresh answer")
|
||||
const requests = yield* llm.requests()
|
||||
expect(requests).toHaveLength(2)
|
||||
expect(requests[1]).toBe(requests[0])
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
for (const reason of [
|
||||
new AuthenticationError({ message: "Invalid credentials" }),
|
||||
new QuotaExceededError({ message: "No quota" }),
|
||||
new InvalidRequestError({ message: "Invalid prompt" }),
|
||||
]) {
|
||||
retryIt.effect(`does not retry stateless ${reason._tag} failures`, () =>
|
||||
Effect.gen(function* () {
|
||||
const generate = yield* Generate.Service
|
||||
const llm = yield* TestLLM.Test
|
||||
yield* llm.push(TestLLM.failAfter(new AIError({ reason })))
|
||||
const result = yield* generate.text({ prompt: "Answer privately", model: reference }).pipe(Effect.flip)
|
||||
expect(result).toBeInstanceOf(Generate.UnavailableError)
|
||||
expect(result.message).toBe(reason.message)
|
||||
expect(yield* llm.requests()).toHaveLength(1)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
retryIt.effect("honors provider backoff and cancels without another generation", () =>
|
||||
Effect.gen(function* () {
|
||||
const generate = yield* Generate.Service
|
||||
const llm = yield* TestLLM.Test
|
||||
yield* llm.always(
|
||||
TestLLM.failAfter(new AIError({ reason: new RateLimitError({ message: "Wait", retryAfterMs: 5_000 }) })),
|
||||
)
|
||||
const result = yield* generate
|
||||
.text({ prompt: "Answer privately", model: reference })
|
||||
.pipe(Effect.forkChild({ startImmediately: true }))
|
||||
yield* llm.wait(1)
|
||||
yield* TestClock.adjust("4999 millis")
|
||||
expect(yield* llm.requests()).toHaveLength(1)
|
||||
yield* Fiber.interrupt(result)
|
||||
yield* TestClock.adjust("1 minute")
|
||||
expect(Exit.hasInterrupts(yield* Fiber.await(result))).toBe(true)
|
||||
expect(yield* llm.requests()).toHaveLength(1)
|
||||
}),
|
||||
)
|
||||
|
||||
for (const [reason, attempts] of [
|
||||
[new ProviderInternalError({ message: "Unavailable" }), 11],
|
||||
[new TransportError({ message: "Timed out", transport: "http", operation: "request", code: "Timeout" }), 4],
|
||||
] as const) {
|
||||
retryIt.effect(`bounds stateless ${reason._tag} retries`, () =>
|
||||
Effect.gen(function* () {
|
||||
const generate = yield* Generate.Service
|
||||
const llm = yield* TestLLM.Test
|
||||
yield* llm.always(TestLLM.failAfter(new AIError({ reason })))
|
||||
const result = yield* generate
|
||||
.text({ prompt: "Answer privately", model: reference })
|
||||
.pipe(Effect.flip, Effect.forkChild({ startImmediately: true }))
|
||||
yield* llm.wait(1)
|
||||
yield* TestClock.adjust("2 minutes")
|
||||
expect(yield* Fiber.join(result)).toBeInstanceOf(Generate.UnavailableError)
|
||||
expect(yield* llm.requests()).toHaveLength(attempts)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
resolverIt.effect("resolves dynamic models with their catalog metadata", () =>
|
||||
Effect.gen(function* () {
|
||||
const resolver = yield* ModelResolver.Service
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { expect } from "bun:test"
|
||||
import {
|
||||
AIError,
|
||||
LLMClient,
|
||||
LLMEvent,
|
||||
LLMResponse,
|
||||
@@ -7,9 +8,11 @@ import {
|
||||
Message,
|
||||
SystemPart,
|
||||
ToolDefinition,
|
||||
TransportError,
|
||||
type LLMRequest,
|
||||
} from "@opencode/ai"
|
||||
import { OpenAIChat } from "@opencode/ai/protocols"
|
||||
import { TestLLM } from "@opencode/ai/testing"
|
||||
import type { StreamOptions } from "@opencode/ai/route"
|
||||
import { Agent } from "@opencode/core/agent"
|
||||
import { Database } from "@opencode/core/database/database"
|
||||
@@ -51,7 +54,8 @@ import { PluginHooks } from "@opencode/core/plugin/hooks"
|
||||
import { PluginSupervisor } from "@opencode/core/plugin/supervisor"
|
||||
import { Tool } from "@opencode/core/tool"
|
||||
import { asc, eq } from "drizzle-orm"
|
||||
import { Effect, Layer, Schema, Stream } from "effect"
|
||||
import { Effect, Fiber, Layer, Schema, Stream } from "effect"
|
||||
import { TestClock } from "effect/testing"
|
||||
import { testEffect } from "./lib/effect"
|
||||
|
||||
const requests: LLMRequest[] = []
|
||||
@@ -394,6 +398,39 @@ it.effect(
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
|
||||
it.effect("regenerates a failed partial answer without mutating Session history", () =>
|
||||
Effect.gen(function* () {
|
||||
instruction = "Initial context"
|
||||
const { db, bus, instructions, session, instances } = yield* setup
|
||||
yield* InstructionState.prepare(db, bus, instructions, sessionID)
|
||||
const before = yield* durableState(db, sessionID)
|
||||
const llm = yield* TestLLM.Test
|
||||
yield* llm.push(
|
||||
TestLLM.failAfter(
|
||||
new AIError({
|
||||
reason: new TransportError({ message: "Disconnected", transport: "http", operation: "read" }),
|
||||
}),
|
||||
LLMEvent.textStart({ id: "failed" }),
|
||||
LLMEvent.textDelta({ id: "failed", text: "Discard this" }),
|
||||
),
|
||||
TestLLM.text("Fresh answer", "success"),
|
||||
)
|
||||
const result = yield* SessionGenerate.generate({ session, prompt: "Summarize privately" }).pipe(
|
||||
Effect.provideService(Instance.Service, instances),
|
||||
Effect.provideService(LLMClient.Service, llm),
|
||||
Effect.forkChild({ startImmediately: true }),
|
||||
)
|
||||
yield* llm.wait(1)
|
||||
yield* TestClock.adjust("3 seconds")
|
||||
expect(yield* Fiber.join(result)).toBe("Fresh answer")
|
||||
const requests = yield* llm.requests()
|
||||
expect(requests).toHaveLength(2)
|
||||
expect(requests[1]).toBe(requests[0])
|
||||
expect(userTexts(requests[1])).toContain("Summarize privately")
|
||||
expect(yield* durableState(db, sessionID)).toEqual(before)
|
||||
}).pipe(Effect.provide(TestLLM.testLayer())),
|
||||
)
|
||||
|
||||
it.effect(
|
||||
"blocks unavailable initial instructions before generation",
|
||||
() =>
|
||||
|
||||
@@ -41,7 +41,7 @@ describe("SessionRunnerRetry.policy", () => {
|
||||
|
||||
it.effect("keeps the general allowance for other transient failures", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(yield* decisions(internal, 10)).toEqual(Array(10).fill(true))
|
||||
expect(yield* decisions(internal, 11)).toEqual([...Array(10).fill(true), false])
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user