mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-19 00:03:13 +03:00
fix(core): keep retrying while the provider's host is unreachable (#61880)
This commit is contained in:
committed by
GitHub
Unverified
parent
d1da4264a9
commit
20c94b79f1
@@ -74,6 +74,11 @@ const eventMessage = (event: Socket.WebSocketEvent) => {
|
||||
return event.type ?? "error"
|
||||
}
|
||||
|
||||
const systemCode = (cause: unknown) =>
|
||||
typeof cause === "object" && cause !== null && "code" in cause && typeof cause.code === "string"
|
||||
? cause.code
|
||||
: undefined
|
||||
|
||||
const binaryMessage = (data: unknown) => {
|
||||
if (data instanceof Uint8Array) return data
|
||||
if (data instanceof ArrayBuffer) return new Uint8Array(data)
|
||||
@@ -115,12 +120,15 @@ const waitOpen = (ws: Socket.WebSocketLike, input: WebSocketRequest) => {
|
||||
}
|
||||
const onError = (event: Socket.WebSocketEvent) => {
|
||||
cleanup()
|
||||
const cause = "error" in event ? (event.error ?? event) : event
|
||||
resume(
|
||||
Effect.fail(
|
||||
transportError(`Failed to open WebSocket: ${eventMessage(event)}`, {
|
||||
cause: "error" in event ? (event.error ?? event) : event,
|
||||
cause,
|
||||
url: input.url,
|
||||
operation: "request",
|
||||
// Bun puts the system code (ENOTFOUND, ECONNREFUSED) on the event's error; lift it like HTTP failures do.
|
||||
code: systemCode(cause),
|
||||
phase: "connect",
|
||||
delivery: "not-sent",
|
||||
}),
|
||||
|
||||
@@ -1459,6 +1459,28 @@ describe("OpenAI Responses route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps the system code when a WebSocket fails to connect", () =>
|
||||
Effect.gen(function* () {
|
||||
const socket = Object.assign(new EventTarget(), { readyState: globalThis.WebSocket.CONNECTING, close: () => {} })
|
||||
const failed = Object.assign(new Event("error"), {
|
||||
error: Object.assign(new Error("Failed to connect"), { code: "ENOTFOUND" }),
|
||||
})
|
||||
queueMicrotask(() => socket.dispatchEvent(failed))
|
||||
const error = yield* WebSocketTransport.fromWebSocket(
|
||||
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- waitOpen only listens for events on this branch.
|
||||
socket as unknown as globalThis.WebSocket,
|
||||
{ url: "wss://api.openai.test/v1/responses", headers: Headers.empty },
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.reason).toMatchObject({
|
||||
_tag: "Transport",
|
||||
code: "ENOTFOUND",
|
||||
phase: "connect",
|
||||
delivery: "not-sent",
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("adds native query params to the Responses URL", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* LLMClient.generate({
|
||||
|
||||
@@ -934,7 +934,8 @@ function llmError(error: unknown, operation: "request" | "read") {
|
||||
const errorValue = (error: unknown) => (isRecord(error) ? error.value : undefined)
|
||||
|
||||
// Runtime-generated network failure shapes. The codes mirror the AI SDK's own
|
||||
// Bun network error list in handleFetchError; the messages are undici's fetch
|
||||
// Bun network error list in handleFetchError, plus the unreachable-host codes that list leaves out and
|
||||
// ModelRetry gives a longer budget; the messages are undici's fetch
|
||||
// TypeError and stream termination strings plus our header and chunk timeout errors.
|
||||
// Unrecognized shapes still retry via the UnknownProvider default; this match
|
||||
// only adds transport semantics (continuation eligibility, display).
|
||||
@@ -943,6 +944,11 @@ const NETWORK_ERROR_CODES = new Set([
|
||||
"ECONNREFUSED",
|
||||
"ETIMEDOUT",
|
||||
"EPIPE",
|
||||
"ENOTFOUND",
|
||||
"EAI_AGAIN",
|
||||
"ENETUNREACH",
|
||||
"EHOSTUNREACH",
|
||||
"ENETDOWN",
|
||||
"ConnectionRefused",
|
||||
"ConnectionClosed",
|
||||
"FailedToOpenSocket",
|
||||
@@ -989,6 +995,8 @@ function apiCallError(error: APICallError) {
|
||||
cause: error,
|
||||
})
|
||||
if (error.statusCode !== undefined || !error.isRetryable) return failure
|
||||
// "Cannot connect to API" wraps the runtime's error; keep its code so retry and display can tell why.
|
||||
const code = networkFailure(error.cause)?.code
|
||||
return new AIError({
|
||||
reason: new TransportError({
|
||||
message: failure.message,
|
||||
@@ -998,6 +1006,7 @@ function apiCallError(error: APICallError) {
|
||||
transport: "http",
|
||||
operation: "request",
|
||||
url: error.url,
|
||||
...(code === undefined ? {} : { code }),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -35,18 +35,47 @@ const schedule = Schedule.max([
|
||||
}),
|
||||
)
|
||||
|
||||
// This machine has no network path to the provider: the host name didn't resolve, or there is no route or interface.
|
||||
// While that lasts every attempt fails before reaching the provider, and the outage behind it (a router restart, a
|
||||
// dropped VPN, a laptop between networks) usually outlasts the general allowance, so these retry on their own budget:
|
||||
// exponential from 2s capped at 30s per gap, for up to an hour after the first one. A refused connection is left out:
|
||||
// something answered, so it is the server's state.
|
||||
const UNREACHABLE_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "ENETUNREACH", "EHOSTUNREACH", "ENETDOWN"])
|
||||
export const UNREACHABLE_WINDOW = Duration.hours(1)
|
||||
const UNREACHABLE_GAP_MAX = 30_000
|
||||
|
||||
/** Whether the failure says this machine has no network path to the provider's host. */
|
||||
export const isUnreachable = (error: AIError) =>
|
||||
error.reason._tag === "Transport" && error.reason.code !== undefined && UNREACHABLE_CODES.has(error.reason.code)
|
||||
|
||||
const unreachableSchedule = Schedule.max([
|
||||
Schedule.min([Schedule.exponential("2 seconds"), Schedule.spaced(Duration.millis(UNREACHABLE_GAP_MAX))]),
|
||||
Schedule.during(UNREACHABLE_WINDOW),
|
||||
]).pipe(
|
||||
Schedule.jittered,
|
||||
Schedule.setInputType<Input>(),
|
||||
// Jitter scales the capped gap by up to 1.2, so cap it again after.
|
||||
Schedule.modifyDelay(({ duration }) =>
|
||||
Effect.succeed(Duration.millis(Math.min(Math.ceil(Duration.toMillis(duration)), UNREACHABLE_GAP_MAX))),
|
||||
),
|
||||
)
|
||||
|
||||
// A timed-out attempt already waited minutes before failing, so the general allowance would let a
|
||||
// dead provider hold a call for most of an hour. Cap those attempts well below it.
|
||||
const MAX_TIMEOUT_RETRIES = 3
|
||||
|
||||
export const policy = Effect.gen(function* () {
|
||||
const step = yield* Schedule.toStep(schedule)
|
||||
const unreachable = yield* Schedule.toStep(unreachableSchedule)
|
||||
let attempt = 1
|
||||
let timeouts = 0
|
||||
return (input: Input) =>
|
||||
Effect.gen(function* () {
|
||||
const now = yield* Clock.currentTimeMillis
|
||||
const next = yield* step(now, input).pipe(Pull.catchDone(() => Effect.succeed(undefined)))
|
||||
// Unreachable failures don't spend the general allowance, so a flaky link that also drops streams keeps it.
|
||||
const next = yield* (input.retry && isUnreachable(input.cause) ? unreachable : step)(now, input).pipe(
|
||||
Pull.catchDone(() => Effect.succeed(undefined)),
|
||||
)
|
||||
if (!next) return undefined
|
||||
const [, duration] = next
|
||||
attempt++
|
||||
@@ -75,6 +104,7 @@ export const retry = <A, R>(effect: Effect.Effect<A, AIError, R>) =>
|
||||
// One-shot generations (stateless text, images, decisions) can be billed per attempt, so they retry only failures
|
||||
// the provider never ran, a few times, with the same backoff and retry-after handling.
|
||||
const ONE_SHOT_RETRIES = 3
|
||||
// Only codes that fail before a connection exists: a dropped interface or route can also fail a response already being read.
|
||||
const NOT_SENT_CODES = new Set(["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN"])
|
||||
|
||||
const notRun = (error: AIError) => {
|
||||
|
||||
@@ -5,6 +5,7 @@ import { Permission } from "../permission.js"
|
||||
import { Integration } from "../integration.js"
|
||||
import { AgentNotFoundError, StepFailedError } from "./error.js"
|
||||
import { ModelResolver } from "../model-resolver.js"
|
||||
import { ModelRetry } from "../model-retry.js"
|
||||
import { SessionRunnerModel } from "./runner/model.js"
|
||||
|
||||
const tokenSharingMessages = {
|
||||
@@ -49,6 +50,7 @@ export function toSessionError(cause: unknown): SessionError.Error {
|
||||
case "ContentPolicy":
|
||||
return providerError("provider.content-filter", cause.reason)
|
||||
case "Transport":
|
||||
if (ModelRetry.isUnreachable(cause)) return unreachable(cause.reason)
|
||||
return providerError("provider.transport", cause.reason)
|
||||
case "ProviderInternal":
|
||||
return providerError("provider.internal", cause.reason)
|
||||
@@ -120,6 +122,15 @@ function goLimit(reason: AIError["reason"]): SessionError.Error {
|
||||
}
|
||||
}
|
||||
|
||||
// Names the network as the problem, so a DNS outage doesn't read like a broken model or provider.
|
||||
function unreachable(reason: AIError["reason"]): SessionError.Error {
|
||||
const host = "url" in reason && reason.url !== undefined ? URL.parse(reason.url)?.hostname : undefined
|
||||
return {
|
||||
...providerError("provider.transport", reason),
|
||||
message: `Can't reach ${host || "the provider"}. Check the network connection. (${reason.message})`,
|
||||
}
|
||||
}
|
||||
|
||||
function providerError(type: string, reason: AIError["reason"]): SessionError.Error {
|
||||
const status = reason.http?.status
|
||||
const retryAfterMs = "retryAfterMs" in reason ? reason.retryAfterMs : undefined
|
||||
|
||||
@@ -2,6 +2,7 @@ import { APICallError } from "@ai-sdk/provider"
|
||||
import type { LanguageModelV3, LanguageModelV3StreamPart } from "@ai-sdk/provider"
|
||||
import { createOpenAICompatible } from "@ai-sdk/openai-compatible"
|
||||
import { AISDK } from "@opencode/core/aisdk"
|
||||
import { ModelRetry } from "@opencode/core/model-retry"
|
||||
import { SessionRunnerRetry } from "@opencode/core/session/runner/retry"
|
||||
import { toSessionError } from "@opencode/core/session/to-session-error"
|
||||
import { Model } from "@opencode/core/model"
|
||||
@@ -1198,6 +1199,21 @@ it.effect("classifies native fetch failures as request transport errors", () =>
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps the code of a host name that didn't resolve", () =>
|
||||
Effect.gen(function* () {
|
||||
const dns = Object.assign(new Error("getaddrinfo ENOTFOUND api.example.com"), { code: "ENOTFOUND" })
|
||||
const raw = yield* streamFailure(dns)
|
||||
expect(raw.reason).toMatchObject({ _tag: "Transport", operation: "request", code: "ENOTFOUND" })
|
||||
expect(ModelRetry.isUnreachable(raw)).toBeTrue()
|
||||
|
||||
const wrapped = yield* streamFailure(
|
||||
apiCallError({ message: "Cannot connect to API: getaddrinfo ENOTFOUND", isRetryable: true, cause: dns }),
|
||||
)
|
||||
expect(wrapped.reason).toMatchObject({ _tag: "Transport", operation: "request", code: "ENOTFOUND" })
|
||||
expect(ModelRetry.isUnreachable(wrapped)).toBeTrue()
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("classifies mid-stream socket drops as read transport errors", () =>
|
||||
Effect.gen(function* () {
|
||||
const cause = Object.assign(new Error("terminated"), {
|
||||
|
||||
@@ -1,18 +1,37 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { AIError, ProviderInternalError, TransportError } from "@opencode/ai/effect"
|
||||
import { ModelRetry } from "@opencode/core/model-retry"
|
||||
import { Agent } from "@opencode/schema/agent"
|
||||
import { Model } from "@opencode/schema/model"
|
||||
import { Provider } from "@opencode/core/provider"
|
||||
import { SessionSchema } from "@opencode/core/session/schema"
|
||||
import { SessionRunnerRetry } from "@opencode/core/session/runner/retry"
|
||||
import { toSessionError } from "@opencode/core/session/to-session-error"
|
||||
import { Effect } from "effect"
|
||||
import { Clock, Duration, Effect } from "effect"
|
||||
import { TestClock } from "effect/testing"
|
||||
import { it } from "./lib/effect"
|
||||
|
||||
const timeout = new AIError({
|
||||
reason: new TransportError({ message: "Timed out", transport: "http", operation: "request", code: "Timeout" }),
|
||||
})
|
||||
const internal = new AIError({ reason: new ProviderInternalError({ message: "internal" }) })
|
||||
const dnsFailure = new AIError({
|
||||
reason: new TransportError({
|
||||
message: "getaddrinfo ENOTFOUND opencode.ai",
|
||||
transport: "http",
|
||||
operation: "request",
|
||||
code: "ENOTFOUND",
|
||||
url: "https://opencode.ai/zen/v1/responses",
|
||||
}),
|
||||
})
|
||||
const refused = new AIError({
|
||||
reason: new TransportError({
|
||||
message: "Unable to connect",
|
||||
transport: "http",
|
||||
operation: "request",
|
||||
code: "ECONNREFUSED",
|
||||
}),
|
||||
})
|
||||
|
||||
const input = (cause: AIError) => ({
|
||||
cause,
|
||||
@@ -44,4 +63,68 @@ describe("SessionRunnerRetry.policy", () => {
|
||||
expect(yield* decisions(internal, 11)).toEqual([...Array(10).fill(true), false])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps retrying an unreachable host for an hour with gaps of at most 30 seconds", () =>
|
||||
Effect.gen(function* () {
|
||||
const policy = yield* SessionRunnerRetry.policy(SessionSchema.ID.make("ses_retry"))
|
||||
const started = yield* Clock.currentTimeMillis
|
||||
const delays: number[] = []
|
||||
while (true) {
|
||||
const decision = yield* policy(input(dnsFailure))
|
||||
if (!decision.retry) break
|
||||
delays.push(decision.delay)
|
||||
yield* TestClock.adjust(Duration.millis(decision.delay))
|
||||
}
|
||||
const waited = (yield* Clock.currentTimeMillis) - started
|
||||
expect(delays.length).toBeGreaterThan(100)
|
||||
expect(Math.max(...delays)).toBeLessThanOrEqual(30_000)
|
||||
expect(waited).toBeGreaterThanOrEqual(Duration.toMillis(ModelRetry.UNREACHABLE_WINDOW))
|
||||
expect(waited).toBeLessThanOrEqual(Duration.toMillis(ModelRetry.UNREACHABLE_WINDOW) + 30_000)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps a refused connection on the general allowance", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(ModelRetry.isUnreachable(refused)).toBe(false)
|
||||
expect(yield* decisions(refused, 11)).toEqual([...Array(10).fill(true), false])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("doesn't spend the general allowance on an unreachable host", () =>
|
||||
Effect.gen(function* () {
|
||||
const policy = yield* SessionRunnerRetry.policy(SessionSchema.ID.make("ses_retry"))
|
||||
for (let i = 0; i < 20; i++) expect((yield* policy(input(dnsFailure))).retry).toBe(true)
|
||||
const results: boolean[] = []
|
||||
for (let i = 0; i < 11; i++) results.push((yield* policy(input(internal))).retry)
|
||||
expect(results).toEqual([...Array(10).fill(true), false])
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
describe("ModelRetry.retryOneShot", () => {
|
||||
it.effect("doesn't replay a call whose response was cut off by the network going down", () =>
|
||||
Effect.gen(function* () {
|
||||
let calls = 0
|
||||
const cutOff = new AIError({
|
||||
reason: new TransportError({
|
||||
message: "Network is down",
|
||||
transport: "http",
|
||||
operation: "read",
|
||||
code: "ENETDOWN",
|
||||
}),
|
||||
})
|
||||
yield* ModelRetry.retryOneShot(Effect.suspend(() => (calls++, Effect.fail(cutOff)))).pipe(Effect.flip)
|
||||
expect(calls).toBe(1)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
describe("toSessionError", () => {
|
||||
test("names the network when the provider's host can't be reached", () => {
|
||||
expect(toSessionError(dnsFailure)).toEqual({
|
||||
type: "provider.transport",
|
||||
message: "Can't reach opencode.ai. Check the network connection. (getaddrinfo ENOTFOUND opencode.ai)",
|
||||
})
|
||||
expect(toSessionError(refused)).toEqual({ type: "provider.transport", message: "Unable to connect" })
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user