diff --git a/packages/ai/src/route/transport/websocket.ts b/packages/ai/src/route/transport/websocket.ts index 4e25c81ee99..2470a61b3ec 100644 --- a/packages/ai/src/route/transport/websocket.ts +++ b/packages/ai/src/route/transport/websocket.ts @@ -74,6 +74,11 @@ const eventMessage = (event: Socket.WebSocketEvent) => { return event.type ?? "error" } +const systemCode = (cause: unknown) => + typeof cause === "object" && cause !== null && "code" in cause && typeof cause.code === "string" + ? cause.code + : undefined + const binaryMessage = (data: unknown) => { if (data instanceof Uint8Array) return data if (data instanceof ArrayBuffer) return new Uint8Array(data) @@ -115,12 +120,15 @@ const waitOpen = (ws: Socket.WebSocketLike, input: WebSocketRequest) => { } const onError = (event: Socket.WebSocketEvent) => { cleanup() + const cause = "error" in event ? (event.error ?? event) : event resume( Effect.fail( transportError(`Failed to open WebSocket: ${eventMessage(event)}`, { - cause: "error" in event ? (event.error ?? event) : event, + cause, url: input.url, operation: "request", + // Bun puts the system code (ENOTFOUND, ECONNREFUSED) on the event's error; lift it like HTTP failures do. + code: systemCode(cause), phase: "connect", delivery: "not-sent", }), diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index c22f716ba24..9de3ab1a814 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -1459,6 +1459,28 @@ describe("OpenAI Responses route", () => { }), ) + it.effect("keeps the system code when a WebSocket fails to connect", () => + Effect.gen(function* () { + const socket = Object.assign(new EventTarget(), { readyState: globalThis.WebSocket.CONNECTING, close: () => {} }) + const failed = Object.assign(new Event("error"), { + error: Object.assign(new Error("Failed to connect"), { code: "ENOTFOUND" }), + }) + queueMicrotask(() => socket.dispatchEvent(failed)) + const error = yield* WebSocketTransport.fromWebSocket( + // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- waitOpen only listens for events on this branch. + socket as unknown as globalThis.WebSocket, + { url: "wss://api.openai.test/v1/responses", headers: Headers.empty }, + ).pipe(Effect.flip) + + expect(error.reason).toMatchObject({ + _tag: "Transport", + code: "ENOTFOUND", + phase: "connect", + delivery: "not-sent", + }) + }), + ) + it.effect("adds native query params to the Responses URL", () => Effect.gen(function* () { yield* LLMClient.generate({ diff --git a/packages/core/src/aisdk.ts b/packages/core/src/aisdk.ts index 9ef87235edd..d0899ddd134 100644 --- a/packages/core/src/aisdk.ts +++ b/packages/core/src/aisdk.ts @@ -934,7 +934,8 @@ function llmError(error: unknown, operation: "request" | "read") { const errorValue = (error: unknown) => (isRecord(error) ? error.value : undefined) // Runtime-generated network failure shapes. The codes mirror the AI SDK's own -// Bun network error list in handleFetchError; the messages are undici's fetch +// Bun network error list in handleFetchError, plus the unreachable-host codes that list leaves out and +// ModelRetry gives a longer budget; the messages are undici's fetch // TypeError and stream termination strings plus our header and chunk timeout errors. // Unrecognized shapes still retry via the UnknownProvider default; this match // only adds transport semantics (continuation eligibility, display). @@ -943,6 +944,11 @@ const NETWORK_ERROR_CODES = new Set([ "ECONNREFUSED", "ETIMEDOUT", "EPIPE", + "ENOTFOUND", + "EAI_AGAIN", + "ENETUNREACH", + "EHOSTUNREACH", + "ENETDOWN", "ConnectionRefused", "ConnectionClosed", "FailedToOpenSocket", @@ -989,6 +995,8 @@ function apiCallError(error: APICallError) { cause: error, }) if (error.statusCode !== undefined || !error.isRetryable) return failure + // "Cannot connect to API" wraps the runtime's error; keep its code so retry and display can tell why. + const code = networkFailure(error.cause)?.code return new AIError({ reason: new TransportError({ message: failure.message, @@ -998,6 +1006,7 @@ function apiCallError(error: APICallError) { transport: "http", operation: "request", url: error.url, + ...(code === undefined ? {} : { code }), }), }) } diff --git a/packages/core/src/model-retry.ts b/packages/core/src/model-retry.ts index 7a2e0fa5263..9ec5dbd1d49 100644 --- a/packages/core/src/model-retry.ts +++ b/packages/core/src/model-retry.ts @@ -35,18 +35,47 @@ const schedule = Schedule.max([ }), ) +// This machine has no network path to the provider: the host name didn't resolve, or there is no route or interface. +// While that lasts every attempt fails before reaching the provider, and the outage behind it (a router restart, a +// dropped VPN, a laptop between networks) usually outlasts the general allowance, so these retry on their own budget: +// exponential from 2s capped at 30s per gap, for up to an hour after the first one. A refused connection is left out: +// something answered, so it is the server's state. +const UNREACHABLE_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "ENETUNREACH", "EHOSTUNREACH", "ENETDOWN"]) +export const UNREACHABLE_WINDOW = Duration.hours(1) +const UNREACHABLE_GAP_MAX = 30_000 + +/** Whether the failure says this machine has no network path to the provider's host. */ +export const isUnreachable = (error: AIError) => + error.reason._tag === "Transport" && error.reason.code !== undefined && UNREACHABLE_CODES.has(error.reason.code) + +const unreachableSchedule = Schedule.max([ + Schedule.min([Schedule.exponential("2 seconds"), Schedule.spaced(Duration.millis(UNREACHABLE_GAP_MAX))]), + Schedule.during(UNREACHABLE_WINDOW), +]).pipe( + Schedule.jittered, + Schedule.setInputType(), + // Jitter scales the capped gap by up to 1.2, so cap it again after. + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.millis(Math.min(Math.ceil(Duration.toMillis(duration)), UNREACHABLE_GAP_MAX))), + ), +) + // A timed-out attempt already waited minutes before failing, so the general allowance would let a // dead provider hold a call for most of an hour. Cap those attempts well below it. const MAX_TIMEOUT_RETRIES = 3 export const policy = Effect.gen(function* () { const step = yield* Schedule.toStep(schedule) + const unreachable = yield* Schedule.toStep(unreachableSchedule) let attempt = 1 let timeouts = 0 return (input: Input) => Effect.gen(function* () { const now = yield* Clock.currentTimeMillis - const next = yield* step(now, input).pipe(Pull.catchDone(() => Effect.succeed(undefined))) + // Unreachable failures don't spend the general allowance, so a flaky link that also drops streams keeps it. + const next = yield* (input.retry && isUnreachable(input.cause) ? unreachable : step)(now, input).pipe( + Pull.catchDone(() => Effect.succeed(undefined)), + ) if (!next) return undefined const [, duration] = next attempt++ @@ -75,6 +104,7 @@ export const retry = (effect: Effect.Effect) => // One-shot generations (stateless text, images, decisions) can be billed per attempt, so they retry only failures // the provider never ran, a few times, with the same backoff and retry-after handling. const ONE_SHOT_RETRIES = 3 +// Only codes that fail before a connection exists: a dropped interface or route can also fail a response already being read. const NOT_SENT_CODES = new Set(["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN"]) const notRun = (error: AIError) => { diff --git a/packages/core/src/session/to-session-error.ts b/packages/core/src/session/to-session-error.ts index 812557ac542..91c90545046 100644 --- a/packages/core/src/session/to-session-error.ts +++ b/packages/core/src/session/to-session-error.ts @@ -5,6 +5,7 @@ import { Permission } from "../permission.js" import { Integration } from "../integration.js" import { AgentNotFoundError, StepFailedError } from "./error.js" import { ModelResolver } from "../model-resolver.js" +import { ModelRetry } from "../model-retry.js" import { SessionRunnerModel } from "./runner/model.js" const tokenSharingMessages = { @@ -49,6 +50,7 @@ export function toSessionError(cause: unknown): SessionError.Error { case "ContentPolicy": return providerError("provider.content-filter", cause.reason) case "Transport": + if (ModelRetry.isUnreachable(cause)) return unreachable(cause.reason) return providerError("provider.transport", cause.reason) case "ProviderInternal": return providerError("provider.internal", cause.reason) @@ -120,6 +122,15 @@ function goLimit(reason: AIError["reason"]): SessionError.Error { } } +// Names the network as the problem, so a DNS outage doesn't read like a broken model or provider. +function unreachable(reason: AIError["reason"]): SessionError.Error { + const host = "url" in reason && reason.url !== undefined ? URL.parse(reason.url)?.hostname : undefined + return { + ...providerError("provider.transport", reason), + message: `Can't reach ${host || "the provider"}. Check the network connection. (${reason.message})`, + } +} + function providerError(type: string, reason: AIError["reason"]): SessionError.Error { const status = reason.http?.status const retryAfterMs = "retryAfterMs" in reason ? reason.retryAfterMs : undefined diff --git a/packages/core/test/aisdk.test.ts b/packages/core/test/aisdk.test.ts index 657f6025cb6..0d5c862e23a 100644 --- a/packages/core/test/aisdk.test.ts +++ b/packages/core/test/aisdk.test.ts @@ -2,6 +2,7 @@ import { APICallError } from "@ai-sdk/provider" import type { LanguageModelV3, LanguageModelV3StreamPart } from "@ai-sdk/provider" import { createOpenAICompatible } from "@ai-sdk/openai-compatible" import { AISDK } from "@opencode/core/aisdk" +import { ModelRetry } from "@opencode/core/model-retry" import { SessionRunnerRetry } from "@opencode/core/session/runner/retry" import { toSessionError } from "@opencode/core/session/to-session-error" import { Model } from "@opencode/core/model" @@ -1198,6 +1199,21 @@ it.effect("classifies native fetch failures as request transport errors", () => }), ) +it.effect("keeps the code of a host name that didn't resolve", () => + Effect.gen(function* () { + const dns = Object.assign(new Error("getaddrinfo ENOTFOUND api.example.com"), { code: "ENOTFOUND" }) + const raw = yield* streamFailure(dns) + expect(raw.reason).toMatchObject({ _tag: "Transport", operation: "request", code: "ENOTFOUND" }) + expect(ModelRetry.isUnreachable(raw)).toBeTrue() + + const wrapped = yield* streamFailure( + apiCallError({ message: "Cannot connect to API: getaddrinfo ENOTFOUND", isRetryable: true, cause: dns }), + ) + expect(wrapped.reason).toMatchObject({ _tag: "Transport", operation: "request", code: "ENOTFOUND" }) + expect(ModelRetry.isUnreachable(wrapped)).toBeTrue() + }), +) + it.effect("classifies mid-stream socket drops as read transport errors", () => Effect.gen(function* () { const cause = Object.assign(new Error("terminated"), { diff --git a/packages/core/test/session-runner-retry.test.ts b/packages/core/test/session-runner-retry.test.ts index 8d344e85571..051549537ba 100644 --- a/packages/core/test/session-runner-retry.test.ts +++ b/packages/core/test/session-runner-retry.test.ts @@ -1,18 +1,37 @@ -import { describe, expect } from "bun:test" +import { describe, expect, test } from "bun:test" import { AIError, ProviderInternalError, TransportError } from "@opencode/ai/effect" +import { ModelRetry } from "@opencode/core/model-retry" import { Agent } from "@opencode/schema/agent" import { Model } from "@opencode/schema/model" import { Provider } from "@opencode/core/provider" import { SessionSchema } from "@opencode/core/session/schema" import { SessionRunnerRetry } from "@opencode/core/session/runner/retry" import { toSessionError } from "@opencode/core/session/to-session-error" -import { Effect } from "effect" +import { Clock, Duration, Effect } from "effect" +import { TestClock } from "effect/testing" import { it } from "./lib/effect" const timeout = new AIError({ reason: new TransportError({ message: "Timed out", transport: "http", operation: "request", code: "Timeout" }), }) const internal = new AIError({ reason: new ProviderInternalError({ message: "internal" }) }) +const dnsFailure = new AIError({ + reason: new TransportError({ + message: "getaddrinfo ENOTFOUND opencode.ai", + transport: "http", + operation: "request", + code: "ENOTFOUND", + url: "https://opencode.ai/zen/v1/responses", + }), +}) +const refused = new AIError({ + reason: new TransportError({ + message: "Unable to connect", + transport: "http", + operation: "request", + code: "ECONNREFUSED", + }), +}) const input = (cause: AIError) => ({ cause, @@ -44,4 +63,68 @@ describe("SessionRunnerRetry.policy", () => { expect(yield* decisions(internal, 11)).toEqual([...Array(10).fill(true), false]) }), ) + + it.effect("keeps retrying an unreachable host for an hour with gaps of at most 30 seconds", () => + Effect.gen(function* () { + const policy = yield* SessionRunnerRetry.policy(SessionSchema.ID.make("ses_retry")) + const started = yield* Clock.currentTimeMillis + const delays: number[] = [] + while (true) { + const decision = yield* policy(input(dnsFailure)) + if (!decision.retry) break + delays.push(decision.delay) + yield* TestClock.adjust(Duration.millis(decision.delay)) + } + const waited = (yield* Clock.currentTimeMillis) - started + expect(delays.length).toBeGreaterThan(100) + expect(Math.max(...delays)).toBeLessThanOrEqual(30_000) + expect(waited).toBeGreaterThanOrEqual(Duration.toMillis(ModelRetry.UNREACHABLE_WINDOW)) + expect(waited).toBeLessThanOrEqual(Duration.toMillis(ModelRetry.UNREACHABLE_WINDOW) + 30_000) + }), + ) + + it.effect("keeps a refused connection on the general allowance", () => + Effect.gen(function* () { + expect(ModelRetry.isUnreachable(refused)).toBe(false) + expect(yield* decisions(refused, 11)).toEqual([...Array(10).fill(true), false]) + }), + ) + + it.effect("doesn't spend the general allowance on an unreachable host", () => + Effect.gen(function* () { + const policy = yield* SessionRunnerRetry.policy(SessionSchema.ID.make("ses_retry")) + for (let i = 0; i < 20; i++) expect((yield* policy(input(dnsFailure))).retry).toBe(true) + const results: boolean[] = [] + for (let i = 0; i < 11; i++) results.push((yield* policy(input(internal))).retry) + expect(results).toEqual([...Array(10).fill(true), false]) + }), + ) +}) + +describe("ModelRetry.retryOneShot", () => { + it.effect("doesn't replay a call whose response was cut off by the network going down", () => + Effect.gen(function* () { + let calls = 0 + const cutOff = new AIError({ + reason: new TransportError({ + message: "Network is down", + transport: "http", + operation: "read", + code: "ENETDOWN", + }), + }) + yield* ModelRetry.retryOneShot(Effect.suspend(() => (calls++, Effect.fail(cutOff)))).pipe(Effect.flip) + expect(calls).toBe(1) + }), + ) +}) + +describe("toSessionError", () => { + test("names the network when the provider's host can't be reached", () => { + expect(toSessionError(dnsFailure)).toEqual({ + type: "provider.transport", + message: "Can't reach opencode.ai. Check the network connection. (getaddrinfo ENOTFOUND opencode.ai)", + }) + expect(toSessionError(refused)).toEqual({ type: "provider.transport", message: "Unable to connect" }) + }) })