mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-13 19:46:16 +03:00
feat(ai): add native Vercel AI Gateway language models (#52643)
This commit is contained in:
Aiden Cline
committed by
GitHub
Unverified
GitHub
parent
4ed2978a30
commit
c2d4db698c
@@ -1283,7 +1283,7 @@ const gateway = CloudflareAIGateway.configure({
|
||||
}).model("workers-ai/@cf/meta/llama-3.1-8b-instruct")
|
||||
```
|
||||
|
||||
Included LLM providers: OpenAI, Anthropic, Google (Gemini), Google Vertex, Amazon Bedrock, Azure OpenAI, Baseten, Cerebras, Cohere, Cloudflare AI Gateway, Cloudflare Workers AI, DeepInfra, DeepSeek, Fireworks, Groq, Mistral, OpenRouter, TogetherAI, and xAI. Z.ai currently exposes image generation. Generic Chat Completions, Responses, and Anthropic Messages-compatible entrypoints support custom endpoints.
|
||||
Included LLM providers: OpenAI, Anthropic, Google (Gemini), Google Vertex, Amazon Bedrock, Azure OpenAI, Baseten, Cerebras, Cohere, Cloudflare AI Gateway, Cloudflare Workers AI, DeepInfra, DeepSeek, Fireworks, Groq, Mistral, OpenRouter, TogetherAI, Vercel AI Gateway, and xAI. Z.ai currently exposes image generation. Generic Chat Completions, Responses, and Anthropic Messages-compatible entrypoints support custom endpoints.
|
||||
|
||||
Each named provider owns its module, endpoint, authentication, and route setup. Providers with the same wire format compose the shared protocol directly:
|
||||
|
||||
|
||||
@@ -48,21 +48,22 @@ const RESPECTS_INLINE_HINTS = new Set([
|
||||
"meta-messages",
|
||||
"minimax-messages",
|
||||
"moonshot-messages",
|
||||
"vercel-ai-gateway-messages",
|
||||
"zai-coding-messages",
|
||||
"bedrock-converse",
|
||||
"openrouter",
|
||||
"digitalocean",
|
||||
])
|
||||
|
||||
// OpenRouter upstreams other than Anthropic and Alibaba Qwen cache without breakpoints. Gemini uses only the last
|
||||
// breakpoint, so a conversation-tail breakpoint writes a new cache every step and costs more than none. Qwen ignores
|
||||
// breakpoints on tool definitions and caches tools with the system prompt.
|
||||
// OpenRouter and Vercel AI Gateway upstreams other than Anthropic and Alibaba Qwen cache without breakpoints.
|
||||
// Gemini uses only the last breakpoint, so a conversation-tail breakpoint writes a new cache every step and costs
|
||||
// more than none. Qwen ignores breakpoints on tool definitions and caches tools with the system prompt.
|
||||
const QWEN: CachePolicyObject = { system: true, messages: { tail: 1 } }
|
||||
const openRouterPolicy = (modelID: string): CachePolicyObject => {
|
||||
const gatewayPolicy = (modelID: string): CachePolicyObject => {
|
||||
// `~anthropic/claude-sonnet-latest` style IDs are OpenRouter aliases for the latest model in a family.
|
||||
const id = modelID.replace(/^~/, "")
|
||||
if (id.startsWith("anthropic/")) return AUTO
|
||||
if (id.startsWith("qwen/")) return QWEN
|
||||
if (id.startsWith("qwen/") || id.startsWith("alibaba/qwen")) return QWEN
|
||||
return NONE
|
||||
}
|
||||
|
||||
@@ -170,11 +171,13 @@ const countHints = (request: LLMRequest) =>
|
||||
)
|
||||
|
||||
export const applyCachePolicy = (request: LLMRequest): LLMRequest => {
|
||||
if (!RESPECTS_INLINE_HINTS.has(request.model.route.id)) return request
|
||||
const route = request.model.route.id
|
||||
if (!RESPECTS_INLINE_HINTS.has(route)) return request
|
||||
const policy =
|
||||
request.model.route.id === "openrouter" && (request.cache === undefined || request.cache === "auto")
|
||||
? openRouterPolicy(request.model.id)
|
||||
: request.model.route.id === "alibaba-chat" && (request.cache === undefined || request.cache === "auto")
|
||||
(route === "openrouter" || route === "vercel-ai-gateway-messages") &&
|
||||
(request.cache === undefined || request.cache === "auto")
|
||||
? gatewayPolicy(request.model.id)
|
||||
: route === "alibaba-chat" && (request.cache === undefined || request.cache === "auto")
|
||||
? request.model.id.toLowerCase().startsWith("qwen")
|
||||
? QWEN
|
||||
: NONE
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import { Effect, Option, Schema } from "effect"
|
||||
import { Protocol } from "../../route/protocol.js"
|
||||
import { LLMEvent, mergeJsonRecords, type AIError, type LLMRequest } from "../../schema/index.js"
|
||||
import { JsonObject, lenient } from "../shared.js"
|
||||
|
||||
interface ParserState<Inner> {
|
||||
readonly inner: Inner
|
||||
readonly gateway?: Record<string, unknown>
|
||||
}
|
||||
|
||||
const GatewayHolder = Schema.Struct({
|
||||
provider_metadata: lenient(
|
||||
Schema.Struct({
|
||||
gateway: lenient(JsonObject),
|
||||
}),
|
||||
),
|
||||
})
|
||||
|
||||
const GatewayEvent = Schema.Struct({
|
||||
...GatewayHolder.fields,
|
||||
response: lenient(GatewayHolder),
|
||||
choices: lenient(
|
||||
Schema.Array(
|
||||
Schema.Struct({
|
||||
delta: lenient(GatewayHolder),
|
||||
}),
|
||||
),
|
||||
),
|
||||
})
|
||||
const decodeGatewayEvent = Schema.decodeUnknownOption(GatewayEvent)
|
||||
|
||||
function attachGatewayMetadata(
|
||||
events: ReadonlyArray<LLMEvent>,
|
||||
gateway: Record<string, unknown> | undefined,
|
||||
): ReadonlyArray<LLMEvent> {
|
||||
if (!gateway || !events.some(LLMEvent.is.finish)) return events
|
||||
return events.map((event) =>
|
||||
LLMEvent.is.finish(event) ? { ...event, providerMetadata: { ...event.providerMetadata, gateway } } : event,
|
||||
)
|
||||
}
|
||||
|
||||
export function gatewayProtocol<Body, Event, State>(
|
||||
protocol: Protocol<Body, string, Event, State>,
|
||||
input: {
|
||||
readonly id: string
|
||||
readonly prepare: (
|
||||
request: LLMRequest,
|
||||
) => Effect.Effect<{ readonly request: LLMRequest; readonly body: Record<string, unknown> }, AIError>
|
||||
},
|
||||
) {
|
||||
const initial = (request: LLMRequest): ParserState<State> => ({
|
||||
inner: protocol.stream.initial(request),
|
||||
})
|
||||
const onHalt = protocol.stream.onHalt
|
||||
|
||||
return Protocol.make({
|
||||
id: input.id,
|
||||
body: {
|
||||
schema: JsonObject,
|
||||
from: Effect.fnUntraced(function* (request: LLMRequest) {
|
||||
const prepared = yield* input.prepare(request)
|
||||
const body = yield* protocol.body.from(prepared.request)
|
||||
return { ...body, ...prepared.body }
|
||||
}),
|
||||
},
|
||||
supportsEffortUpdates: protocol.supportsEffortUpdates,
|
||||
sanitizer: protocol.sanitizer,
|
||||
stream: {
|
||||
event: protocol.stream.event,
|
||||
initial,
|
||||
step: Effect.fnUntraced(function* (state: ParserState<State>, event: Event) {
|
||||
const decoded = Option.getOrUndefined(decodeGatewayEvent(event))
|
||||
const gateway = decoded
|
||||
? mergeJsonRecords(
|
||||
state.gateway,
|
||||
decoded.provider_metadata?.gateway,
|
||||
decoded.response?.provider_metadata?.gateway,
|
||||
...(decoded.choices ?? []).map((choice) => choice.delta?.provider_metadata?.gateway),
|
||||
)
|
||||
: state.gateway
|
||||
const [inner, events] = yield* protocol.stream.step(state.inner, event)
|
||||
return [{ inner, gateway }, attachGatewayMetadata(events, gateway)] as const
|
||||
}),
|
||||
terminal: protocol.stream.terminal,
|
||||
onHalt: onHalt
|
||||
? (state) => onHalt(state.inner).pipe(Effect.map((events) => attachGatewayMetadata(events, state.gateway)))
|
||||
: undefined,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -8,38 +8,226 @@ import {
|
||||
EvaluationResponse,
|
||||
EvaluationRounding,
|
||||
} from "../experimental/evaluation.js"
|
||||
import { AnthropicMessages } from "../protocols/anthropic-messages.js"
|
||||
import { OpenAIChat } from "../protocols/openai-chat.js"
|
||||
import { OpenResponses } from "../protocols/open-responses.js"
|
||||
import { optionalNull, ProviderShared } from "../protocols/shared.js"
|
||||
import { gatewayProtocol } from "../protocols/utils/gateway-protocol.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { Auth } from "../route/auth.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import type { Protocol } from "../route/protocol.js"
|
||||
import {
|
||||
AIError,
|
||||
HttpContext,
|
||||
HttpOptions,
|
||||
InvalidProviderOutputError,
|
||||
InvalidRequestError,
|
||||
LLMRequest,
|
||||
ModelID,
|
||||
ProviderID,
|
||||
ProviderMetadata,
|
||||
ReasoningEffort,
|
||||
Usage,
|
||||
} from "../schema/index.js"
|
||||
import type { OpenResponsesProviderOptionsInput } from "./open-responses-options.js"
|
||||
|
||||
export const id = ProviderID.make("vercel-ai-gateway")
|
||||
const baseURL = "https://ai-gateway.vercel.sh/v1"
|
||||
|
||||
export interface EvaluationOptions {
|
||||
export interface GatewayOptions {
|
||||
/** Service-owned options added by the Gateway without requiring an SDK update. */
|
||||
readonly [key: string]: unknown
|
||||
readonly gateway?: Readonly<{
|
||||
/** Enables Gateway automatic prompt-cache breakpoint injection (`"auto"`). */
|
||||
readonly caching?: "auto" | (string & {})
|
||||
/** Provider slugs that are the only ones allowed to serve the request (e.g. `["anthropic", "vertex"]`). */
|
||||
readonly only?: ReadonlyArray<string>
|
||||
/** Provider slugs specifying the order in which providers are tried (e.g. `["bedrock", "anthropic"]`). */
|
||||
readonly order?: ReadonlyArray<string>
|
||||
/** Sort candidate providers by cost (`"cost"`), throughput (`"tps"`), or time-to-first-token (`"ttft"`). */
|
||||
readonly sort?: "cost" | "tps" | "ttft" | (string & {})
|
||||
/** Fallback models to try in order, or a conditional `{ model, when }` entry on evaluation requests. */
|
||||
readonly models?: ReadonlyArray<string | Readonly<Record<string, unknown>>>
|
||||
/** Restrict routing to providers with zero data retention agreements. */
|
||||
readonly zeroDataRetention?: boolean
|
||||
/** Restrict routing to providers that do not train on prompt data. */
|
||||
readonly disallowPromptTraining?: boolean
|
||||
/**
|
||||
* Restrict routing to provider models that satisfy every entry: capability
|
||||
* tags (`"implicit-caching"`, `"reasoning"`, `"structured-output"`,
|
||||
* `"tool-use"`, `"vision"`) or weight-format filters (`"quantization:fp8"`,
|
||||
* `"!quantization:fp8"`).
|
||||
*/
|
||||
readonly has?: ReadonlyArray<
|
||||
| "implicit-caching"
|
||||
| "reasoning"
|
||||
| "structured-output"
|
||||
| "tool-use"
|
||||
| "vision"
|
||||
| `quantization:${string}`
|
||||
| `!quantization:${string}`
|
||||
| (string & {})
|
||||
>
|
||||
/** Entity identifier against which Gateway quota is tracked. */
|
||||
readonly quotaEntityId?: string
|
||||
/** Unified service tier intent (`"flex"` or `"priority"`). */
|
||||
readonly serviceTier?: "flex" | "priority" | (string & {})
|
||||
/** End-user identifier for spend tracking and attribution. */
|
||||
readonly user?: string
|
||||
/** User-specified tags for reporting and filtering usage. */
|
||||
readonly tags?: ReadonlyArray<string>
|
||||
/** Request-scoped BYOK credentials keyed by provider slug, used instead of cached workspace credentials. */
|
||||
readonly byok?: Readonly<Record<string, ReadonlyArray<Readonly<Record<string, unknown>>>>>
|
||||
/** Preferred inference region for upstream provider routing. */
|
||||
readonly inferenceRegion?: string
|
||||
/** Per-provider timeouts in milliseconds (e.g. `{ byok: { anthropic: 3000 } }`). */
|
||||
readonly providerTimeouts?: {
|
||||
readonly [key: string]: unknown
|
||||
readonly zeroDataRetention?: boolean
|
||||
readonly only?: ReadonlyArray<string>
|
||||
}>
|
||||
readonly byok?: Readonly<Record<string, number>>
|
||||
}
|
||||
}
|
||||
|
||||
export type Options = ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions.Input
|
||||
export type ProviderOptionsInput = OpenResponsesProviderOptionsInput &
|
||||
Omit<AnthropicMessages.OptionsInput, "thinking"> & {
|
||||
/** Reasoning configuration for Messages (`thinking`) or Chat (`reasoning.enabled` + `reasoning.max_tokens`). */
|
||||
readonly thinking?:
|
||||
| AnthropicMessages.OptionsInput["thinking"]
|
||||
| {
|
||||
readonly type: "enabled" | "adaptive" | "disabled" | (string & {})
|
||||
readonly budgetTokens?: number
|
||||
readonly budget_tokens?: number
|
||||
}
|
||||
/** Gateway routing, fallback, BYOK, compliance, and attribution options sent under `body.providerOptions.gateway`. */
|
||||
readonly gateway?: GatewayOptions
|
||||
/** Provider-specific options forwarded under their upstream namespace in `body.providerOptions` (e.g. `{ anthropic: { ... } }`). */
|
||||
readonly upstream?: Readonly<Record<string, Readonly<Record<string, unknown>>>>
|
||||
/** Responses API automatic-cache lifetime (`"5m"` or `"1h"`), sent as top-level `cache_ttl`. */
|
||||
readonly cacheTTL?: "5m" | "1h" | (string & {})
|
||||
/** Responses API count of stable input items to anchor for caching, sent as top-level `cache_anchor_items`. */
|
||||
readonly cacheAnchorItems?: number
|
||||
}
|
||||
|
||||
export interface EvaluationOptions {
|
||||
readonly [key: string]: unknown
|
||||
readonly gateway?: GatewayOptions
|
||||
}
|
||||
|
||||
export type Options = Omit<RouteDefaultsInput, "providerOptions"> &
|
||||
ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
readonly providerOptions?: ProviderOptionsInput
|
||||
}
|
||||
|
||||
export type Settings = ProviderPackage.Settings & ProviderOptionsInput & { readonly apiKey?: string }
|
||||
|
||||
const GatewayOptionsSchema = Schema.Struct({
|
||||
gateway: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
upstream: Schema.optional(Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))),
|
||||
reasoningEffort: Schema.optional(ReasoningEffort),
|
||||
thinking: Schema.optional(
|
||||
Schema.Struct({
|
||||
type: Schema.String,
|
||||
budgetTokens: Schema.optional(Schema.Number),
|
||||
budget_tokens: Schema.optional(Schema.Number),
|
||||
}),
|
||||
),
|
||||
cacheTTL: Schema.optional(Schema.String),
|
||||
cacheAnchorItems: Schema.optional(Schema.Number),
|
||||
})
|
||||
const decodeOptions = ProviderShared.validateWith(Schema.decodeUnknownEffect(GatewayOptionsSchema))
|
||||
|
||||
const prepare = (api: "messages" | "responses" | "chat") =>
|
||||
Effect.fnUntraced(function* (request: LLMRequest) {
|
||||
const options = yield* decodeOptions(request.providerOptions ?? {})
|
||||
const providerOptions = (() => {
|
||||
if (!options.upstream && !options.gateway) return undefined
|
||||
if (!options.gateway) return options.upstream
|
||||
return { ...options.upstream, gateway: options.gateway }
|
||||
})()
|
||||
switch (api) {
|
||||
case "messages": {
|
||||
const effort = options.reasoningEffort
|
||||
if (effort === undefined) return { request, body: { providerOptions } }
|
||||
const enabled = effort !== "none"
|
||||
const thinking = request.providerOptions?.thinking ?? { type: enabled ? "adaptive" : "disabled" }
|
||||
const next = LLMRequest.update(request, {
|
||||
providerOptions: {
|
||||
...request.providerOptions,
|
||||
effort: enabled ? effort : undefined,
|
||||
thinking,
|
||||
},
|
||||
})
|
||||
return { request: next, body: { providerOptions } }
|
||||
}
|
||||
case "responses":
|
||||
return {
|
||||
request,
|
||||
body: {
|
||||
providerOptions,
|
||||
cache_ttl: options.cacheTTL,
|
||||
cache_anchor_items: options.cacheAnchorItems,
|
||||
},
|
||||
}
|
||||
case "chat": {
|
||||
if (!options.thinking) return { request, body: { providerOptions } }
|
||||
const reasoning = {
|
||||
enabled: options.thinking.type !== "disabled",
|
||||
max_tokens: options.thinking.budgetTokens ?? options.thinking.budget_tokens,
|
||||
}
|
||||
return { request, body: { providerOptions, reasoning } }
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const route = <Body, Event, State>(input: {
|
||||
readonly id: string
|
||||
readonly protocol: Protocol<Body, string, Event, State>
|
||||
readonly api: "messages" | "responses" | "chat"
|
||||
readonly path: string
|
||||
readonly framing: Framing.Definition<string>
|
||||
readonly defaults?: RouteDefaultsInput
|
||||
}) =>
|
||||
Route.make({
|
||||
id: input.id,
|
||||
provider: id,
|
||||
providerMetadataKey: id,
|
||||
protocol: gatewayProtocol(input.protocol, { id: input.id, prepare: prepare(input.api) }),
|
||||
endpoint: Endpoint.path(input.path, { baseURL }),
|
||||
framing: input.framing,
|
||||
headers: ({ request }): Record<string, string> =>
|
||||
request.promptCacheKey ? { "x-session-affinity": request.promptCacheKey } : {},
|
||||
defaults: input.defaults,
|
||||
})
|
||||
|
||||
const messagesRoute = route({
|
||||
id: "vercel-ai-gateway-messages",
|
||||
protocol: AnthropicMessages.protocol,
|
||||
api: "messages",
|
||||
path: "/messages",
|
||||
framing: AnthropicMessages.framing,
|
||||
defaults: { headers: { "anthropic-version": "2023-06-01" } },
|
||||
})
|
||||
const responsesRoute = route({
|
||||
id: "vercel-ai-gateway-responses",
|
||||
protocol: OpenResponses.protocol,
|
||||
api: "responses",
|
||||
path: "/responses",
|
||||
framing: Framing.sse,
|
||||
defaults: { providerOptions: { store: false, include: ["reasoning.encrypted_content"] } },
|
||||
})
|
||||
const chatRoute = route({
|
||||
id: "vercel-ai-gateway-chat",
|
||||
protocol: OpenAIChat.protocol,
|
||||
api: "chat",
|
||||
path: "/chat/completions",
|
||||
framing: OpenAIChat.framing,
|
||||
})
|
||||
|
||||
export const routes = [messagesRoute, responsesRoute, chatRoute]
|
||||
|
||||
const Request = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
model: Schema.String,
|
||||
@@ -50,19 +238,43 @@ const Request = Schema.StructWithRest(
|
||||
[Schema.Record(Schema.String, Schema.Any)],
|
||||
)
|
||||
const Response = Schema.Struct({
|
||||
model: Schema.optional(Schema.String),
|
||||
model: optionalNull(Schema.String),
|
||||
answers: Schema.Record(Schema.String, EvaluationAnswer),
|
||||
usage: Schema.optional(
|
||||
usage: optionalNull(
|
||||
Schema.Struct({
|
||||
inputTokens: Schema.optional(Schema.Number),
|
||||
outputTokens: Schema.optional(Schema.Number),
|
||||
inputTokens: optionalNull(Schema.Number),
|
||||
outputTokens: optionalNull(Schema.Number),
|
||||
}),
|
||||
),
|
||||
rounding: Schema.optional(EvaluationRounding),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
rounding: optionalNull(EvaluationRounding),
|
||||
providerMetadata: optionalNull(ProviderMetadata),
|
||||
})
|
||||
|
||||
export const configure = (input: Options = {}) => {
|
||||
const { apiKey: _apiKey, auth: _auth, baseURL: endpoint, ...defaults } = input
|
||||
const configured = {
|
||||
...defaults,
|
||||
endpoint: { baseURL: endpoint ?? baseURL },
|
||||
auth: AuthOptions.bearer(input, ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN"]),
|
||||
}
|
||||
const messages = (modelID: string | ModelID) =>
|
||||
messagesRoute.with(configured).model<ProviderOptionsInput>({
|
||||
id: modelID,
|
||||
// Recorded Gateway translations for non-Claude models return thinking with empty signatures.
|
||||
compatibility: { requireSignature: modelID.startsWith("anthropic/") },
|
||||
})
|
||||
const responses = (modelID: string | ModelID) =>
|
||||
responsesRoute.with(configured).model<ProviderOptionsInput>({ id: modelID })
|
||||
const chat = (modelID: string | ModelID) =>
|
||||
chatRoute
|
||||
.with(configured)
|
||||
.model<ProviderOptionsInput>({ id: modelID, compatibility: { reasoningField: "reasoning" } })
|
||||
// Each family uses the API whose Gateway translation carries its reasoning state across turns.
|
||||
const model = (modelID: string | ModelID) => {
|
||||
if (/^(openai\/gpt-|spacexai\/grok-)/.test(modelID)) return responses(modelID)
|
||||
if (modelID.startsWith("meta/muse-")) return chat(modelID)
|
||||
return messages(modelID)
|
||||
}
|
||||
const evaluation = (modelID: string | ModelID) =>
|
||||
EvaluationModel.make<EvaluationOptions>({
|
||||
id: modelID,
|
||||
@@ -85,9 +297,7 @@ export const configure = (input: Options = {}) => {
|
||||
(cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }),
|
||||
),
|
||||
)
|
||||
const headers = yield* Auth.toEffect(
|
||||
AuthOptions.bearer(input, ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN"]),
|
||||
)({
|
||||
const headers = yield* Auth.toEffect(configured.auth)({
|
||||
request: req,
|
||||
method: "POST",
|
||||
url: url.toString(),
|
||||
@@ -119,30 +329,54 @@ export const configure = (input: Options = {}) => {
|
||||
fail("Vercel AI Gateway returned an invalid evaluation response", cause, text),
|
||||
),
|
||||
)
|
||||
const inputTokens = data.usage?.inputTokens ?? undefined
|
||||
const outputTokens = data.usage?.outputTokens ?? undefined
|
||||
const usage = data.usage
|
||||
? new Usage({
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens: ProviderShared.totalTokens(inputTokens, outputTokens, undefined),
|
||||
providerMetadata: { gateway: data.usage },
|
||||
})
|
||||
: undefined
|
||||
return new EvaluationResponse({
|
||||
model: ModelID.make(data.model ?? req.model.id),
|
||||
answers: data.answers,
|
||||
usage: data.usage
|
||||
? new Usage({
|
||||
inputTokens: data.usage.inputTokens,
|
||||
outputTokens: data.usage.outputTokens,
|
||||
totalTokens:
|
||||
data.usage.inputTokens === undefined && data.usage.outputTokens === undefined
|
||||
? undefined
|
||||
: (data.usage.inputTokens ?? 0) + (data.usage.outputTokens ?? 0),
|
||||
providerMetadata: { gateway: data.usage },
|
||||
})
|
||||
: undefined,
|
||||
rounding: data.rounding,
|
||||
providerMetadata: data.providerMetadata,
|
||||
usage,
|
||||
rounding: data.rounding ?? undefined,
|
||||
providerMetadata: data.providerMetadata ?? undefined,
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
return { id, experimental: { evaluation }, configure }
|
||||
return { id, model, messages, responses, chat, experimental: { evaluation }, configure }
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const experimental = provider.experimental
|
||||
export const messages = provider.messages
|
||||
export const responses = provider.responses
|
||||
export const chat = provider.chat
|
||||
|
||||
export const model: ProviderPackage.Definition<Settings, ProviderOptionsInput>["model"] = (modelID, settings) =>
|
||||
fromSettings(settings).model(modelID)
|
||||
export const messagesModel: ProviderPackage.Definition<Settings, ProviderOptionsInput>["model"] = (modelID, settings) =>
|
||||
fromSettings(settings).messages(modelID)
|
||||
export const responsesModel: ProviderPackage.Definition<Settings, ProviderOptionsInput>["model"] = (
|
||||
modelID,
|
||||
settings,
|
||||
) => fromSettings(settings).responses(modelID)
|
||||
export const chatModel: ProviderPackage.Definition<Settings, ProviderOptionsInput>["model"] = (modelID, settings) =>
|
||||
fromSettings(settings).chat(modelID)
|
||||
|
||||
function fromSettings({ apiKey, baseURL, headers, body, ...providerOptions }: Settings) {
|
||||
return configure({
|
||||
apiKey,
|
||||
baseURL,
|
||||
headers,
|
||||
http: body === undefined ? undefined : { body: { ...body } },
|
||||
providerOptions,
|
||||
})
|
||||
}
|
||||
|
||||
export * as VercelAIGateway from "./vercel-ai-gateway.js"
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
export { chatModel as model } from "../vercel-ai-gateway.js"
|
||||
export type { Settings } from "../vercel-ai-gateway.js"
|
||||
@@ -0,0 +1,2 @@
|
||||
export { messagesModel as model } from "../vercel-ai-gateway.js"
|
||||
export type { Settings } from "../vercel-ai-gateway.js"
|
||||
@@ -0,0 +1,2 @@
|
||||
export { responsesModel as model } from "../vercel-ai-gateway.js"
|
||||
export type { Settings } from "../vercel-ai-gateway.js"
|
||||
+49
File diff suppressed because one or more lines are too long
Vendored
+49
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:vercel-native",
|
||||
"provider:vercel-ai-gateway"
|
||||
],
|
||||
"name": "vercel-native/continues-chat-claude-reasoning-and-tools",
|
||||
"recordedAt": "2026-10-02T03:35:31.909Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-5.5\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"reasoning_effort\":\"low\",\"max_completion_tokens\":2048}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"id\":\"gen_01M3XAWTT89S61AZ5BSE3W3SA9\",\"object\":\"chat.completion.chunk\",\"created\":1790912130,\"model\":\"anthropic/claude-sonnet-5.5\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_9rbv9d39c7\"}\n\ndata: {\"id\":\"gen_01M3XAWTT89S61AZ5BSE3W3SA9\",\"object\":\"chat.completion.chunk\",\"created\":1790912130,\"model\":\"anthropic/claude-sonnet-5.5\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"toolu_01GV2kwNUpDVDUXK3nekcJGf\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_9rbv9d39c7\"}\n\ndata: {\"id\":\"gen_01M3XAWTT89S61AZ5BSE3W3SA9\",\"object\":\"chat.completion.chunk\",\"created\":1790912130,\"model\":\"anthropic/claude-sonnet-5.5\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\": \\\"Paris\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_9rbv9d39c7\"}\n\ndata: {\"id\":\"gen_01M3XAWTT89S61AZ5BSE3W3SA9\",\"object\":\"chat.completion.chunk\",\"created\":1790912130,\"model\":\"anthropic/claude-sonnet-5.5\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_9rbv9d39c7\"}\n\ndata: {\"id\":\"gen_01M3XAWTT89S61AZ5BSE3W3SA9\",\"object\":\"chat.completion.chunk\",\"created\":1790912130,\"model\":\"anthropic/claude-sonnet-5.5\",\"choices\":[{\"index\":0,\"delta\":{\"provider_metadata\":{\"anthropic\":{\"usage\":{\"input_tokens\":420,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":50,\"service_tier\":\"standard\",\"inference_geo\":\"global\",\"output_tokens_details\":{\"thinking_tokens\":0}},\"cacheCreationInputTokens\":0,\"stopSequence\":null,\"iterations\":null,\"container\":null,\"contextManagement\":null},\"gateway\":{\"routing\":{\"originalModelId\":\"anthropic/claude-sonnet-5.5\",\"resolvedProvider\":\"anthropic\",\"fallbacksAvailable\":[\"claudeaws\",\"bedrock\",\"vertexAnthropic\"],\"planningReasoning\":\"Weighted tie pick: anthropic won tier 0 over claudeaws. System credentials planned for: anthropic, claudeaws, bedrock, vertexAnthropic. Total execution order: anthropic(system) → claudeaws(system) → bedrock(system) → vertexAnthropic(system)\",\"canonicalSlug\":\"anthropic/claude-sonnet-5.5\",\"finalProvider\":\"anthropic\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"anthropic/claude-sonnet-5.5\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"anthropic\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1790912129901,\"endTime\":1790912130999,\"providerRequestId\":\"req_011Cfci8AT17gzqZn2jJEXeT\",\"statusCode\":200,\"providerResponseId\":\"msg_011Cfci8AtJRw1vue5qnmohn\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"skipped_below_min_prefix\"},\"clientSessionId\":\"a47cd177650917b88c4d443d7bd8be68\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.00134\",\"marketCost\":\"0.00134\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.00134\",\"inferenceCost\":\"0.00134\",\"inputInferenceCost\":\"0.00084\",\"outputInferenceCost\":\"0.0005\",\"generationId\":\"gen_01M3XAWTT89S61AZ5BSE3W3SA9\"}}},\"logprobs\":null,\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":420,\"completion_tokens\":50,\"total_tokens\":470,\"cost\":0.00134,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0,\"upstream_inference_completions_cost\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0},\"cache_creation_input_tokens\":0,\"market_cost\":0.00134,\"gateway_cost\":0.00134},\"system_fingerprint\":\"fp_9rbv9d39c7\",\"generationId\":\"gen_01M3XAWTT89S61AZ5BSE3W3SA9\"}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-5.5\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"toolu_01GV2kwNUpDVDUXK3nekcJGf\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}],\"reasoning\":\"\"},{\"role\":\"tool\",\"tool_call_id\":\"toolu_01GV2kwNUpDVDUXK3nekcJGf\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"reasoning_effort\":\"low\",\"max_completion_tokens\":2048}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"id\":\"gen_01M3XAWW1EQYN8E229MXPGQ69V\",\"object\":\"chat.completion.chunk\",\"created\":1790912131,\"model\":\"anthropic/claude-sonnet-5.5\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_deco7uk6x4\"}\n\ndata: {\"id\":\"gen_01M3XAWW1EQYN8E229MXPGQ69V\",\"object\":\"chat.completion.chunk\",\"created\":1790912131,\"model\":\"anthropic/claude-sonnet-5.5\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Paris is sunny.\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_deco7uk6x4\"}\n\ndata: {\"id\":\"gen_01M3XAWW1EQYN8E229MXPGQ69V\",\"object\":\"chat.completion.chunk\",\"created\":1790912131,\"model\":\"anthropic/claude-sonnet-5.5\",\"choices\":[{\"index\":0,\"delta\":{\"provider_metadata\":{\"anthropic\":{\"usage\":{\"input_tokens\":493,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":10,\"service_tier\":\"standard\",\"inference_geo\":\"global\",\"output_tokens_details\":{\"thinking_tokens\":0}},\"cacheCreationInputTokens\":0,\"stopSequence\":null,\"iterations\":null,\"container\":null,\"contextManagement\":null},\"gateway\":{\"routing\":{\"originalModelId\":\"anthropic/claude-sonnet-5.5\",\"resolvedProvider\":\"anthropic\",\"fallbacksAvailable\":[\"claudeaws\",\"bedrock\",\"vertexAnthropic\"],\"planningReasoning\":\"Weighted tie pick: anthropic won tier 0 over claudeaws. System credentials planned for: anthropic, claudeaws, bedrock, vertexAnthropic. Total execution order: anthropic(system) → claudeaws(system) → bedrock(system) → vertexAnthropic(system)\",\"canonicalSlug\":\"anthropic/claude-sonnet-5.5\",\"finalProvider\":\"anthropic\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"anthropic/claude-sonnet-5.5\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"anthropic\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1790912131152,\"endTime\":1790912131960,\"providerRequestId\":\"req_011Cfci8FjVzcog3yJtTFw81\",\"statusCode\":200,\"providerResponseId\":\"msg_011Cfci8G5pr3sikiEmpeSSo\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"skipped_below_min_prefix\"},\"clientSessionId\":\"a47cd177650917b88c4d443d7bd8be68\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.001086\",\"marketCost\":\"0.001086\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.001086\",\"inferenceCost\":\"0.001086\",\"inputInferenceCost\":\"0.000986\",\"outputInferenceCost\":\"0.0001\",\"generationId\":\"gen_01M3XAWW1EQYN8E229MXPGQ69V\"}}},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":493,\"completion_tokens\":10,\"total_tokens\":503,\"cost\":0.001086,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0,\"upstream_inference_completions_cost\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0},\"cache_creation_input_tokens\":0,\"market_cost\":0.001086,\"gateway_cost\":0.001086},\"system_fingerprint\":\"fp_deco7uk6x4\",\"generationId\":\"gen_01M3XAWW1EQYN8E229MXPGQ69V\"}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+49
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:vercel-native",
|
||||
"provider:vercel-ai-gateway"
|
||||
],
|
||||
"name": "vercel-native/continues-chat-muse-reasoning-and-tools",
|
||||
"recordedAt": "2026-10-05T02:56:27.988Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"meta/muse-spark-1.3\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"reasoning_effort\":\"low\",\"max_completion_tokens\":2048}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"id\":\"gen_01M44ZVD02HF12MJ8X2S7SK1E6\",\"object\":\"chat.completion.chunk\",\"created\":1791168985,\"model\":\"meta/muse-spark-1.3\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_h4uvdop29h\"}\n\ndata: {\"id\":\"gen_01M44ZVD02HF12MJ8X2S7SK1E6\",\"object\":\"chat.completion.chunk\",\"created\":1791168985,\"model\":\"meta/muse-spark-1.3\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"I'll look up the weather in Paris for you.\",\"phase\":\"commentary\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_h4uvdop29h\"}\n\ndata: {\"id\":\"gen_01M44ZVD02HF12MJ8X2S7SK1E6\",\"object\":\"chat.completion.chunk\",\"created\":1791168985,\"model\":\"meta/muse-spark-1.3\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_01a109fdbbdd7347a6918c3662c8128e\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_h4uvdop29h\"}\n\ndata: {\"id\":\"gen_01M44ZVD02HF12MJ8X2S7SK1E6\",\"object\":\"chat.completion.chunk\",\"created\":1791168985,\"model\":\"meta/muse-spark-1.3\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_h4uvdop29h\"}\n\ndata: {\"id\":\"gen_01M44ZVD02HF12MJ8X2S7SK1E6\",\"object\":\"chat.completion.chunk\",\"created\":1791168985,\"model\":\"meta/muse-spark-1.3\",\"choices\":[{\"index\":0,\"delta\":{\"provider_metadata\":{\"openai\":{\"responseId\":\"resp_6ac311d9306b8844d6104957\",\"serviceTier\":\"auto\"},\"gateway\":{\"routing\":{\"originalModelId\":\"meta/muse-spark-1.3\",\"resolvedProvider\":\"meta\",\"fallbacksAvailable\":[],\"planningReasoning\":\"System credentials planned for: meta. Total execution order: meta(system)\",\"canonicalSlug\":\"meta/muse-spark-1.3\",\"finalProvider\":\"meta\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"meta/muse-spark-1.3\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"meta\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1791168984107,\"endTime\":1791168986296,\"providerRequestId\":\"f93f8fd8-1b40-49b2-8a0c-222d6e70af95\",\"statusCode\":200,\"providerResponseId\":\"resp_6ac311d9306b8844d6104957\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"skipped_below_min_prefix\"},\"clientSessionId\":\"a47cd177650917b88c4d443d7bd8be68\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.0012462\",\"marketCost\":\"0.0012462\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.0012462\",\"inferenceCost\":\"0.0012462\",\"inputInferenceCost\":\"0.0005832\",\"outputInferenceCost\":\"0.000663\",\"generationId\":\"gen_01M44ZVD02HF12MJ8X2S7SK1E6\"}}},\"logprobs\":null,\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":566,\"completion_tokens\":156,\"total_tokens\":722,\"cost\":0.0012462,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":113,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0,\"upstream_inference_completions_cost\":0},\"completion_tokens_details\":{\"reasoning_tokens\":87,\"image_tokens\":0},\"cache_creation_input_tokens\":0,\"market_cost\":0.0012462,\"gateway_cost\":0.0012462},\"system_fingerprint\":\"fp_h4uvdop29h\",\"service_tier\":\"auto\",\"generationId\":\"gen_01M44ZVD02HF12MJ8X2S7SK1E6\"}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"meta/muse-spark-1.3\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"I'll look up the weather in Paris for you.\",\"tool_calls\":[{\"id\":\"call_01a109fdbbdd7347a6918c3662c8128e\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}],\"reasoning\":\"\"},{\"role\":\"tool\",\"tool_call_id\":\"call_01a109fdbbdd7347a6918c3662c8128e\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"reasoning_effort\":\"low\",\"max_completion_tokens\":2048}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"id\":\"gen_01M44ZVF8Y196MQE5QNXBYK63X\",\"object\":\"chat.completion.chunk\",\"created\":1791168987,\"model\":\"meta/muse-spark-1.3\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_t8q87incvs\"}\n\ndata: {\"id\":\"gen_01M44ZVF8Y196MQE5QNXBYK63X\",\"object\":\"chat.completion.chunk\",\"created\":1791168987,\"model\":\"meta/muse-spark-1.3\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Paris is sunny.\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_t8q87incvs\"}\n\ndata: {\"id\":\"gen_01M44ZVF8Y196MQE5QNXBYK63X\",\"object\":\"chat.completion.chunk\",\"created\":1791168987,\"model\":\"meta/muse-spark-1.3\",\"choices\":[{\"index\":0,\"delta\":{\"provider_metadata\":{\"openai\":{\"responseId\":\"resp_6ac311db491fe86e00954e5c\",\"serviceTier\":\"auto\"},\"gateway\":{\"routing\":{\"originalModelId\":\"meta/muse-spark-1.3\",\"resolvedProvider\":\"meta\",\"fallbacksAvailable\":[],\"planningReasoning\":\"System credentials planned for: meta. Total execution order: meta(system)\",\"canonicalSlug\":\"meta/muse-spark-1.3\",\"finalProvider\":\"meta\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"meta/muse-spark-1.3\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"meta\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1791168986439,\"endTime\":1791168988032,\"providerRequestId\":\"88abdbee-80b6-444b-a295-16cd5b25801f\",\"statusCode\":200,\"providerResponseId\":\"resp_6ac311db491fe86e00954e5c\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"skipped_below_min_prefix\"},\"clientSessionId\":\"a47cd177650917b88c4d443d7bd8be68\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.0011\",\"marketCost\":\"0.0011\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.0011\",\"inferenceCost\":\"0.0011\",\"inputInferenceCost\":\"0.00082375\",\"outputInferenceCost\":\"0.00027625\",\"generationId\":\"gen_01M44ZVF8Y196MQE5QNXBYK63X\"}}},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":659,\"completion_tokens\":65,\"total_tokens\":724,\"cost\":0.0011,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0,\"upstream_inference_completions_cost\":0},\"completion_tokens_details\":{\"reasoning_tokens\":51,\"image_tokens\":0},\"cache_creation_input_tokens\":0,\"market_cost\":0.0011,\"gateway_cost\":0.0011},\"system_fingerprint\":\"fp_t8q87incvs\",\"service_tier\":\"auto\",\"generationId\":\"gen_01M44ZVF8Y196MQE5QNXBYK63X\"}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:vercel-native",
|
||||
"provider:vercel-ai-gateway"
|
||||
],
|
||||
"name": "vercel-native/continues-messages-claude-reasoning-and-tools",
|
||||
"recordedAt": "2026-10-05T05:01:43.431Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/messages",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-5.5\",\"system\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":2048,\"thinking\":{\"type\":\"adaptive\",\"block_binding\":{\"prefix_mismatch_behavior\":\"drop_block\"}},\"output_config\":{\"effort\":\"low\"}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"gen_01M4570TCVSBA1XA414GAQ4CXD\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"anthropic/claude-sonnet-5.5\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01HUuDeJSxrGVzp36mnVyXyv\",\"name\":\"get_weather\",\"input\":{}}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"city\\\": \\\"Paris\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":420,\"output_tokens\":50,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0}},\"input_transformations\":[],\"provider_metadata\":{\"anthropic\":{\"usage\":{\"input_tokens\":420,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":50,\"service_tier\":\"standard\",\"inference_geo\":\"global\",\"output_tokens_details\":{\"thinking_tokens\":0}},\"cacheCreationInputTokens\":0,\"stopSequence\":null,\"inputTransformations\":[],\"iterations\":null,\"container\":null,\"contextManagement\":null},\"gateway\":{\"routing\":{\"originalModelId\":\"anthropic/claude-sonnet-5.5\",\"resolvedProvider\":\"anthropic\",\"fallbacksAvailable\":[\"bedrock\",\"vertexAnthropic\",\"claudeaws\"],\"planningReasoning\":\"System credentials planned for: anthropic, bedrock, vertexAnthropic, claudeaws. Total execution order: anthropic(system) → bedrock(system) → vertexAnthropic(system) → claudeaws(system)\",\"canonicalSlug\":\"anthropic/claude-sonnet-5.5\",\"finalProvider\":\"anthropic\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"anthropic/claude-sonnet-5.5\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"anthropic\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1791176501800,\"endTime\":1791176502677,\"providerRequestId\":\"req_011CfiW89nSemMDq17221pkX\",\"statusCode\":200,\"providerResponseId\":\"msg_011CfiW8ATdTzcemBmD1hcuT\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"skipped_below_min_prefix\"},\"clientSessionId\":\"a47cd177650917b88c4d443d7bd8be68\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.00134\",\"marketCost\":\"0.00134\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.00134\",\"inferenceCost\":\"0.00134\",\"inputInferenceCost\":\"0.00084\",\"outputInferenceCost\":\"0.0005\",\"generationId\":\"gen_01M4570TCVSBA1XA414GAQ4CXD\"}}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/messages",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-5.5\",\"system\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01HUuDeJSxrGVzp36mnVyXyv\",\"name\":\"get_weather\",\"input\":{\"city\":\"Paris\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_01HUuDeJSxrGVzp36mnVyXyv\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":2048,\"thinking\":{\"type\":\"adaptive\",\"block_binding\":{\"prefix_mismatch_behavior\":\"drop_block\"}},\"output_config\":{\"effort\":\"low\"}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"gen_01M4570VFYNAEGVS60AYMPC6NG\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"anthropic/claude-sonnet-5.5\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Paris is sunny.\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":493,\"output_tokens\":10,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0}},\"input_transformations\":[],\"provider_metadata\":{\"anthropic\":{\"usage\":{\"input_tokens\":493,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":10,\"service_tier\":\"standard\",\"inference_geo\":\"global\",\"output_tokens_details\":{\"thinking_tokens\":0}},\"cacheCreationInputTokens\":0,\"stopSequence\":null,\"inputTransformations\":[],\"iterations\":null,\"container\":null,\"contextManagement\":null},\"gateway\":{\"routing\":{\"originalModelId\":\"anthropic/claude-sonnet-5.5\",\"resolvedProvider\":\"anthropic\",\"fallbacksAvailable\":[\"bedrock\",\"vertexAnthropic\",\"claudeaws\"],\"planningReasoning\":\"System credentials planned for: anthropic, bedrock, vertexAnthropic, claudeaws. Total execution order: anthropic(system) → bedrock(system) → vertexAnthropic(system) → claudeaws(system)\",\"canonicalSlug\":\"anthropic/claude-sonnet-5.5\",\"finalProvider\":\"anthropic\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"anthropic/claude-sonnet-5.5\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"anthropic\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1791176502820,\"endTime\":1791176503470,\"providerRequestId\":\"req_011CfiW8EBMRJC1mJAyNfuhc\",\"statusCode\":200,\"providerResponseId\":\"msg_011CfiW8EYw9svWDkh3UxuCK\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"skipped_below_min_prefix\"},\"clientSessionId\":\"a47cd177650917b88c4d443d7bd8be68\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.001086\",\"marketCost\":\"0.001086\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.001086\",\"inferenceCost\":\"0.001086\",\"inputInferenceCost\":\"0.000986\",\"outputInferenceCost\":\"0.0001\",\"generationId\":\"gen_01M4570VFYNAEGVS60AYMPC6NG\"}}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:vercel-native",
|
||||
"provider:vercel-ai-gateway"
|
||||
],
|
||||
"name": "vercel-native/continues-messages-deepseek-reasoning-and-tools",
|
||||
"recordedAt": "2026-10-05T02:56:29.224Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/messages",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"deepseek/deepseek-v4.1-flash\",\"system\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":2048,\"thinking\":{\"type\":\"adaptive\"},\"output_config\":{\"effort\":\"high\"}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"gen_01M44ZVH1C4678Q0TYP7DRJ7J2\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"deepseek/deepseek-v4.1-flash\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"The\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" user wants\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" me to call\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" get_weather once\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" for Paris, then\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" reply exactly \\\"Paris\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" is sunny.\\\"\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"\\n\\n\"}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":2,\"content_block\":{\"type\":\"tool_use\",\"id\":\"chatcmpl-tool-51e80fcbd5064b6e8bc9b8d0fa1d4e1d\",\"name\":\"get_weather\",\"input\":{}}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"city\\\": \\\"Paris\\\"}\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":2}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":324,\"output_tokens\":61},\"provider_metadata\":{\"baseten\":{\"acceptedPredictionTokens\":44,\"rejectedPredictionTokens\":4},\"gateway\":{\"routing\":{\"originalModelId\":\"deepseek/deepseek-v4.1-flash\",\"resolvedProvider\":\"baseten\",\"fallbacksAvailable\":[\"fireworks\",\"alibaba\",\"runware\",\"relace\",\"particle\",\"novita\",\"togetherai\",\"deepinfra\",\"wafer\",\"parasail\",\"gmicloud\",\"modal\",\"morph\",\"deepseek\",\"runinfra\",\"boundless\",\"inference-net\"],\"planningReasoning\":\"System credentials planned for: baseten, fireworks, alibaba, runware, relace, particle, novita, togetherai, deepinfra, wafer, parasail, gmicloud, modal, morph, deepseek, runinfra, boundless, inference-net. Total execution order: baseten(system) → fireworks(system) → alibaba(system) → runware(system) → relace(system) → particle(system) → novita(system) → togetherai(system) → deepinfra(system) → wafer(system) → parasail(system) → gmicloud(system) → modal(system) → morph(system) → deepseek(system) → runinfra(system) → boundless(system) → inference-net(system)\",\"canonicalSlug\":\"deepseek/deepseek-v4.1-flash\",\"finalProvider\":\"baseten\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"deepseek/deepseek-v4.1-flash\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"baseten\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1791168988247,\"endTime\":1791168988517,\"providerRequestId\":\"54ca701a109fdc46b7da8b6ab43570cc519dd\",\"statusCode\":200,\"providerResponseId\":\"chatcmpl-4234517cc89342a79b53a68ac41601eb\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"skipped_below_min_prefix\"},\"clientSessionId\":\"a47cd177650917b88c4d443d7bd8be68\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.0001704\",\"marketCost\":\"0.0001704\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.0001704\",\"inferenceCost\":\"0.0001704\",\"inputInferenceCost\":\"0.0000972\",\"outputInferenceCost\":\"0.0000732\",\"generationId\":\"gen_01M44ZVH1C4678Q0TYP7DRJ7J2\"}}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/messages",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"deepseek/deepseek-v4.1-flash\",\"system\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"thinking\",\"thinking\":\"The user wants me to call get_weather once for Paris, then reply exactly \\\"Paris is sunny.\\\"\",\"signature\":\"\"},{\"type\":\"tool_use\",\"id\":\"chatcmpl-tool-51e80fcbd5064b6e8bc9b8d0fa1d4e1d\",\"name\":\"get_weather\",\"input\":{\"city\":\"Paris\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"chatcmpl-tool-51e80fcbd5064b6e8bc9b8d0fa1d4e1d\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":2048,\"thinking\":{\"type\":\"adaptive\"},\"output_config\":{\"effort\":\"high\"}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"gen_01M44ZVHFBW1M2N3JSPQG4PXFG\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"deepseek/deepseek-v4.1-flash\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Paris is sunny.\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":406,\"output_tokens\":6},\"provider_metadata\":{\"baseten\":{\"acceptedPredictionTokens\":4,\"rejectedPredictionTokens\":2},\"gateway\":{\"routing\":{\"originalModelId\":\"deepseek/deepseek-v4.1-flash\",\"resolvedProvider\":\"baseten\",\"fallbacksAvailable\":[\"fireworks\",\"alibaba\",\"runware\",\"relace\",\"particle\",\"novita\",\"togetherai\",\"deepinfra\",\"wafer\",\"parasail\",\"gmicloud\",\"modal\",\"morph\",\"deepseek\",\"runinfra\",\"boundless\",\"inference-net\"],\"planningReasoning\":\"System credentials planned for: baseten, fireworks, alibaba, runware, relace, particle, novita, togetherai, deepinfra, wafer, parasail, gmicloud, modal, morph, deepseek, runinfra, boundless, inference-net. Total execution order: baseten(system) → fireworks(system) → alibaba(system) → runware(system) → relace(system) → particle(system) → novita(system) → togetherai(system) → deepinfra(system) → wafer(system) → parasail(system) → gmicloud(system) → modal(system) → morph(system) → deepseek(system) → runinfra(system) → boundless(system) → inference-net(system)\",\"canonicalSlug\":\"deepseek/deepseek-v4.1-flash\",\"finalProvider\":\"baseten\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"deepseek/deepseek-v4.1-flash\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"baseten\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1791168989072,\"endTime\":1791168989270,\"providerRequestId\":\"54ca701a109fdc7a977f6a444f59bbed7ee04\",\"statusCode\":200,\"providerResponseId\":\"chatcmpl-01cf067760f44cc1bca4b7a01722e964\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"no_pin\"},\"clientSessionId\":\"a47cd177650917b88c4d443d7bd8be68\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.000129\",\"marketCost\":\"0.000129\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.000129\",\"inferenceCost\":\"0.000129\",\"inputInferenceCost\":\"0.0001218\",\"outputInferenceCost\":\"0.0000072\",\"generationId\":\"gen_01M44ZVHFBW1M2N3JSPQG4PXFG\"}}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:vercel-native",
|
||||
"provider:vercel-ai-gateway"
|
||||
],
|
||||
"name": "vercel-native/continues-messages-gemini-reasoning-and-tools",
|
||||
"recordedAt": "2026-10-02T03:35:15.999Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/messages",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"google/gemini-3.8-flash\",\"system\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":2048,\"thinking\":{\"type\":\"adaptive\"},\"output_config\":{\"effort\":\"low\"}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"gen_01M3XAWAZ6J0VZ2XBQD2GHA031\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"google/gemini-3.8-flash\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"AY89a19bLXkNRJcCTLXJhlklpdU5hxo3vEit19RMmh8bC8Jm2SmGq6E5NZdq177AdJfOKRYxZkFuEaJ/ngpxoqLeB47ZMMK4VSZD2lhRbfFR6S8bITZ1\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"tool_use\",\"id\":\"call_355115\",\"name\":\"get_weather\",\"input\":{}}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":52,\"output_tokens\":16},\"provider_metadata\":{\"vertex\":{\"promptFeedback\":null,\"groundingMetadata\":null,\"urlContextMetadata\":null,\"safetyRatings\":null,\"usageMetadata\":{\"promptTokenCount\":52,\"candidatesTokenCount\":16,\"totalTokenCount\":68,\"trafficType\":\"ON_DEMAND\",\"promptTokensDetails\":[{\"modality\":\"TEXT\",\"tokenCount\":52}],\"candidatesTokensDetails\":[{\"modality\":\"TEXT\",\"tokenCount\":16}]},\"finishMessage\":null,\"serviceTier\":null},\"gateway\":{\"routing\":{\"originalModelId\":\"google/gemini-3.8-flash\",\"resolvedProvider\":\"vertex\",\"fallbacksAvailable\":[\"google\"],\"planningReasoning\":\"System credentials planned for: vertex, google. Total execution order: vertex(system) → google(system)\",\"canonicalSlug\":\"google/gemini-3.8-flash\",\"finalProvider\":\"vertex\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"google/gemini-3.8-flash\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"vertex\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1790912113658,\"endTime\":1790912114756,\"statusCode\":200,\"providerResponseId\":\"cSa_avLEL-zFiLsP5L_a4Ak\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"skipped_below_min_prefix\"},\"clientSessionId\":\"a47cd177650917b88c4d443d7bd8be68\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.000099\",\"marketCost\":\"0.000099\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.000099\",\"inferenceCost\":\"0.000099\",\"inputInferenceCost\":\"0.000039\",\"outputInferenceCost\":\"0.00006\",\"generationId\":\"gen_01M3XAWAZ6J0VZ2XBQD2GHA031\"}}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/messages",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"google/gemini-3.8-flash\",\"system\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"AY89a19bLXkNRJcCTLXJhlklpdU5hxo3vEit19RMmh8bC8Jm2SmGq6E5NZdq177AdJfOKRYxZkFuEaJ/ngpxoqLeB47ZMMK4VSZD2lhRbfFR6S8bITZ1\"},{\"type\":\"tool_use\",\"id\":\"call_355115\",\"name\":\"get_weather\",\"input\":{\"city\":\"Paris\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"call_355115\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":2048,\"thinking\":{\"type\":\"adaptive\"},\"output_config\":{\"effort\":\"low\"}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"gen_01M3XAWC57YKVKCYGSHBPW86GH\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"google/gemini-3.8-flash\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Paris is\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" sunny.\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":83,\"output_tokens\":4},\"provider_metadata\":{\"vertex\":{\"promptFeedback\":null,\"groundingMetadata\":null,\"urlContextMetadata\":null,\"safetyRatings\":null,\"usageMetadata\":{\"promptTokenCount\":83,\"candidatesTokenCount\":4,\"totalTokenCount\":87,\"trafficType\":\"ON_DEMAND\",\"promptTokensDetails\":[{\"modality\":\"TEXT\",\"tokenCount\":83}],\"candidatesTokensDetails\":[{\"modality\":\"TEXT\",\"tokenCount\":4}]},\"finishMessage\":null,\"serviceTier\":null},\"gateway\":{\"routing\":{\"originalModelId\":\"google/gemini-3.8-flash\",\"resolvedProvider\":\"vertex\",\"fallbacksAvailable\":[\"google\"],\"planningReasoning\":\"System credentials planned for: vertex, google. Total execution order: vertex(system) → google(system)\",\"canonicalSlug\":\"google/gemini-3.8-flash\",\"finalProvider\":\"vertex\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"google/gemini-3.8-flash\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"vertex\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1790912114880,\"endTime\":1790912116051,\"statusCode\":200,\"providerResponseId\":\"cia_auGLOt3Mw8cPopm3wAY\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"skipped_below_min_prefix\"},\"clientSessionId\":\"a47cd177650917b88c4d443d7bd8be68\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.00007725\",\"marketCost\":\"0.00007725\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.00007725\",\"inferenceCost\":\"0.00007725\",\"inputInferenceCost\":\"0.00006225\",\"outputInferenceCost\":\"0.000015\",\"generationId\":\"gen_01M3XAWC57YKVKCYGSHBPW86GH\"}}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+49
File diff suppressed because one or more lines are too long
packages/ai/test/fixtures/recordings/vercel-native/continues-responses-grok-reasoning-and-tools.json
Vendored
+49
File diff suppressed because one or more lines are too long
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:vercel-native",
|
||||
"provider:vercel-ai-gateway"
|
||||
],
|
||||
"name": "vercel-native/rejects-invalid-credentials-without-retryable-classification",
|
||||
"recordedAt": "2026-10-02T03:35:40.803Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://ai-gateway.vercel.sh/v1/messages",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-5.5\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"stream\":true,\"max_tokens\":32,\"thinking\":{\"type\":\"adaptive\",\"block_binding\":{\"prefix_mismatch_behavior\":\"drop_block\"}}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 401,
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"error\":{\"message\":\"Authentication failed. Create an API key and set in AI_GATEWAY_API_KEY environment variable: https://vercel.com/d?to=%2F%5Bteam%5D%2F%7E%2Fai-gateway%2Fapi-keys%3FshowCreateKeyModal\",\"type\":\"authentication_error\"}}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { LLM } from "../../src/index.js"
|
||||
import { VercelAIGateway } from "../../src/providers/vercel-ai-gateway.js"
|
||||
|
||||
const gateway = VercelAIGateway.configure({
|
||||
apiKey: "fixture",
|
||||
providerOptions: { gateway: { only: ["anthropic", "bedrock"], caching: "auto" }, reasoningEffort: "high" },
|
||||
})
|
||||
LLM.request({
|
||||
model: gateway.model("meta/muse-spark-1.3"),
|
||||
prompt: "Hello",
|
||||
providerOptions: { reasoningEffort: "max" },
|
||||
})
|
||||
LLM.request({
|
||||
model: gateway.messages("anthropic/claude-sonnet-5.5"),
|
||||
prompt: "Hello",
|
||||
providerOptions: { thinking: { type: "enabled", budgetTokens: 2048 } },
|
||||
})
|
||||
LLM.request({
|
||||
model: gateway.responses("openai/gpt-6-luna"),
|
||||
prompt: "Hello",
|
||||
providerOptions: { reasoningSummary: "detailed", cacheTTL: "1h" },
|
||||
})
|
||||
VercelAIGateway.configure({
|
||||
providerOptions: {
|
||||
gateway: { byok: { anthropic: [{ apiKey: "fixture" }] } },
|
||||
upstream: { google: { thinkingConfig: { includeThoughts: true, thinkingBudget: 1024 } } },
|
||||
},
|
||||
})
|
||||
// @ts-expect-error A thinking budget is numeric.
|
||||
VercelAIGateway.configure({ providerOptions: { thinking: { type: "enabled", budgetTokens: "many" } } })
|
||||
@@ -0,0 +1,127 @@
|
||||
import { expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMClient, LLMEvent, LLMRequest } from "../../src/index.js"
|
||||
import { isRetryable } from "../../src/provider-error.js"
|
||||
import { VercelAIGateway } from "../../src/providers/vercel-ai-gateway.js"
|
||||
import { recordedTests } from "../recorded-test.js"
|
||||
import { expectWeatherToolLoop, goldenWeatherToolLoopRequest, runWeatherToolLoop } from "../recorded-scenarios.js"
|
||||
|
||||
const gateway = VercelAIGateway.configure({ apiKey: process.env.AI_GATEWAY_API_KEY ?? "fixture" })
|
||||
const recorded = recordedTests({
|
||||
prefix: "vercel-native",
|
||||
provider: "vercel-ai-gateway",
|
||||
requires: ["AI_GATEWAY_API_KEY"],
|
||||
})
|
||||
|
||||
for (const item of [
|
||||
{
|
||||
name: "messages Claude",
|
||||
model: gateway.messages("anthropic/claude-sonnet-5.5"),
|
||||
options: { reasoningEffort: "low" },
|
||||
reasoning: false,
|
||||
},
|
||||
{
|
||||
name: "messages Gemini",
|
||||
model: gateway.messages("google/gemini-3.8-flash"),
|
||||
options: { reasoningEffort: "low" },
|
||||
reasoning: true,
|
||||
},
|
||||
{
|
||||
name: "responses GPT",
|
||||
model: gateway.model("openai/gpt-6-luna"),
|
||||
options: { reasoningEffort: "low" },
|
||||
reasoning: false,
|
||||
},
|
||||
{
|
||||
name: "chat Muse",
|
||||
model: gateway.model("meta/muse-spark-1.3"),
|
||||
options: { reasoningEffort: "low" },
|
||||
reasoning: false,
|
||||
},
|
||||
{
|
||||
name: "messages DeepSeek",
|
||||
model: gateway.model("deepseek/deepseek-v4.1-flash"),
|
||||
options: { reasoningEffort: "high" },
|
||||
reasoning: true,
|
||||
},
|
||||
{
|
||||
name: "responses Grok",
|
||||
model: gateway.model("spacexai/grok-4.7"),
|
||||
options: { reasoningEffort: "low" },
|
||||
reasoning: true,
|
||||
},
|
||||
{
|
||||
name: "chat Claude",
|
||||
model: gateway.chat("anthropic/claude-sonnet-5.5"),
|
||||
options: { reasoningEffort: "low" },
|
||||
reasoning: false,
|
||||
},
|
||||
] as const) {
|
||||
recorded.effect(
|
||||
`continues ${item.name} reasoning and tools`,
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const events = yield* runWeatherToolLoop(
|
||||
LLMRequest.update(
|
||||
goldenWeatherToolLoopRequest({
|
||||
id: item.name,
|
||||
model: item.model,
|
||||
maxTokens: 2048,
|
||||
temperature: false,
|
||||
}),
|
||||
{ providerOptions: item.options },
|
||||
),
|
||||
)
|
||||
expectWeatherToolLoop(events)
|
||||
expect(events.filter(LLMEvent.is.finish).every((event) => event.providerMetadata?.gateway !== undefined)).toBe(
|
||||
true,
|
||||
)
|
||||
if (item.reasoning) expect(events.some(LLMEvent.is.reasoningEnd)).toBe(true)
|
||||
}),
|
||||
120_000,
|
||||
)
|
||||
}
|
||||
|
||||
recorded.effect(
|
||||
"rejects invalid credentials without retryable classification",
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
model: VercelAIGateway.configure({ apiKey: "invalid-gateway-key" }).messages("anthropic/claude-sonnet-5.5"),
|
||||
prompt: "Hello",
|
||||
generation: { maxTokens: 32 },
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
expect(error.reason.http?.status).toBe(401)
|
||||
expect(isRetryable(error)).toBe(false)
|
||||
}),
|
||||
30_000,
|
||||
)
|
||||
|
||||
recorded.effect(
|
||||
"automatically caches distinct Claude system blocks",
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const request = LLM.request({
|
||||
model: gateway.messages("anthropic/claude-sonnet-5.5"),
|
||||
system: [
|
||||
{
|
||||
type: "text",
|
||||
text: Array.from(
|
||||
{ length: 250 },
|
||||
(_, i) => `Rule ${i}: Preserve provider reasoning state and keep stable prompt prefixes intact.`,
|
||||
).join("\n"),
|
||||
},
|
||||
{ type: "text", text: "Reply with OK only." },
|
||||
],
|
||||
prompt: "Confirm.",
|
||||
generation: { maxTokens: 32 },
|
||||
providerOptions: { gateway: { only: ["anthropic"] } },
|
||||
})
|
||||
yield* LLMClient.generate(request)
|
||||
const second = yield* LLMClient.generate(request)
|
||||
expect(second.usage?.cacheReadInputTokens).toBeGreaterThan(0)
|
||||
}),
|
||||
60_000,
|
||||
)
|
||||
@@ -0,0 +1,303 @@
|
||||
import { expect } from "bun:test"
|
||||
import { ConfigProvider, Effect } from "effect"
|
||||
import { LLM, LLMClient, LLMEvent, Message, CacheHint } from "../../src/index.js"
|
||||
import { VercelAIGateway } from "../../src/providers/vercel-ai-gateway.js"
|
||||
import { compileRequest } from "../../src/route/client.js"
|
||||
import { it } from "../lib/effect.js"
|
||||
import { dynamicResponse } from "../lib/http.js"
|
||||
import { sseEvents } from "../lib/sse.js"
|
||||
import { isRetryable } from "../../src/provider-error.js"
|
||||
|
||||
it.effect("Gateway defaults preserve full model IDs and select the requested families", () =>
|
||||
Effect.gen(function* () {
|
||||
const gateway = VercelAIGateway.configure({ apiKey: "fixture" })
|
||||
for (const [id, path] of [
|
||||
["openai/gpt-6.1", "/responses"],
|
||||
["meta/muse-spark-1.3", "/chat/completions"],
|
||||
["spacexai/grok-4.7", "/responses"],
|
||||
["meta/llama-4", "/messages"],
|
||||
["openai/gpt-oss-120b", "/responses"],
|
||||
["anthropic/claude-sonnet-5.5", "/messages"],
|
||||
["google/gemini-3.8-flash", "/messages"],
|
||||
["moonshotai/kimi-k3", "/messages"],
|
||||
["deepseek/deepseek-v4.1-flash", "/messages"],
|
||||
]) {
|
||||
const model = gateway.model(id)
|
||||
expect(model.id).toBe(id)
|
||||
expect(model.provider).toBe("vercel-ai-gateway")
|
||||
expect(model.route.endpoint.path).toBe(path)
|
||||
const compiled = yield* compileRequest(LLM.request({ model, prompt: "Hello" }))
|
||||
expect(compiled.body.model).toBe(id)
|
||||
expect(compiled.body.providerOptions).toBeUndefined()
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("Gateway Messages applies inline cache markers only for Anthropic and Qwen models", () =>
|
||||
Effect.gen(function* () {
|
||||
const gateway = VercelAIGateway.configure({ apiKey: "fixture" })
|
||||
const input = {
|
||||
system: "System prompt",
|
||||
tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object" } }],
|
||||
prompt: "Hello",
|
||||
}
|
||||
const claude = yield* compileRequest(
|
||||
LLM.request({ ...input, model: gateway.messages("anthropic/claude-sonnet-5.5") }),
|
||||
)
|
||||
expect(claude.body).toMatchObject({
|
||||
tools: [{ name: "lookup", cache_control: { type: "ephemeral" } }],
|
||||
system: [{ type: "text", text: "System prompt", cache_control: { type: "ephemeral" } }],
|
||||
messages: [{ role: "user", content: [{ type: "text", text: "Hello", cache_control: { type: "ephemeral" } }] }],
|
||||
})
|
||||
const qwen = yield* compileRequest(LLM.request({ ...input, model: gateway.messages("alibaba/qwen3.8-max") }))
|
||||
expect(qwen.body).toMatchObject({
|
||||
tools: [{ name: "lookup", cache_control: undefined }],
|
||||
system: [{ type: "text", text: "System prompt", cache_control: { type: "ephemeral" } }],
|
||||
messages: [{ role: "user", content: [{ type: "text", text: "Hello", cache_control: { type: "ephemeral" } }] }],
|
||||
})
|
||||
const gemini = yield* compileRequest(LLM.request({ ...input, model: gateway.messages("google/gemini-3.8-flash") }))
|
||||
expect(JSON.stringify(gemini.body)).not.toContain("cache_control")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("Gateway keeps distinct system blocks, cache hints, and upstream-scoped controls", () =>
|
||||
Effect.gen(function* () {
|
||||
const compiled = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: VercelAIGateway.configure({ apiKey: "fixture" }).messages("anthropic/claude-sonnet-5.5"),
|
||||
system: [
|
||||
{ type: "text", text: "Stable", cache: new CacheHint({ type: "ephemeral" }) },
|
||||
{ type: "text", text: "Project" },
|
||||
],
|
||||
prompt: "Hello",
|
||||
cache: "none",
|
||||
providerOptions: {
|
||||
gateway: { only: ["anthropic", "bedrock"], order: ["bedrock"], zeroDataRetention: true },
|
||||
upstream: {
|
||||
anthropic: { thinking: { type: "adaptive" } },
|
||||
bedrock: { reasoningConfig: { type: "adaptive" } },
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
expect(compiled.body.system).toEqual([
|
||||
{ type: "text", text: "Stable", cache_control: { type: "ephemeral" } },
|
||||
{ type: "text", text: "Project" },
|
||||
])
|
||||
expect(compiled.body.providerOptions).toEqual({
|
||||
gateway: { only: ["anthropic", "bedrock"], order: ["bedrock"], zeroDataRetention: true },
|
||||
anthropic: { thinking: { type: "adaptive" } },
|
||||
bedrock: { reasoningConfig: { type: "adaptive" } },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("Gateway effort settings lower to the selected API", () =>
|
||||
Effect.gen(function* () {
|
||||
const gateway = VercelAIGateway.configure({ apiKey: "fixture", providerOptions: { reasoningEffort: "high" } })
|
||||
const messages = yield* compileRequest(
|
||||
LLM.request({ model: gateway.messages("anthropic/claude-sonnet-5.5"), prompt: "Hello" }),
|
||||
)
|
||||
expect(messages.body).toMatchObject({ thinking: { type: "adaptive" }, output_config: { effort: "high" } })
|
||||
const responses = yield* compileRequest(
|
||||
LLM.request({ model: gateway.responses("openai/gpt-6-luna"), prompt: "Hello" }),
|
||||
)
|
||||
expect(responses.body).toMatchObject({
|
||||
reasoning: { effort: "high" },
|
||||
store: false,
|
||||
include: ["reasoning.encrypted_content"],
|
||||
})
|
||||
const chat = yield* compileRequest(LLM.request({ model: gateway.chat("google/gemini-3.8-flash"), prompt: "Hello" }))
|
||||
expect(chat.body).toMatchObject({ reasoning_effort: "high" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("Gateway Responses keeps cache-key and cache controls out of upstream namespaces", () =>
|
||||
Effect.gen(function* () {
|
||||
const compiled = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: VercelAIGateway.configure({ apiKey: "fixture" }).responses("openai/gpt-6-luna"),
|
||||
prompt: "Hello",
|
||||
promptCacheKey: "opaque-session",
|
||||
providerOptions: { cacheTTL: "1h", cacheAnchorItems: 1 },
|
||||
}),
|
||||
)
|
||||
expect(compiled.body).toMatchObject({ prompt_cache_key: "opaque-session", cache_ttl: "1h", cache_anchor_items: 1 })
|
||||
expect(compiled.body.providerOptions).toBeUndefined()
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("Gateway Messages preserves signatures and billing metadata through tool continuation", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = VercelAIGateway.configure({ apiKey: "fixture" }).messages("anthropic/claude-sonnet-5.5")
|
||||
const response = yield* LLMClient.generate(LLM.request({ model, prompt: "Hello" })).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.sync(() => {
|
||||
expect(input.request.url).toBe("https://ai-gateway.vercel.sh/v1/messages")
|
||||
expect(input.request.headers.authorization).toBe("Bearer fixture")
|
||||
return input.respond(
|
||||
sseEvents(
|
||||
{
|
||||
type: "message_start",
|
||||
message: { usage: { input_tokens: 10, cache_read_input_tokens: 20 } },
|
||||
},
|
||||
{
|
||||
type: "content_block_start",
|
||||
index: 0,
|
||||
content_block: { type: "thinking", thinking: "Plan", signature: "signed-state" },
|
||||
},
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{
|
||||
type: "content_block_start",
|
||||
index: 1,
|
||||
content_block: { type: "tool_use", id: "tool_1", name: "lookup", input: {} },
|
||||
},
|
||||
{ type: "content_block_stop", index: 1 },
|
||||
{
|
||||
type: "message_delta",
|
||||
delta: { stop_reason: "tool_use" },
|
||||
usage: { output_tokens: 5 },
|
||||
provider_metadata: { gateway: { cost: "0.001", routing: { finalProvider: "bedrock" } } },
|
||||
},
|
||||
{ type: "message_stop" },
|
||||
),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
expect(response.events.find(LLMEvent.is.finish)?.providerMetadata?.gateway).toMatchObject({
|
||||
cost: "0.001",
|
||||
routing: { finalProvider: "bedrock" },
|
||||
})
|
||||
expect(response.usage?.cacheReadInputTokens).toBe(20)
|
||||
const compiled = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.user("Hello"),
|
||||
response.message,
|
||||
Message.tool({ id: "tool_1", name: "lookup", result: { found: true } }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
expect(compiled.body.messages).toContainEqual({
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "thinking", thinking: "Plan", signature: "signed-state" },
|
||||
{ type: "tool_use", id: "tool_1", name: "lookup", input: {} },
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("Gateway Responses replays encrypted reasoning alongside tool calls", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = VercelAIGateway.configure({ apiKey: "fixture" }).responses("openai/gpt-6-luna")
|
||||
const response = yield* LLMClient.generate(LLM.request({ model, prompt: "Hello" })).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.sync(() =>
|
||||
input.respond(
|
||||
sseEvents(
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
output_index: 0,
|
||||
item: { type: "reasoning", id: "rs_1", encrypted_content: "opaque-state", summary: [] },
|
||||
},
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
output_index: 1,
|
||||
item: { type: "function_call", id: "fc_1", call_id: "call_1", name: "lookup", arguments: "{}" },
|
||||
},
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: "resp_1",
|
||||
output: [],
|
||||
usage: { input_tokens: 10, output_tokens: 5 },
|
||||
provider_metadata: { gateway: { cost: "0.001" } },
|
||||
},
|
||||
},
|
||||
),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
const compiled = yield* compileRequest(LLM.request({ model, messages: [Message.user("Hello"), response.message] }))
|
||||
expect(compiled.body.input).toContainEqual({
|
||||
type: "reasoning",
|
||||
id: "rs_1",
|
||||
encrypted_content: "opaque-state",
|
||||
summary: [],
|
||||
})
|
||||
expect(compiled.body.input).toContainEqual({
|
||||
type: "function_call",
|
||||
id: "fc_1",
|
||||
call_id: "call_1",
|
||||
name: "lookup",
|
||||
arguments: "{}",
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("Gateway 401 and 403 failures are terminal and retain HTTP context", () =>
|
||||
Effect.gen(function* () {
|
||||
for (const status of [401, 403]) {
|
||||
const error = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
model: VercelAIGateway.configure({ apiKey: "fixture" }).chat("openai/gpt-6-luna"),
|
||||
prompt: "Hello",
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.succeed(
|
||||
input.respond(
|
||||
JSON.stringify({ error: { type: "authentication_error", message: "Invalid credentials" } }),
|
||||
{ status, headers: { "content-type": "application/json" } },
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
Effect.flip,
|
||||
)
|
||||
expect(error.reason.http?.status).toBe(status)
|
||||
expect(isRetryable(error)).toBe(false)
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("Gateway authentication supports OIDC fallback and affinity without reading session state", () =>
|
||||
Effect.gen(function* () {
|
||||
for (const api of ["messages", "responses", "chat"] as const) {
|
||||
yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
model: VercelAIGateway.configure()[api]("openai/gpt-6-luna"),
|
||||
prompt: "Hello",
|
||||
promptCacheKey: "opaque-session",
|
||||
}),
|
||||
).pipe(
|
||||
Effect.result,
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.sync(() => {
|
||||
expect(input.request.headers.authorization).toBe("Bearer oidc-fixture")
|
||||
expect(input.request.headers["x-session-affinity"]).toBe("opaque-session")
|
||||
return input.respond("{}", { status: 400, headers: { "content-type": "application/json" } })
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
}
|
||||
}).pipe(
|
||||
Effect.provideService(
|
||||
ConfigProvider.ConfigProvider,
|
||||
ConfigProvider.fromUnknown({ VERCEL_OIDC_TOKEN: "oidc-fixture" }),
|
||||
),
|
||||
),
|
||||
)
|
||||
Reference in New Issue
Block a user