From 79fd806343db0de150f23b09e4b9199434d6779c Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Tue, 20 Oct 2026 23:13:14 -0700 Subject: [PATCH] feat(server): list experimental voice models (#57194) --- packages/ai/src/experimental.ts | 1 + packages/ai/src/experimental/realtime.ts | 36 ++++-- packages/ai/test/realtime.test.ts | 3 + packages/client/src/effect/api/api.ts | 12 ++ .../client/src/effect/generated/client.ts | 14 ++- .../client/src/promise/generated/client.ts | 12 ++ .../client/src/promise/generated/types.ts | 8 ++ packages/client/test/promise.test.ts | 2 +- packages/core/src/voice.ts | 42 ++++++- packages/core/test/voice.test.ts | 113 ++++++++++++----- packages/protocol/openapi.json | 118 ++++++++++++++++++ packages/protocol/src/groups/voice.ts | 29 +++++ packages/server/src/handlers/voice.ts | 55 ++++---- packages/server/test/voice.test.ts | 28 ++++- services/www/openapi.json | 118 ++++++++++++++++++ services/www/public/openapi.json | 118 ++++++++++++++++++ 16 files changed, 634 insertions(+), 75 deletions(-) diff --git a/packages/ai/src/experimental.ts b/packages/ai/src/experimental.ts index 4128d367e0d..6e67322a546 100644 --- a/packages/ai/src/experimental.ts +++ b/packages/ai/src/experimental.ts @@ -33,4 +33,5 @@ export type { RealtimeSession, RealtimeTransport, RealtimeTransportInput, + RealtimeTransportType, } from "./experimental/realtime.js" diff --git a/packages/ai/src/experimental/realtime.ts b/packages/ai/src/experimental/realtime.ts index a7bd681984e..c5f5026a005 100644 --- a/packages/ai/src/experimental/realtime.ts +++ b/packages/ai/src/experimental/realtime.ts @@ -12,6 +12,8 @@ export type RealtimeTransportInput = | { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number } | { readonly type: "websocket" } +export type RealtimeTransportType = RealtimeTransportInput["type"] + export type RealtimeTransport = | { readonly type: "webrtc"; readonly sdp: string } | { readonly type: "websocket"; readonly url: string; readonly token: string; readonly expiresAt: number } @@ -25,11 +27,23 @@ export interface RealtimeConnectInput { readonly transport: RealtimeTransportInput } -const engines = [ - { family: "gpt-live", id: "openai-live", provider: "openai" }, - { family: "gpt-realtime", id: "openai-realtime", provider: "openai" }, - { family: "grok-voice", id: "xai-realtime", provider: "xai" }, - { family: "gemini-3.8-live", id: "gemini-live", provider: "google" }, +export const engines = [ + { family: "gpt-live", model: "gpt-live-1", id: "openai-live", provider: "openai", transports: ["webrtc"] }, + { + family: "gpt-realtime", + model: "gpt-realtime-2.1", + id: "openai-realtime", + provider: "openai", + transports: ["webrtc", "websocket"], + }, + { family: "grok-voice", model: "grok-voice-3.0", id: "xai-realtime", provider: "xai", transports: ["websocket"] }, + { + family: "gemini-3.8-live", + model: "gemini-3.8-live", + id: "gemini-live", + provider: "google", + transports: ["websocket"], + }, ] as const // Matches the bare family name (`gpt-realtime`) and its versions (`gpt-realtime-2.1`, `gpt-realtime-mini`). @@ -39,12 +53,12 @@ export const engine = (model: string) => export const connect = Effect.fn("Realtime.connect")(function* (input: RealtimeConnectInput) { const resolved = engine(input.model) if (!resolved) return yield* invalidRequest(`Unsupported voice model: ${input.model}`) + if (!resolved.transports.some((type) => type === input.transport.type)) + return yield* invalidRequest(`${input.model} only supports ${resolved.transports.join(", ")} transport`) const session = { ...input.session, model: input.model } switch (resolved.id) { case "openai-live": { - if (input.transport.type !== "webrtc") - return yield* invalidRequest("OpenAI GPT-Live only supports WebRTC transport") const data = yield* post( resolved.id, `${OPENAI_BASE_URL}/live/sessions`, @@ -73,16 +87,11 @@ export const connect = Effect.fn("Realtime.connect")(function* (input: RealtimeC ) return { type: "webrtc" as const, sdp } } - case "xai-realtime": { - if (input.transport.type !== "websocket") - return yield* invalidRequest("xAI Grok Voice only supports WebSocket transport") + case "xai-realtime": return yield* clientSecret(resolved.id, XAI_BASE_URL, input.model, input.apiKey, { expires_after: { seconds: 60 }, }) - } case "gemini-live": { - if (input.transport.type !== "websocket") - return yield* invalidRequest("Google Gemini Live only supports WebSocket transport") const now = yield* Clock.currentTimeMillis const expiresAt = now + 60_000 const data = yield* post( @@ -109,6 +118,7 @@ export const connect = Effect.fn("Realtime.connect")(function* (input: RealtimeC }) export const Realtime = { + engines, engine, connect, } as const diff --git a/packages/ai/test/realtime.test.ts b/packages/ai/test/realtime.test.ts index a65df8078dd..ac39877f260 100644 --- a/packages/ai/test/realtime.test.ts +++ b/packages/ai/test/realtime.test.ts @@ -138,6 +138,9 @@ describe("experimental Realtime", () => { ) test("routes bare and versioned model names to their engine", () => { + expect(Realtime.engines.map((item) => Realtime.engine(item.model)?.id)).toEqual( + Realtime.engines.map((item) => item.id), + ) expect( [ "gpt-realtime", diff --git a/packages/client/src/effect/api/api.ts b/packages/client/src/effect/api/api.ts index 83c4b0cb4c9..a525ed3652c 100644 --- a/packages/client/src/effect/api/api.ts +++ b/packages/client/src/effect/api/api.ts @@ -1418,6 +1418,17 @@ export interface GenerateApi { readonly text: GenerateTextOperation } +export type VoiceModelsOutput = { + readonly models: ReadonlyArray<{ + readonly model: string + readonly provider: string + readonly transports: ReadonlyArray<"webrtc" | "websocket"> + readonly available: boolean + }> + readonly default?: string | undefined +} +export type VoiceModelsOperation = () => Effect.Effect + export type VoiceSessionInput = { readonly model: string readonly session?: { readonly [x: string]: unknown } | undefined @@ -1433,6 +1444,7 @@ export type VoiceSessionOutput = { export type VoiceSessionOperation = (input: VoiceSessionInput) => Effect.Effect export interface VoiceApi { + readonly models: VoiceModelsOperation readonly session: VoiceSessionOperation } diff --git a/packages/client/src/effect/generated/client.ts b/packages/client/src/effect/generated/client.ts index bcff357c9da..b8b5032951e 100644 --- a/packages/client/src/effect/generated/client.ts +++ b/packages/client/src/effect/generated/client.ts @@ -115,6 +115,7 @@ import type { ModelDefaultOutput, GenerateTextInput, GenerateTextOutput, + VoiceModelsOutput, VoiceSessionInput, VoiceSessionOutput, ProviderListInput, @@ -873,6 +874,14 @@ const EndpointGenerateText = (raw: RawClient["server.generate"]) => (input: Gene const adaptGroupGenerate = (raw: RawClient["server.generate"]) => ({ text: EndpointGenerateText(raw) }) +const EndpointVoiceModels = (raw: RawClient["server.voice"]) => () => + preserveEffect()( + raw["voice.models"]({}).pipe( + Effect.mapError(mapClientError), + Effect.map((value) => value.data), + ), + ) + const EndpointVoiceSession = (raw: RawClient["server.voice"]) => (input: VoiceSessionInput) => preserveEffect()( raw["voice.session"]({ @@ -883,7 +892,10 @@ const EndpointVoiceSession = (raw: RawClient["server.voice"]) => (input: VoiceSe ), ) -const adaptGroupVoice = (raw: RawClient["server.voice"]) => ({ session: EndpointVoiceSession(raw) }) +const adaptGroupVoice = (raw: RawClient["server.voice"]) => ({ + models: EndpointVoiceModels(raw), + session: EndpointVoiceSession(raw), +}) const EndpointProviderList = (raw: RawClient["server.provider"]) => (input?: ProviderListInput) => preserveEffect()( diff --git a/packages/client/src/promise/generated/client.ts b/packages/client/src/promise/generated/client.ts index ca3c2ad885c..5a01a05d9e5 100644 --- a/packages/client/src/promise/generated/client.ts +++ b/packages/client/src/promise/generated/client.ts @@ -109,6 +109,7 @@ import type { ModelDefaultOutput, GenerateTextInput, GenerateTextOutput, + VoiceModelsOutput, VoiceSessionInput, VoiceSessionOutput, ProviderListInput, @@ -1162,6 +1163,17 @@ export function make(options: ClientOptions) { ).then((value) => value.data), }, voice: { + models: (requestOptions?: RequestOptions) => + request<{ readonly data: VoiceModelsOutput }>( + { + method: "GET", + path: `/api/experimental/voice/models`, + successStatus: 200, + declaredStatuses: [400, 401, 404, 503], + empty: false, + }, + requestOptions, + ).then((value) => value.data), session: (input: VoiceSessionInput, requestOptions?: RequestOptions) => request<{ readonly data: VoiceSessionOutput }>( { diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index aff828dc4f6..0d3f680c490 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -180,6 +180,8 @@ export type MoneyUSDPerMillionTokens = number export type GenerateTextResponse = { data: { text: string } } +export type VoiceTransportType = "webrtc" | "websocket" + export type VoiceWebRTCTransport = { type: "webrtc"; sdp: string } export type VoiceWebSocketTransport = { type: "websocket"; url: string; token: string; expiresAt: number } @@ -662,6 +664,8 @@ export type ModelCost = { cache: { read: MoneyUSDPerMillionTokens; write: MoneyUSDPerMillionTokens } } +export type VoiceModel = { model: string; provider: string; transports: Array; available: boolean } + export type VoiceTransport = VoiceWebRTCTransport | VoiceWebSocketTransport export type IntegrationCommandMethod = { @@ -1985,6 +1989,8 @@ export type ProviderInfo = { body?: { [x: string]: any } } +export type VoiceModelsResponse = { data: { models: Array; default?: string | undefined } } + export type VoiceSessionResponse = { data: { transport: VoiceTransport } } export type CredentialValue = CredentialOAuth | CredentialKey | CredentialExternal @@ -4966,6 +4972,8 @@ export type GenerateTextInput = { export type GenerateTextOutput = GenerateTextResponse["data"] +export type VoiceModelsOutput = VoiceModelsResponse["data"] + export type VoiceSessionInput = { readonly model: { readonly model: string diff --git a/packages/client/test/promise.test.ts b/packages/client/test/promise.test.ts index d2f8b4c28d5..ef935fc9baf 100644 --- a/packages/client/test/promise.test.ts +++ b/packages/client/test/promise.test.ts @@ -46,7 +46,7 @@ test("exposes every standard HTTP API group", () => { expect(Object.keys(client.integration.oauth)).toEqual(["connect", "status", "complete", "cancel"]) expect(Object.keys(client.integration.command)).toEqual(["connect", "status", "cancel"]) expect(Object.keys(client.websearch)).toEqual(["providers", "query"]) - expect(Object.keys(client.voice)).toEqual(["session"]) + expect(Object.keys(client.voice)).toEqual(["models", "session"]) expect(Object.keys(client.file)).toEqual(["read", "list", "find", "write"]) expect(Object.keys(client.vcs)).toEqual(["init", "get", "base", "status", "branch", "diff"]) expect(Object.keys(client.pty)).toEqual(["list", "create", "get", "update", "remove", "connect"]) diff --git a/packages/core/src/voice.ts b/packages/core/src/voice.ts index fdfd664bf65..0e3849a9242 100644 --- a/packages/core/src/voice.ts +++ b/packages/core/src/voice.ts @@ -6,6 +6,7 @@ import { type RealtimeSession, type RealtimeTransport, type RealtimeTransportInput, + type RealtimeTransportType, } from "@opencode/ai/experimental" import { RequestExecutor } from "@opencode/ai/route" import { makeLocationNode } from "@opencode/util/effect/app-node" @@ -13,6 +14,18 @@ import { Context, Effect, Layer, Redacted, Schema } from "effect" import { requestExecutor } from "./effect/app-node-platform.js" import { Integration } from "./integration.js" +export interface ModelInfo { + readonly model: string + readonly provider: string + readonly transports: readonly RealtimeTransportType[] + readonly available: boolean +} + +export interface Models { + readonly models: readonly ModelInfo[] + readonly default?: string +} + export interface SessionInput { readonly model: string readonly session?: RealtimeSession @@ -31,6 +44,7 @@ export class UnavailableError extends Schema.TaggedError()("Vo export type Error = InvalidRequestError | UnavailableError export interface Interface { + readonly models: () => Effect.Effect readonly session: (input: SessionInput) => Effect.Effect } @@ -42,14 +56,32 @@ export const layer = Layer.effect( const executor = yield* RequestExecutor.Service const integrations = yield* Integration.Service + // Integration lists the active credential first, then newer stored credentials, then non-empty env vars. + const findConnection = (provider: string) => + integrations + .get(Integration.ID.make(provider)) + .pipe(Effect.map((info) => info?.connections.find((item) => item.type === "env" || item.method === "key"))) + + const models = Effect.fn("Voice.models")(function* () { + const list = yield* Effect.forEach(Realtime.engines, (engine) => + findConnection(engine.provider).pipe( + Effect.map((connection) => ({ + model: engine.model, + provider: engine.provider, + transports: engine.transports, + available: Boolean(connection), + })), + ), + ) + const first = list.find((item) => item.available)?.model + return first ? { models: list, default: first } : { models: list } + }) + const session = Effect.fn("Voice.session")(function* (input: SessionInput) { const engine = Realtime.engine(input.model) if (!engine) return yield* new InvalidRequestError({ message: `Unsupported voice model: ${input.model}` }) - // Integration lists the active credential first, then newer stored credentials, then its env vars. - const connection = (yield* integrations.get(Integration.ID.make(engine.provider)))?.connections.find( - (item) => item.type === "env" || item.method === "key", - ) + const connection = yield* findConnection(engine.provider) const credential = connection ? yield* integrations.connection .resolve(connection) @@ -74,7 +106,7 @@ export const layer = Layer.effect( ) }) - return Service.of({ session }) + return Service.of({ models, session }) }), ) diff --git a/packages/core/test/voice.test.ts b/packages/core/test/voice.test.ts index 14970f94bbb..7663e958a47 100644 --- a/packages/core/test/voice.test.ts +++ b/packages/core/test/voice.test.ts @@ -24,6 +24,29 @@ const keys: Record = { GEMINI_API_KEY: "gemini-env-key", } +const unused = () => Effect.die("unused") + +const mockIntegration = ( + connections: (id: Integration.ID) => Integration.Info["connections"], + resolve: Integration.Interface["connection"]["resolve"] = () => Effect.die("Voice.models resolved a credential"), +) => + Layer.mock(Integration.Service, { + revision: () => 0, + get: (id) => Effect.succeed({ id, name: id, methods: [], connections: connections(id) }), + connection: { + active: unused, + resolve, + key: unused, + external: unused, + activate: unused, + update: unused, + remove: unused, + status: unused, + }, + oauth: { connect: unused, status: unused, complete: unused, cancel: unused }, + command: { connect: unused, status: unused, cancel: unused }, + }) + it.effect("uses the first key or env connection Integration lists and never a ChatGPT sign-in", () => Effect.gen(function* () { yield* TestClock.setTime(1760000000_000) @@ -42,39 +65,18 @@ it.effect("uses the first key or env connection Integration lists and never a Ch Voice.layer.pipe( Layer.provide( Layer.merge( - Layer.mock(Integration.Service, { - revision: () => 0, - get: (id) => Effect.succeed({ id, name: id, methods: [], connections }), - connection: { - active: () => Effect.die("unused"), - resolve: (connection) => - connection.type === "credential" && connection.method === "oauth" - ? Effect.die("a ChatGPT sign-in was resolved") - : Effect.succeed( - Credential.Key.make({ - type: "key", - key: keys[connection.type === "env" ? connection.name : connection.id], - }), - ), - key: () => Effect.die("unused"), - external: () => Effect.die("unused"), - activate: () => Effect.die("unused"), - update: () => Effect.die("unused"), - remove: () => Effect.die("unused"), - status: () => Effect.die("unused"), - }, - oauth: { - connect: () => Effect.die("unused"), - status: () => Effect.die("unused"), - complete: () => Effect.die("unused"), - cancel: () => Effect.die("unused"), - }, - command: { - connect: () => Effect.die("unused"), - status: () => Effect.die("unused"), - cancel: () => Effect.die("unused"), - }, - }), + mockIntegration( + () => connections, + (connection) => + connection.type === "credential" && connection.method === "oauth" + ? Effect.die("a ChatGPT sign-in was resolved") + : Effect.succeed( + Credential.Key.make({ + type: "key", + key: keys[connection.type === "env" ? connection.name : connection.id], + }), + ), + ), RequestExecutor.layer.pipe( Layer.provide( Layer.succeed( @@ -128,3 +130,48 @@ it.effect("uses the first key or env connection Integration lists and never a Ch expect(missingGoogle.message).toContain("GEMINI_API_KEY") }), ) + +it.effect("lists voice models with availability and default without resolving credentials", () => + Effect.gen(function* () { + const list = (connections: Partial>) => + Effect.flatMap(Voice.Service, (voice) => voice.models()).pipe( + Effect.provide( + Voice.layer.pipe( + Layer.provide( + Layer.merge( + mockIntegration((id) => connections[id] ?? []), + Layer.mock(RequestExecutor.Service, {}), + ), + ), + ), + ), + ) + + expect( + yield* list({ + openai: [credential("chatgpt", "oauth")], + xai: [{ type: "env", name: "XAI_API_KEY" }], + google: [credential("gemini", "key")], + }), + ).toEqual({ + models: [ + { model: "gpt-live-1", provider: "openai", transports: ["webrtc"], available: false }, + { model: "gpt-realtime-2.1", provider: "openai", transports: ["webrtc", "websocket"], available: false }, + { model: "grok-voice-3.0", provider: "xai", transports: ["websocket"], available: true }, + { model: "gemini-3.8-live", provider: "google", transports: ["websocket"], available: true }, + ], + default: "grok-voice-3.0", + }) + + const none = yield* list({ openai: [credential("chatgpt", "oauth")] }) + expect(none).toEqual({ + models: [ + { model: "gpt-live-1", provider: "openai", transports: ["webrtc"], available: false }, + { model: "gpt-realtime-2.1", provider: "openai", transports: ["webrtc", "websocket"], available: false }, + { model: "grok-voice-3.0", provider: "xai", transports: ["websocket"], available: false }, + { model: "gemini-3.8-live", provider: "google", transports: ["websocket"], available: false }, + ], + }) + expect("default" in none).toBe(false) + }), +) diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index 89a23a1d439..ef200d872ad 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -5382,6 +5382,68 @@ } } }, + "/api/experimental/voice/models": { + "get": { + "tags": ["voice"], + "operationId": "experimental.voice.models", + "parameters": [], + "security": [], + "responses": { + "200": { + "description": "VoiceModelsResponse", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VoiceModelsResponse" + } + } + } + }, + "400": { + "description": "InvalidRequestError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InvalidRequestErrorEncoded" + } + } + } + }, + "401": { + "description": "UnauthorizedError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UnauthorizedErrorEncoded" + } + } + } + }, + "404": { + "description": "LocationNotFoundError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LocationNotFoundErrorEncoded" + } + } + } + }, + "503": { + "description": "ServiceUnavailableError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServiceUnavailableErrorEncoded" + } + } + } + } + }, + "description": "List supported voice models, their transports, and whether the server has a usable API key for each provider.", + "summary": "List voice models" + } + }, "/api/experimental/voice/session": { "post": { "tags": ["voice"], @@ -20536,6 +20598,58 @@ "required": ["_tag", "providerID", "message"], "additionalProperties": false }, + "VoiceModel": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "transports": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VoiceTransportType" + } + }, + "available": { + "type": "boolean" + } + }, + "required": ["model", "provider", "transports", "available"], + "additionalProperties": false + }, + "VoiceModelsResponse": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "models": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VoiceModel" + } + }, + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": ["models"], + "additionalProperties": false + } + }, + "required": ["data"], + "additionalProperties": false + }, "VoiceSessionResponse": { "type": "object", "properties": { @@ -20573,6 +20687,10 @@ } ] }, + "VoiceTransportType": { + "type": "string", + "enum": ["webrtc", "websocket"] + }, "VoiceWebRTCTransport": { "type": "object", "properties": { diff --git a/packages/protocol/src/groups/voice.ts b/packages/protocol/src/groups/voice.ts index 30e150bcd96..83d52fed305 100644 --- a/packages/protocol/src/groups/voice.ts +++ b/packages/protocol/src/groups/voice.ts @@ -2,6 +2,17 @@ import { Schema } from "effect" import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/http-api" import { InvalidRequestError, LocationNotFoundError, ServiceUnavailableError } from "../errors.js" +export const VoiceTransportType = Schema.Literals(["webrtc", "websocket"]).annotate({ + identifier: "VoiceTransportType", +}) + +export const VoiceModel = Schema.Struct({ + model: Schema.String, + provider: Schema.String, + transports: Schema.Array(VoiceTransportType), + available: Schema.Boolean, +}).annotate({ identifier: "VoiceModel" }) + export const VoiceWebRTCTransportInput = Schema.Struct({ type: Schema.Literal("webrtc"), sdp: Schema.String, @@ -35,6 +46,24 @@ export const VoiceTransport = Schema.Union([ ]).annotate({ identifier: "VoiceTransport" }) export const VoiceGroup = HttpApiGroup.make("server.voice") + .add( + HttpApiEndpoint.get("voice.models", "/api/experimental/voice/models", { + success: Schema.Struct({ + data: Schema.Struct({ + models: Schema.Array(VoiceModel), + default: Schema.optional(Schema.String), + }), + }).annotate({ identifier: "VoiceModelsResponse" }), + error: [ServiceUnavailableError, LocationNotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "experimental.voice.models", + summary: "List voice models", + description: + "List supported voice models, their transports, and whether the server has a usable API key for each provider.", + }), + ), + ) .add( HttpApiEndpoint.post("voice.session", "/api/experimental/voice/session", { payload: Schema.Struct({ diff --git a/packages/server/src/handlers/voice.ts b/packages/server/src/handlers/voice.ts index afdcb21113d..f9507a98c33 100644 --- a/packages/server/src/handlers/voice.ts +++ b/packages/server/src/handlers/voice.ts @@ -14,26 +14,39 @@ export const VoiceHandler = HttpApiBuilder.group(Api, "server.voice", (handlers) const global = yield* Global.Service const locations = yield* LocationServiceMap.Service const services = locations.get(Location.Ref.make({ directory: AbsolutePath.make(global.config) })) - return handlers.handle( - "voice.session", - Effect.fn("server.voice.session")( - function* (request) { - yield* awaitPlugins - const voice = yield* Voice.Service - const transport = yield* voice - .session(request.payload) - .pipe( - Effect.mapError((error) => - error._tag === "Voice.InvalidRequestError" - ? new InvalidRequestError({ message: error.message }) - : new ServiceUnavailableError({ message: error.message, service: error.service }), - ), - ) - return { data: { transport } } - }, - Effect.provide(services), - locationErrors, - ), - ) + return handlers + .handle( + "voice.models", + Effect.fn("server.voice.models")( + function* () { + yield* awaitPlugins + const voice = yield* Voice.Service + return { data: yield* voice.models() } + }, + Effect.provide(services), + locationErrors, + ), + ) + .handle( + "voice.session", + Effect.fn("server.voice.session")( + function* (request) { + yield* awaitPlugins + const voice = yield* Voice.Service + const transport = yield* voice + .session(request.payload) + .pipe( + Effect.mapError((error) => + error._tag === "Voice.InvalidRequestError" + ? new InvalidRequestError({ message: error.message }) + : new ServiceUnavailableError({ message: error.message, service: error.service }), + ), + ) + return { data: { transport } } + }, + Effect.provide(services), + locationErrors, + ), + ) }), ) diff --git a/packages/server/test/voice.test.ts b/packages/server/test/voice.test.ts index eda31544a10..38f50d2d404 100644 --- a/packages/server/test/voice.test.ts +++ b/packages/server/test/voice.test.ts @@ -13,6 +13,16 @@ const voice = makeLocationNode({ layer: Layer.succeed( Voice.Service, Voice.Service.of({ + models: () => + Effect.succeed({ + models: [ + { model: "gpt-live-1", provider: "openai", transports: ["webrtc"], available: true }, + { model: "gpt-realtime-2.1", provider: "openai", transports: ["webrtc", "websocket"], available: true }, + { model: "grok-voice-3.0", provider: "xai", transports: ["websocket"], available: false }, + { model: "gemini-3.8-live", provider: "google", transports: ["websocket"], available: false }, + ], + default: "gpt-live-1", + }), session: (input) => { if (input.model === "invalid") return Effect.fail(new Voice.InvalidRequestError({ message: "Unsupported voice model" })) @@ -34,7 +44,7 @@ const voice = makeLocationNode({ deps: [], }) -it.live("routes POST /api/experimental/voice/session and maps errors", () => +it.live("routes experimental voice models and session endpoints", () => Effect.gen(function* () { const tmp = yield* Effect.acquireDisposable(Effect.promise(() => tmpdir("opencode-voice-endpoint-"))) const global = path.join(tmp.path, "global") @@ -48,6 +58,22 @@ it.live("routes POST /api/experimental/voice/session and maps errors", () => }, { overrides: [Voice.node.replace(voice)] }, ) + const modelsRes = yield* Effect.promise(() => + handler(new Request("http://opencode.local/api/experimental/voice/models")), + ) + expect(modelsRes.status).toBe(200) + expect(yield* Effect.promise(() => modelsRes.json())).toEqual({ + data: { + models: [ + { model: "gpt-live-1", provider: "openai", transports: ["webrtc"], available: true }, + { model: "gpt-realtime-2.1", provider: "openai", transports: ["webrtc", "websocket"], available: true }, + { model: "grok-voice-3.0", provider: "xai", transports: ["websocket"], available: false }, + { model: "gemini-3.8-live", provider: "google", transports: ["websocket"], available: false }, + ], + default: "gpt-live-1", + }, + }) + const post = (body: unknown) => Effect.promise(() => handler( diff --git a/services/www/openapi.json b/services/www/openapi.json index 89a23a1d439..ef200d872ad 100644 --- a/services/www/openapi.json +++ b/services/www/openapi.json @@ -5382,6 +5382,68 @@ } } }, + "/api/experimental/voice/models": { + "get": { + "tags": ["voice"], + "operationId": "experimental.voice.models", + "parameters": [], + "security": [], + "responses": { + "200": { + "description": "VoiceModelsResponse", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VoiceModelsResponse" + } + } + } + }, + "400": { + "description": "InvalidRequestError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InvalidRequestErrorEncoded" + } + } + } + }, + "401": { + "description": "UnauthorizedError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UnauthorizedErrorEncoded" + } + } + } + }, + "404": { + "description": "LocationNotFoundError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LocationNotFoundErrorEncoded" + } + } + } + }, + "503": { + "description": "ServiceUnavailableError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServiceUnavailableErrorEncoded" + } + } + } + } + }, + "description": "List supported voice models, their transports, and whether the server has a usable API key for each provider.", + "summary": "List voice models" + } + }, "/api/experimental/voice/session": { "post": { "tags": ["voice"], @@ -20536,6 +20598,58 @@ "required": ["_tag", "providerID", "message"], "additionalProperties": false }, + "VoiceModel": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "transports": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VoiceTransportType" + } + }, + "available": { + "type": "boolean" + } + }, + "required": ["model", "provider", "transports", "available"], + "additionalProperties": false + }, + "VoiceModelsResponse": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "models": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VoiceModel" + } + }, + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": ["models"], + "additionalProperties": false + } + }, + "required": ["data"], + "additionalProperties": false + }, "VoiceSessionResponse": { "type": "object", "properties": { @@ -20573,6 +20687,10 @@ } ] }, + "VoiceTransportType": { + "type": "string", + "enum": ["webrtc", "websocket"] + }, "VoiceWebRTCTransport": { "type": "object", "properties": { diff --git a/services/www/public/openapi.json b/services/www/public/openapi.json index 89a23a1d439..ef200d872ad 100644 --- a/services/www/public/openapi.json +++ b/services/www/public/openapi.json @@ -5382,6 +5382,68 @@ } } }, + "/api/experimental/voice/models": { + "get": { + "tags": ["voice"], + "operationId": "experimental.voice.models", + "parameters": [], + "security": [], + "responses": { + "200": { + "description": "VoiceModelsResponse", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VoiceModelsResponse" + } + } + } + }, + "400": { + "description": "InvalidRequestError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InvalidRequestErrorEncoded" + } + } + } + }, + "401": { + "description": "UnauthorizedError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UnauthorizedErrorEncoded" + } + } + } + }, + "404": { + "description": "LocationNotFoundError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LocationNotFoundErrorEncoded" + } + } + } + }, + "503": { + "description": "ServiceUnavailableError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServiceUnavailableErrorEncoded" + } + } + } + } + }, + "description": "List supported voice models, their transports, and whether the server has a usable API key for each provider.", + "summary": "List voice models" + } + }, "/api/experimental/voice/session": { "post": { "tags": ["voice"], @@ -20536,6 +20598,58 @@ "required": ["_tag", "providerID", "message"], "additionalProperties": false }, + "VoiceModel": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "transports": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VoiceTransportType" + } + }, + "available": { + "type": "boolean" + } + }, + "required": ["model", "provider", "transports", "available"], + "additionalProperties": false + }, + "VoiceModelsResponse": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "models": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VoiceModel" + } + }, + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": ["models"], + "additionalProperties": false + } + }, + "required": ["data"], + "additionalProperties": false + }, "VoiceSessionResponse": { "type": "object", "properties": { @@ -20573,6 +20687,10 @@ } ] }, + "VoiceTransportType": { + "type": "string", + "enum": ["webrtc", "websocket"] + }, "VoiceWebRTCTransport": { "type": "object", "properties": {