diff --git a/packages/ai/src/experimental.ts b/packages/ai/src/experimental.ts index 50b54a41113..4128d367e0d 100644 --- a/packages/ai/src/experimental.ts +++ b/packages/ai/src/experimental.ts @@ -27,3 +27,10 @@ export type { EvaluationResponseFor, EvaluationRoute, } from "./experimental/evaluation.js" +export { Realtime } from "./experimental/realtime.js" +export type { + RealtimeConnectInput, + RealtimeSession, + RealtimeTransport, + RealtimeTransportInput, +} from "./experimental/realtime.js" diff --git a/packages/ai/src/experimental/realtime.ts b/packages/ai/src/experimental/realtime.ts new file mode 100644 index 00000000000..42191fedd40 --- /dev/null +++ b/packages/ai/src/experimental/realtime.ts @@ -0,0 +1,150 @@ +import { Effect, type Redacted, Schema } from "effect" +import { HttpClientRequest } from "effect/http" +import { RequestExecutor } from "../route/executor.js" +import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "../schema/index.js" +import { encodeJson } from "../utils/json.js" + +const OPENAI_BASE_URL = "https://api.openai.com/v1" +const XAI_BASE_URL = "https://api.x.ai/v1" + +export type RealtimeTransportInput = + | { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number } + | { readonly type: "websocket" } + +export type RealtimeTransport = + | { readonly type: "webrtc"; readonly sdp: string } + | { readonly type: "websocket"; readonly url: string; readonly token: string; readonly expiresAt: number } + +export type RealtimeSession = Readonly> + +export interface RealtimeConnectInput { + readonly model: string + readonly apiKey: Redacted.Redacted + readonly session?: RealtimeSession + readonly transport: RealtimeTransportInput +} + +const engines = [ + { family: "gpt-live", id: "openai-live", provider: "openai" }, + { family: "gpt-realtime", id: "openai-realtime", provider: "openai" }, + { family: "grok-voice", id: "xai-realtime", provider: "xai" }, +] as const + +// Matches the bare family name (`gpt-realtime`) and its versions (`gpt-realtime-2.1`, `gpt-realtime-mini`). +export const engine = (model: string) => + engines.find((item) => model === item.family || model.startsWith(`${item.family}-`)) + +export const connect = Effect.fn("Realtime.connect")(function* (input: RealtimeConnectInput) { + const resolved = engine(input.model) + if (!resolved) return yield* invalidRequest(`Unsupported voice model: ${input.model}`) + const session = { ...input.session, model: input.model } + + switch (resolved.id) { + case "openai-live": { + if (input.transport.type !== "webrtc") + return yield* invalidRequest("OpenAI GPT-Live only supports WebRTC transport") + const data = yield* post( + resolved.id, + `${OPENAI_BASE_URL}/live/sessions`, + input.apiKey, + { session, transport: input.transport }, + LiveSessionResponse, + ) + return { type: "webrtc" as const, sdp: data.transport.sdp } + } + case "openai-realtime": { + if (input.transport.type === "websocket") + return yield* clientSecret(resolved.id, OPENAI_BASE_URL, input.model, input.apiKey, session) + const form = new FormData() + form.append("sdp", input.transport.sdp) + form.append("session", encodeJson(session)) + const query = input.transport.dcid === undefined ? "" : `?dcid=${input.transport.dcid}` + const sdp = yield* post( + resolved.id, + `${OPENAI_BASE_URL}/realtime/calls${query}`, + input.apiKey, + form, + Schema.String, + ) + return { type: "webrtc" as const, sdp } + } + case "xai-realtime": { + if (input.transport.type !== "websocket") + return yield* invalidRequest("xAI Grok Voice only supports WebSocket transport") + return yield* clientSecret(resolved.id, XAI_BASE_URL, input.model, input.apiKey) + } + } +}) + +export const Realtime = { + engine, + connect, +} as const + +const LiveSessionResponse = Schema.fromJsonString( + Schema.Struct({ + transport: Schema.Struct({ sdp: Schema.String }), + }), +) + +// xAI's ephemeral-token docs (https://docs.x.ai/developers/model-capabilities/audio/ephemeral-tokens) +// document the request body (`{ expires_after: { seconds } }`) but not the response payload; we assume +// it mirrors OpenAI's `{ value, expires_at }` response shape. +const ClientSecretResponse = Schema.fromJsonString( + Schema.Struct({ + value: Schema.String, + expires_at: Schema.Number, + }), +) + +const invalidRequest = (message: string) => new AIError({ reason: new InvalidRequestError({ message }) }) + +const post = ( + route: string, + url: string, + apiKey: Redacted.Redacted, + body: Record | FormData, + response: Schema.Codec, +) => + Effect.gen(function* () { + const executor = yield* RequestExecutor.Service + const res = yield* executor.execute( + HttpClientRequest.post(url).pipe( + HttpClientRequest.bearerToken(apiKey), + body instanceof FormData + ? HttpClientRequest.bodyFormData(body) + : HttpClientRequest.bodyText(encodeJson(body), "application/json"), + ), + ) + const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers }) + const fail = (message: string, cause?: unknown, raw?: string) => + new AIError({ reason: new InvalidProviderOutputError({ route, message, body: raw, http, cause }) }) + const text = yield* res.text.pipe(Effect.mapError((cause) => fail(`Failed to read ${route} response`, cause))) + return yield* Schema.decodeEffect(response)(text).pipe( + Effect.mapError((cause) => fail(`${route} returned an invalid response`, cause, text)), + ) + }) + +const clientSecret = ( + route: string, + baseURL: string, + model: string, + apiKey: Redacted.Redacted, + session?: Record, +) => + post( + route, + `${baseURL}/realtime/client_secrets`, + apiKey, + { expires_after: { seconds: 60 }, session }, + ClientSecretResponse, + ).pipe( + Effect.map( + (data): RealtimeTransport => ({ + type: "websocket", + url: `${baseURL.replace(/^http/, "ws")}/realtime?model=${encodeURIComponent(model)}`, + token: data.value, + expiresAt: data.expires_at, + }), + ), + ) diff --git a/packages/ai/test/realtime.test.ts b/packages/ai/test/realtime.test.ts new file mode 100644 index 00000000000..61b8be24fbb --- /dev/null +++ b/packages/ai/test/realtime.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, test } from "bun:test" +import { Effect, Redacted } from "effect" +import { HttpClientRequest } from "effect/http" +import { Realtime } from "../src/experimental.js" +import { it } from "./lib/effect.js" +import { dynamicResponse } from "./lib/http.js" + +describe("experimental Realtime", () => { + it.effect("connects OpenAI GPT-Live, OpenAI Realtime (WebRTC + WebSocket), and xAI Grok Voice", () => + Effect.gen(function* () { + expect( + yield* Realtime.connect({ + model: "gpt-live-1", + apiKey: Redacted.make("sk-openai"), + session: { instructions: "Be concise.", voice: "marin" }, + transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 }, + }), + ).toEqual({ type: "webrtc", sdp: "v=0\r\no=live-answer\r\n" }) + + expect( + yield* Realtime.connect({ + model: "gpt-realtime-2.1", + apiKey: Redacted.make("sk-openai"), + session: { type: "realtime", instructions: "Brief." }, + transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 }, + }), + ).toEqual({ type: "webrtc", sdp: "v=0\r\no=rtc-answer\r\n" }) + + expect( + yield* Realtime.connect({ + model: "gpt-realtime-2.1", + apiKey: Redacted.make("sk-openai"), + session: { type: "realtime", instructions: "Brief." }, + transport: { type: "websocket" }, + }), + ).toEqual({ + type: "websocket", + url: "wss://api.openai.com/v1/realtime?model=gpt-realtime-2.1", + token: "ek_1", + expiresAt: 1760000060, + }) + + expect( + yield* Realtime.connect({ + model: "grok-voice-3.0", + apiKey: Redacted.make("xai-key"), + session: { voice: "ara" }, + transport: { type: "websocket" }, + }), + ).toEqual({ + type: "websocket", + url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0", + token: "ek_2", + expiresAt: 1760000120, + }) + }).pipe( + Effect.provide( + dynamicResponse((input) => + Effect.gen(function* () { + const req = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie) + if (req.url === "https://api.openai.com/v1/live/sessions") { + expect(req.headers.get("authorization")).toBe("Bearer sk-openai") + expect(JSON.parse(input.text)).toEqual({ + session: { instructions: "Be concise.", voice: "marin", model: "gpt-live-1" }, + transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 }, + }) + return input.respond( + JSON.stringify({ session: { id: "s1" }, transport: { sdp: "v=0\r\no=live-answer\r\n" } }), + ) + } + if (req.url === "https://api.openai.com/v1/realtime/calls?dcid=4") { + expect(req.headers.get("authorization")).toBe("Bearer sk-openai") + const boundary = input.text.slice(2, input.text.indexOf("\r\n")) + const form = yield* Effect.promise(() => + new Response(input.text, { + headers: { "content-type": `multipart/form-data; boundary=${boundary}` }, + }).formData(), + ) + expect(form.get("sdp")).toBe("v=0\r\no=offer\r\n") + expect(JSON.parse(String(form.get("session")))).toEqual({ + type: "realtime", + instructions: "Brief.", + model: "gpt-realtime-2.1", + }) + return input.respond("v=0\r\no=rtc-answer\r\n", { headers: { "content-type": "application/sdp" } }) + } + if (req.url === "https://api.openai.com/v1/realtime/client_secrets") { + expect(req.headers.get("authorization")).toBe("Bearer sk-openai") + expect(JSON.parse(input.text)).toEqual({ + expires_after: { seconds: 60 }, + session: { type: "realtime", instructions: "Brief.", model: "gpt-realtime-2.1" }, + }) + return input.respond(JSON.stringify({ value: "ek_1", expires_at: 1760000060 })) + } + expect(req.url).toBe("https://api.x.ai/v1/realtime/client_secrets") + expect(req.headers.get("authorization")).toBe("Bearer xai-key") + expect(JSON.parse(input.text)).toEqual({ expires_after: { seconds: 60 } }) + return input.respond(JSON.stringify({ value: "ek_2", expires_at: 1760000120 })) + }), + ), + ), + ), + ) + + test("routes bare and versioned model names to their engine", () => { + expect( + ["gpt-realtime", "gpt-realtime-2.1", "gpt-live", "grok-voice-latest"].map((model) => Realtime.engine(model)?.id), + ).toEqual(["openai-realtime", "openai-realtime", "openai-live", "xai-realtime"]) + expect(Realtime.engine("gpt-realtimex")).toBeUndefined() + }) + + it.effect("rejects unsupported transports before network I/O", () => + Effect.gen(function* () { + const liveWs = yield* Realtime.connect({ + model: "gpt-live-1", + apiKey: Redacted.make("k"), + transport: { type: "websocket" }, + }).pipe(Effect.flip) + const grokRtc = yield* Realtime.connect({ + model: "grok-voice-3.0", + apiKey: Redacted.make("k"), + transport: { type: "webrtc", sdp: "v=0" }, + }).pipe(Effect.flip) + expect([liveWs.reason._tag, grokRtc.reason._tag]).toEqual(["InvalidRequest", "InvalidRequest"]) + }).pipe(Effect.provide(dynamicResponse(() => Effect.die("unsupported transport reached the network")))), + ) +}) diff --git a/packages/client/src/effect/api/api.ts b/packages/client/src/effect/api/api.ts index 5310d67cdd2..4580c6bbfe3 100644 --- a/packages/client/src/effect/api/api.ts +++ b/packages/client/src/effect/api/api.ts @@ -1402,6 +1402,24 @@ export interface GenerateApi { readonly text: GenerateTextOperation } +export type VoiceSessionInput = { + readonly model: string + readonly session?: { readonly [x: string]: unknown } | undefined + readonly transport: + | { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined } + | { readonly type: "websocket" } +} +export type VoiceSessionOutput = { + readonly transport: + | { readonly type: "webrtc"; readonly sdp: string } + | { readonly type: "websocket"; readonly url: string; readonly token: string; readonly expiresAt: number } +} +export type VoiceSessionOperation = (input: VoiceSessionInput) => Effect.Effect + +export interface VoiceApi { + readonly session: VoiceSessionOperation +} + export type ProviderListInput = { readonly location?: { readonly directory?: string | undefined } | undefined } export type ProviderListOutput = { readonly location: Location.PublicRef; readonly data: ReadonlyArray } export type ProviderListOperation = (input?: ProviderListInput) => Effect.Effect @@ -2272,6 +2290,7 @@ export interface AppApi { readonly message: MessageApi readonly model: ModelApi readonly generate: GenerateApi + readonly voice: VoiceApi readonly provider: ProviderApi readonly integration: IntegrationApi readonly mcp: McpApi diff --git a/packages/client/src/effect/generated/client.ts b/packages/client/src/effect/generated/client.ts index c407bd2fcc8..057240d30d2 100644 --- a/packages/client/src/effect/generated/client.ts +++ b/packages/client/src/effect/generated/client.ts @@ -115,6 +115,8 @@ import type { ModelDefaultOutput, GenerateTextInput, GenerateTextOutput, + VoiceSessionInput, + VoiceSessionOutput, ProviderListInput, ProviderListOutput, ProviderGetInput, @@ -863,6 +865,18 @@ const EndpointGenerateText = (raw: RawClient["server.generate"]) => (input: Gene const adaptGroupGenerate = (raw: RawClient["server.generate"]) => ({ text: EndpointGenerateText(raw) }) +const EndpointVoiceSession = (raw: RawClient["server.voice"]) => (input: VoiceSessionInput) => + preserveEffect()( + raw["voice.session"]({ + payload: { model: input["model"], session: input["session"], transport: input["transport"] }, + }).pipe( + Effect.mapError(mapClientError), + Effect.map((value) => value.data), + ), + ) + +const adaptGroupVoice = (raw: RawClient["server.voice"]) => ({ session: EndpointVoiceSession(raw) }) + const EndpointProviderList = (raw: RawClient["server.provider"]) => (input?: ProviderListInput) => preserveEffect()( raw["provider.list"]({ query: { location: input?.["location"] } }).pipe(Effect.mapError(mapClientError)), @@ -1616,6 +1630,7 @@ const adaptClient = (raw: RawClient) => ({ message: adaptGroupMessage(raw["server.message"]), model: adaptGroupModel(raw["server.model"]), generate: adaptGroupGenerate(raw["server.generate"]), + voice: adaptGroupVoice(raw["server.voice"]), provider: adaptGroupProvider(raw["server.provider"]), integration: adaptGroupIntegration(raw["server.integration"]), mcp: adaptGroupMcp(raw["server.mcp"]), diff --git a/packages/client/src/promise/generated/client.ts b/packages/client/src/promise/generated/client.ts index 50e15484974..444118120a6 100644 --- a/packages/client/src/promise/generated/client.ts +++ b/packages/client/src/promise/generated/client.ts @@ -109,6 +109,8 @@ import type { ModelDefaultOutput, GenerateTextInput, GenerateTextOutput, + VoiceSessionInput, + VoiceSessionOutput, ProviderListInput, ProviderListOutput, ProviderGetInput, @@ -1151,6 +1153,20 @@ export function make(options: ClientOptions) { requestOptions, ).then((value) => value.data), }, + voice: { + session: (input: VoiceSessionInput, requestOptions?: RequestOptions) => + request<{ readonly data: VoiceSessionOutput }>( + { + method: "POST", + path: `/api/experimental/voice/session`, + body: { model: input["model"], session: input["session"], transport: input["transport"] }, + successStatus: 200, + declaredStatuses: [400, 401, 404, 503], + empty: false, + }, + requestOptions, + ).then((value) => value.data), + }, provider: { list: (input?: ProviderListInput, requestOptions?: RequestOptions) => request( diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index 106a250eb6b..96ba475e5b7 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -174,6 +174,10 @@ export type MoneyUSDPerMillionTokens = number export type GenerateTextResponse = { data: { text: string } } +export type VoiceWebRTCTransport = { type: "webrtc"; sdp: string } + +export type VoiceWebSocketTransport = { type: "websocket"; url: string; token: string; expiresAt: number } + export type IntegrationID = string export type IntegrationMethodID = string @@ -658,6 +662,8 @@ export type ModelCost = { cache: { read: MoneyUSDPerMillionTokens; write: MoneyUSDPerMillionTokens } } +export type VoiceTransport = VoiceWebRTCTransport | VoiceWebSocketTransport + export type IntegrationCommandMethod = { id: IntegrationMethodID type: "command" @@ -1941,6 +1947,8 @@ export type ProviderInfo = { body?: { [x: string]: any } } +export type VoiceSessionResponse = { data: { transport: VoiceTransport } } + export type ConnectionInfo = ConnectionCredentialInfo | ConnectionEnvInfo export type PtyCreated = { @@ -5780,6 +5788,32 @@ export type GenerateTextInput = { export type GenerateTextOutput = GenerateTextResponse["data"] +export type VoiceSessionInput = { + readonly model: { + readonly model: string + readonly session?: { readonly [x: string]: unknown } | undefined + readonly transport: + | { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined } + | { readonly type: "websocket" } + }["model"] + readonly session?: { + readonly model: string + readonly session?: { readonly [x: string]: unknown } | undefined + readonly transport: + | { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined } + | { readonly type: "websocket" } + }["session"] + readonly transport: { + readonly model: string + readonly session?: { readonly [x: string]: unknown } | undefined + readonly transport: + | { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined } + | { readonly type: "websocket" } + }["transport"] +} + +export type VoiceSessionOutput = VoiceSessionResponse["data"] + export type ProviderListInput = { readonly location?: { readonly location?: { readonly directory?: string | undefined } | undefined }["location"] } diff --git a/packages/client/test/promise.test.ts b/packages/client/test/promise.test.ts index c76b07639dc..d2f8b4c28d5 100644 --- a/packages/client/test/promise.test.ts +++ b/packages/client/test/promise.test.ts @@ -13,6 +13,7 @@ test("exposes every standard HTTP API group", () => { "message", "model", "generate", + "voice", "provider", "integration", "mcp", @@ -45,6 +46,7 @@ test("exposes every standard HTTP API group", () => { expect(Object.keys(client.integration.oauth)).toEqual(["connect", "status", "complete", "cancel"]) expect(Object.keys(client.integration.command)).toEqual(["connect", "status", "cancel"]) expect(Object.keys(client.websearch)).toEqual(["providers", "query"]) + expect(Object.keys(client.voice)).toEqual(["session"]) expect(Object.keys(client.file)).toEqual(["read", "list", "find", "write"]) expect(Object.keys(client.vcs)).toEqual(["init", "get", "base", "status", "branch", "diff"]) expect(Object.keys(client.pty)).toEqual(["list", "create", "get", "update", "remove", "connect"]) diff --git a/packages/core/src/instance.ts b/packages/core/src/instance.ts index 0586fbaf9dc..9bcde2c5922 100644 --- a/packages/core/src/instance.ts +++ b/packages/core/src/instance.ts @@ -13,6 +13,7 @@ import { Formatter } from "./formatter.js" import { FileSystem } from "./filesystem.js" import { FileSystemSearch } from "./filesystem/search.js" import { Generate } from "./generate.js" +import { Voice } from "./voice.js" import { Form } from "./form.js" import { Image } from "./image.js" import { LocationWatcher } from "./filesystem/location-watcher.js" @@ -97,6 +98,7 @@ const nodes = [ InstructionEntry.node, Form.node, Generate.node, + Voice.node, ReadToolFileSystem.node, McpTool.node, SessionInstructions.node, diff --git a/packages/core/src/voice.ts b/packages/core/src/voice.ts new file mode 100644 index 00000000000..811ea10f691 --- /dev/null +++ b/packages/core/src/voice.ts @@ -0,0 +1,88 @@ +export * as Voice from "./voice.js" + +import { type AIError } from "@opencode/ai" +import { + Realtime, + type RealtimeSession, + type RealtimeTransport, + type RealtimeTransportInput, +} from "@opencode/ai/experimental" +import { RequestExecutor } from "@opencode/ai/route" +import { makeLocationNode } from "@opencode/util/effect/app-node" +import { Context, Effect, Layer, Redacted, Schema } from "effect" +import { requestExecutor } from "./effect/app-node-platform.js" +import { Integration } from "./integration.js" + +export interface SessionInput { + readonly model: string + readonly session?: RealtimeSession + readonly transport: RealtimeTransportInput +} + +export class InvalidRequestError extends Schema.TaggedError()("Voice.InvalidRequestError", { + message: Schema.String, +}) {} + +export class UnavailableError extends Schema.TaggedError()("Voice.UnavailableError", { + message: Schema.String, + service: Schema.optional(Schema.String), +}) {} + +export type Error = InvalidRequestError | UnavailableError + +export interface Interface { + readonly session: (input: SessionInput) => Effect.Effect +} + +export class Service extends Context.Service()("@opencode/Voice") {} + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const executor = yield* RequestExecutor.Service + const integrations = yield* Integration.Service + + const session = Effect.fn("Voice.session")(function* (input: SessionInput) { + const engine = Realtime.engine(input.model) + if (!engine) return yield* new InvalidRequestError({ message: `Unsupported voice model: ${input.model}` }) + + // Integration lists the active credential first, then newer stored credentials, then its env vars. + const connection = (yield* integrations.get(Integration.ID.make(engine.provider)))?.connections.find( + (item) => item.type === "env" || item.method === "key", + ) + const credential = connection + ? yield* integrations.connection + .resolve(connection) + .pipe( + Effect.mapError((error) => new UnavailableError({ service: engine.provider, message: error.message })), + ) + : undefined + if (credential?.type !== "key") + return yield* new UnavailableError({ + service: engine.provider, + message: + engine.provider === "openai" + ? "Voice needs an OpenAI API key on the server (a stored key or OPENAI_API_KEY). ChatGPT sign-in can't start voice sessions." + : "Voice needs an xAI API key on the server (a stored key or XAI_API_KEY).", + }) + + return yield* Realtime.connect({ ...input, apiKey: Redacted.make(credential.key) }).pipe( + Effect.provideService(RequestExecutor.Service, executor), + Effect.mapError( + (error: AIError): Error => + error.reason._tag === "InvalidRequest" || error.reason._tag === "UnsupportedOperation" + ? new InvalidRequestError({ message: error.message }) + : new UnavailableError({ message: error.message, service: engine.provider }), + ), + ) + }) + + return Service.of({ session }) + }), +) + +export const node = makeLocationNode({ + service: Service, + layer, + deps: [Integration.node, requestExecutor], +}) diff --git a/packages/core/test/voice.test.ts b/packages/core/test/voice.test.ts new file mode 100644 index 00000000000..82b045db996 --- /dev/null +++ b/packages/core/test/voice.test.ts @@ -0,0 +1,107 @@ +import { expect } from "bun:test" +import { RequestExecutor } from "@opencode/ai/route" +import { Credential } from "@opencode/core/credential" +import { Integration } from "@opencode/core/integration" +import { Voice } from "@opencode/core/voice" +import { Effect, Layer } from "effect" +import { HttpClient, type HttpClientRequest, HttpClientResponse } from "effect/http" +import { testEffect } from "./lib/effect" + +const it = testEffect(Layer.empty) + +const credential = (id: string, method: "key" | "oauth") => ({ + type: "credential" as const, + id: Credential.ID.make(id), + label: id, + method, +}) + +const keys: Record = { active: "sk-active", stored: "sk-stored", XAI_API_KEY: "xai-env-key" } + +it.effect("uses the first key or env connection Integration lists and never a ChatGPT sign-in", () => + Effect.gen(function* () { + const auths: string[] = [] + const call = (model: string, connections: Integration.Info["connections"]) => + Effect.flatMap(Voice.Service, (voice) => + voice.session({ + model, + transport: model.startsWith("grok-") ? { type: "websocket" } : { type: "webrtc", sdp: "v=0" }, + }), + ).pipe( + Effect.provide( + Voice.layer.pipe( + Layer.provide( + Layer.merge( + Layer.mock(Integration.Service, { + revision: () => 0, + get: (id) => Effect.succeed({ id, name: id, methods: [], connections }), + connection: { + active: () => Effect.die("unused"), + resolve: (connection) => + connection.type === "credential" && connection.method === "oauth" + ? Effect.die("a ChatGPT sign-in was resolved") + : Effect.succeed( + Credential.Key.make({ + type: "key", + key: keys[connection.type === "env" ? connection.name : connection.id], + }), + ), + key: () => Effect.die("unused"), + external: () => Effect.die("unused"), + activate: () => Effect.die("unused"), + update: () => Effect.die("unused"), + remove: () => Effect.die("unused"), + status: () => Effect.die("unused"), + }, + oauth: { + connect: () => Effect.die("unused"), + status: () => Effect.die("unused"), + complete: () => Effect.die("unused"), + cancel: () => Effect.die("unused"), + }, + command: { + connect: () => Effect.die("unused"), + status: () => Effect.die("unused"), + cancel: () => Effect.die("unused"), + }, + }), + RequestExecutor.layer.pipe( + Layer.provide( + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((req: HttpClientRequest.HttpClientRequest) => + Effect.sync(() => { + auths.push(req.headers.authorization ?? "") + return HttpClientResponse.fromWeb( + req, + model.startsWith("grok-") + ? Response.json({ value: "ek_xai", expires_at: 1760000999 }) + : Response.json({ transport: { sdp: "v=0\r\no=answer\r\n" } }), + ) + }), + ), + ), + ), + ), + ), + ), + ), + ), + ) + + yield* call("gpt-live-1", [credential("active", "key"), credential("stored", "key")]) + yield* call("gpt-live-1", [credential("chatgpt", "oauth"), credential("stored", "key")]) + const grok = yield* call("grok-voice-3.0", [{ type: "env", name: "XAI_API_KEY" }]) + expect(auths).toEqual(["Bearer sk-active", "Bearer sk-stored", "Bearer xai-env-key"]) + expect(grok).toEqual({ + type: "websocket", + url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0", + token: "ek_xai", + expiresAt: 1760000999, + }) + + const signInOnly = yield* call("gpt-live-1", [credential("chatgpt", "oauth")]).pipe(Effect.flip) + expect(signInOnly._tag).toBe("Voice.UnavailableError") + expect(signInOnly.message).toContain("OpenAI API key") + }), +) diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index 9839d62e828..c502278ffcc 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -5357,6 +5357,98 @@ } } }, + "/api/experimental/voice/session": { + "post": { + "tags": ["voice"], + "operationId": "experimental.voice.session", + "parameters": [], + "security": [], + "responses": { + "200": { + "description": "VoiceSessionResponse", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VoiceSessionResponse" + } + } + } + }, + "400": { + "description": "InvalidRequestError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InvalidRequestErrorEncoded" + } + } + } + }, + "401": { + "description": "UnauthorizedError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UnauthorizedErrorEncoded" + } + } + } + }, + "404": { + "description": "LocationNotFoundError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LocationNotFoundErrorEncoded" + } + } + } + }, + "503": { + "description": "ServiceUnavailableError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServiceUnavailableErrorEncoded" + } + } + } + } + }, + "description": "Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.", + "summary": "Create voice session", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "session": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ] + }, + "transport": { + "$ref": "#/components/schemas/VoiceTransportInput" + } + }, + "required": ["model", "transport"], + "additionalProperties": false + } + } + }, + "required": true + } + } + }, "/api/provider": { "get": { "tags": ["provider"], @@ -20330,6 +20422,128 @@ "required": ["_tag", "providerID", "message"], "additionalProperties": false }, + "VoiceSessionResponse": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "transport": { + "$ref": "#/components/schemas/VoiceTransport" + } + }, + "required": ["transport"], + "additionalProperties": false + } + }, + "required": ["data"], + "additionalProperties": false + }, + "VoiceTransport": { + "anyOf": [ + { + "$ref": "#/components/schemas/VoiceWebRTCTransport" + }, + { + "$ref": "#/components/schemas/VoiceWebSocketTransport" + } + ] + }, + "VoiceTransportInput": { + "anyOf": [ + { + "$ref": "#/components/schemas/VoiceWebRTCTransportInput" + }, + { + "$ref": "#/components/schemas/VoiceWebSocketTransportInput" + } + ] + }, + "VoiceWebRTCTransport": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["webrtc"] + }, + "sdp": { + "type": "string" + } + }, + "required": ["type", "sdp"], + "additionalProperties": false + }, + "VoiceWebRTCTransportInput": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["webrtc"] + }, + "sdp": { + "type": "string" + }, + "dcid": { + "anyOf": [ + { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "enum": ["Infinity", "-Infinity", "NaN"] + } + ] + }, + { + "type": "null" + } + ] + } + }, + "required": ["type", "sdp"], + "additionalProperties": false + }, + "VoiceWebSocketTransport": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["websocket"] + }, + "url": { + "type": "string" + }, + "token": { + "type": "string" + }, + "expiresAt": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "enum": ["Infinity", "-Infinity", "NaN"] + } + ] + } + }, + "required": ["type", "url", "token", "expiresAt"], + "additionalProperties": false + }, + "VoiceWebSocketTransportInput": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["websocket"] + } + }, + "required": ["type"], + "additionalProperties": false + }, "WebSearch.ID": { "type": "string" }, @@ -20523,6 +20737,10 @@ "name": "generate", "description": "Experimental one-shot generation routes." }, + { + "name": "voice", + "description": "Experimental server-brokered voice session routes." + }, { "name": "provider", "description": "Experimental provider routes." diff --git a/packages/protocol/src/api.ts b/packages/protocol/src/api.ts index fe653d55b4f..d509a8df8f6 100644 --- a/packages/protocol/src/api.ts +++ b/packages/protocol/src/api.ts @@ -2,6 +2,7 @@ import { Context } from "effect" import { HttpApi, HttpApiGroup, HttpApiMiddleware, OpenApi } from "effect/http-api" import { SchemaErrorMiddleware } from "./middleware/schema-error.js" import { GenerateGroup } from "./groups/generate.js" +import { VoiceGroup } from "./groups/voice.js" import { MessageGroup } from "./groups/message.js" import { ModelGroup } from "./groups/model.js" import { ProviderGroup } from "./groups/provider.js" @@ -94,6 +95,7 @@ type ApiGroups< | typeof MigrationGroup | typeof WorktreeGroup | typeof GenerateGroup + | typeof VoiceGroup | typeof PersistentPtyGroup | typeof CredentialGroup | LocationGroups @@ -165,6 +167,7 @@ const makeApiFromGroup = < .add(MessageGroup) .add(ModelGroup.middleware(locationMiddleware)) .add(GenerateGroup) + .add(VoiceGroup) .add(ProviderGroup.middleware(locationMiddleware)) .add(IntegrationGroup.middleware(locationMiddleware)) .add(McpGroup.middleware(locationMiddleware)) diff --git a/packages/protocol/src/client.ts b/packages/protocol/src/client.ts index 3af3e3b41a6..ea860b14972 100644 --- a/packages/protocol/src/client.ts +++ b/packages/protocol/src/client.ts @@ -44,6 +44,7 @@ export const groupNames = { "server.message": "message", "server.model": "model", "server.generate": "generate", + "server.voice": "voice", "server.provider": "provider", "server.integration": "integration", "server.websearch": "websearch", diff --git a/packages/protocol/src/groups/voice.ts b/packages/protocol/src/groups/voice.ts new file mode 100644 index 00000000000..30e150bcd96 --- /dev/null +++ b/packages/protocol/src/groups/voice.ts @@ -0,0 +1,65 @@ +import { Schema } from "effect" +import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/http-api" +import { InvalidRequestError, LocationNotFoundError, ServiceUnavailableError } from "../errors.js" + +export const VoiceWebRTCTransportInput = Schema.Struct({ + type: Schema.Literal("webrtc"), + sdp: Schema.String, + dcid: Schema.optional(Schema.Number), +}).annotate({ identifier: "VoiceWebRTCTransportInput" }) + +export const VoiceWebSocketTransportInput = Schema.Struct({ + type: Schema.Literal("websocket"), +}).annotate({ identifier: "VoiceWebSocketTransportInput" }) + +export const VoiceTransportInput = Schema.Union([ + VoiceWebRTCTransportInput, + VoiceWebSocketTransportInput, +]).annotate({ identifier: "VoiceTransportInput" }) + +export const VoiceWebRTCTransport = Schema.Struct({ + type: Schema.Literal("webrtc"), + sdp: Schema.String, +}).annotate({ identifier: "VoiceWebRTCTransport" }) + +export const VoiceWebSocketTransport = Schema.Struct({ + type: Schema.Literal("websocket"), + url: Schema.String, + token: Schema.String, + expiresAt: Schema.Number, +}).annotate({ identifier: "VoiceWebSocketTransport" }) + +export const VoiceTransport = Schema.Union([ + VoiceWebRTCTransport, + VoiceWebSocketTransport, +]).annotate({ identifier: "VoiceTransport" }) + +export const VoiceGroup = HttpApiGroup.make("server.voice") + .add( + HttpApiEndpoint.post("voice.session", "/api/experimental/voice/session", { + payload: Schema.Struct({ + model: Schema.String, + session: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + transport: VoiceTransportInput, + }), + success: Schema.Struct({ + data: Schema.Struct({ + transport: VoiceTransport, + }), + }).annotate({ identifier: "VoiceSessionResponse" }), + error: [InvalidRequestError, ServiceUnavailableError, LocationNotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "experimental.voice.session", + summary: "Create voice session", + description: + "Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.", + }), + ), + ) + .annotateMerge( + OpenApi.annotations({ + title: "voice", + description: "Experimental server-brokered voice session routes.", + }), + ) diff --git a/packages/server/src/handlers.ts b/packages/server/src/handlers.ts index 5dccd1d847a..1e20913605c 100644 --- a/packages/server/src/handlers.ts +++ b/packages/server/src/handlers.ts @@ -1,5 +1,6 @@ import { Layer } from "effect" import { GenerateHandler } from "./handlers/generate" +import { VoiceHandler } from "./handlers/voice" import { MessageHandler } from "./handlers/message" import { ModelHandler } from "./handlers/model" import { ProviderHandler } from "./handlers/provider" @@ -42,6 +43,7 @@ export const handlers = Layer.mergeAll( MessageHandler, ModelHandler, GenerateHandler, + VoiceHandler, ProviderHandler, IntegrationHandler, WebSearchHandler, diff --git a/packages/server/src/handlers/voice.ts b/packages/server/src/handlers/voice.ts new file mode 100644 index 00000000000..afdcb21113d --- /dev/null +++ b/packages/server/src/handlers/voice.ts @@ -0,0 +1,39 @@ +import { Location } from "@opencode/core/location" +import { LocationServiceMap } from "@opencode/core/location-services" +import { AbsolutePath } from "@opencode/core/schema" +import { Voice } from "@opencode/core/voice" +import { InvalidRequestError, ServiceUnavailableError } from "@opencode/protocol/errors" +import { Global } from "@opencode/util/global" +import { Effect } from "effect" +import { HttpApiBuilder } from "effect/http-api" +import { Api } from "../api" +import { awaitPlugins, locationErrors } from "../location" + +export const VoiceHandler = HttpApiBuilder.group(Api, "server.voice", (handlers) => + Effect.gen(function* () { + const global = yield* Global.Service + const locations = yield* LocationServiceMap.Service + const services = locations.get(Location.Ref.make({ directory: AbsolutePath.make(global.config) })) + return handlers.handle( + "voice.session", + Effect.fn("server.voice.session")( + function* (request) { + yield* awaitPlugins + const voice = yield* Voice.Service + const transport = yield* voice + .session(request.payload) + .pipe( + Effect.mapError((error) => + error._tag === "Voice.InvalidRequestError" + ? new InvalidRequestError({ message: error.message }) + : new ServiceUnavailableError({ message: error.message, service: error.service }), + ), + ) + return { data: { transport } } + }, + Effect.provide(services), + locationErrors, + ), + ) + }), +) diff --git a/packages/server/test/voice.test.ts b/packages/server/test/voice.test.ts new file mode 100644 index 00000000000..eda31544a10 --- /dev/null +++ b/packages/server/test/voice.test.ts @@ -0,0 +1,88 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { expect } from "bun:test" +import { Voice } from "@opencode/core/voice" +import { makeLocationNode } from "@opencode/util/effect/app-node" +import { Effect, Layer } from "effect" +import { tmpdir } from "../../core/test/fixture/tmpdir" +import { it } from "../../core/test/lib/effect" +import { ServerFetch } from "../src/fetch" + +const voice = makeLocationNode({ + service: Voice.Service, + layer: Layer.succeed( + Voice.Service, + Voice.Service.of({ + session: (input) => { + if (input.model === "invalid") + return Effect.fail(new Voice.InvalidRequestError({ message: "Unsupported voice model" })) + if (input.model === "unavailable") + return Effect.fail(new Voice.UnavailableError({ message: "OpenAI API key is required", service: "openai" })) + return Effect.succeed( + input.transport.type === "webrtc" + ? { type: "webrtc" as const, sdp: `answer:${input.model}:${input.transport.sdp}` } + : { + type: "websocket" as const, + url: `wss://api.x.ai/v1/realtime?model=${input.model}`, + token: "ek_test", + expiresAt: 1760000000, + }, + ) + }, + }), + ), + deps: [], +}) + +it.live("routes POST /api/experimental/voice/session and maps errors", () => + Effect.gen(function* () { + const tmp = yield* Effect.acquireDisposable(Effect.promise(() => tmpdir("opencode-voice-endpoint-"))) + const global = path.join(tmp.path, "global") + yield* Effect.promise(() => fs.mkdir(global)) + const handler = yield* ServerFetch.make( + { + database: { path: ":memory:" }, + config: { directory: global }, + fs: { filewatcher: false }, + models: { fetch: false }, + }, + { overrides: [Voice.node.replace(voice)] }, + ) + const post = (body: unknown) => + Effect.promise(() => + handler( + new Request("http://opencode.local/api/experimental/voice/session", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ), + ) + + const rtcRes = yield* post({ + model: "gpt-live-1", + session: { voice: "marin" }, + transport: { type: "webrtc", sdp: "v=0", dcid: 4 }, + }) + expect(rtcRes.status).toBe(200) + expect(yield* Effect.promise(() => rtcRes.json())).toEqual({ + data: { transport: { type: "webrtc", sdp: "answer:gpt-live-1:v=0" } }, + }) + + const wsRes = yield* post({ model: "grok-voice-3.0", transport: { type: "websocket" } }) + expect(wsRes.status).toBe(200) + expect(yield* Effect.promise(() => wsRes.json())).toEqual({ + data: { + transport: { + type: "websocket", + url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0", + token: "ek_test", + expiresAt: 1760000000, + }, + }, + }) + + expect((yield* post({ model: "invalid", transport: { type: "websocket" } })).status).toBe(400) + expect((yield* post({ model: "unavailable", transport: { type: "websocket" } })).status).toBe(503) + }), +) diff --git a/services/www/openapi.json b/services/www/openapi.json index 9839d62e828..c502278ffcc 100644 --- a/services/www/openapi.json +++ b/services/www/openapi.json @@ -5357,6 +5357,98 @@ } } }, + "/api/experimental/voice/session": { + "post": { + "tags": ["voice"], + "operationId": "experimental.voice.session", + "parameters": [], + "security": [], + "responses": { + "200": { + "description": "VoiceSessionResponse", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VoiceSessionResponse" + } + } + } + }, + "400": { + "description": "InvalidRequestError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InvalidRequestErrorEncoded" + } + } + } + }, + "401": { + "description": "UnauthorizedError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UnauthorizedErrorEncoded" + } + } + } + }, + "404": { + "description": "LocationNotFoundError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LocationNotFoundErrorEncoded" + } + } + } + }, + "503": { + "description": "ServiceUnavailableError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServiceUnavailableErrorEncoded" + } + } + } + } + }, + "description": "Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.", + "summary": "Create voice session", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "session": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ] + }, + "transport": { + "$ref": "#/components/schemas/VoiceTransportInput" + } + }, + "required": ["model", "transport"], + "additionalProperties": false + } + } + }, + "required": true + } + } + }, "/api/provider": { "get": { "tags": ["provider"], @@ -20330,6 +20422,128 @@ "required": ["_tag", "providerID", "message"], "additionalProperties": false }, + "VoiceSessionResponse": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "transport": { + "$ref": "#/components/schemas/VoiceTransport" + } + }, + "required": ["transport"], + "additionalProperties": false + } + }, + "required": ["data"], + "additionalProperties": false + }, + "VoiceTransport": { + "anyOf": [ + { + "$ref": "#/components/schemas/VoiceWebRTCTransport" + }, + { + "$ref": "#/components/schemas/VoiceWebSocketTransport" + } + ] + }, + "VoiceTransportInput": { + "anyOf": [ + { + "$ref": "#/components/schemas/VoiceWebRTCTransportInput" + }, + { + "$ref": "#/components/schemas/VoiceWebSocketTransportInput" + } + ] + }, + "VoiceWebRTCTransport": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["webrtc"] + }, + "sdp": { + "type": "string" + } + }, + "required": ["type", "sdp"], + "additionalProperties": false + }, + "VoiceWebRTCTransportInput": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["webrtc"] + }, + "sdp": { + "type": "string" + }, + "dcid": { + "anyOf": [ + { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "enum": ["Infinity", "-Infinity", "NaN"] + } + ] + }, + { + "type": "null" + } + ] + } + }, + "required": ["type", "sdp"], + "additionalProperties": false + }, + "VoiceWebSocketTransport": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["websocket"] + }, + "url": { + "type": "string" + }, + "token": { + "type": "string" + }, + "expiresAt": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "enum": ["Infinity", "-Infinity", "NaN"] + } + ] + } + }, + "required": ["type", "url", "token", "expiresAt"], + "additionalProperties": false + }, + "VoiceWebSocketTransportInput": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["websocket"] + } + }, + "required": ["type"], + "additionalProperties": false + }, "WebSearch.ID": { "type": "string" }, @@ -20523,6 +20737,10 @@ "name": "generate", "description": "Experimental one-shot generation routes." }, + { + "name": "voice", + "description": "Experimental server-brokered voice session routes." + }, { "name": "provider", "description": "Experimental provider routes." diff --git a/services/www/public/openapi.json b/services/www/public/openapi.json index 9839d62e828..c502278ffcc 100644 --- a/services/www/public/openapi.json +++ b/services/www/public/openapi.json @@ -5357,6 +5357,98 @@ } } }, + "/api/experimental/voice/session": { + "post": { + "tags": ["voice"], + "operationId": "experimental.voice.session", + "parameters": [], + "security": [], + "responses": { + "200": { + "description": "VoiceSessionResponse", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VoiceSessionResponse" + } + } + } + }, + "400": { + "description": "InvalidRequestError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InvalidRequestErrorEncoded" + } + } + } + }, + "401": { + "description": "UnauthorizedError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UnauthorizedErrorEncoded" + } + } + } + }, + "404": { + "description": "LocationNotFoundError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LocationNotFoundErrorEncoded" + } + } + } + }, + "503": { + "description": "ServiceUnavailableError", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServiceUnavailableErrorEncoded" + } + } + } + } + }, + "description": "Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.", + "summary": "Create voice session", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "session": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ] + }, + "transport": { + "$ref": "#/components/schemas/VoiceTransportInput" + } + }, + "required": ["model", "transport"], + "additionalProperties": false + } + } + }, + "required": true + } + } + }, "/api/provider": { "get": { "tags": ["provider"], @@ -20330,6 +20422,128 @@ "required": ["_tag", "providerID", "message"], "additionalProperties": false }, + "VoiceSessionResponse": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "transport": { + "$ref": "#/components/schemas/VoiceTransport" + } + }, + "required": ["transport"], + "additionalProperties": false + } + }, + "required": ["data"], + "additionalProperties": false + }, + "VoiceTransport": { + "anyOf": [ + { + "$ref": "#/components/schemas/VoiceWebRTCTransport" + }, + { + "$ref": "#/components/schemas/VoiceWebSocketTransport" + } + ] + }, + "VoiceTransportInput": { + "anyOf": [ + { + "$ref": "#/components/schemas/VoiceWebRTCTransportInput" + }, + { + "$ref": "#/components/schemas/VoiceWebSocketTransportInput" + } + ] + }, + "VoiceWebRTCTransport": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["webrtc"] + }, + "sdp": { + "type": "string" + } + }, + "required": ["type", "sdp"], + "additionalProperties": false + }, + "VoiceWebRTCTransportInput": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["webrtc"] + }, + "sdp": { + "type": "string" + }, + "dcid": { + "anyOf": [ + { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "enum": ["Infinity", "-Infinity", "NaN"] + } + ] + }, + { + "type": "null" + } + ] + } + }, + "required": ["type", "sdp"], + "additionalProperties": false + }, + "VoiceWebSocketTransport": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["websocket"] + }, + "url": { + "type": "string" + }, + "token": { + "type": "string" + }, + "expiresAt": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "enum": ["Infinity", "-Infinity", "NaN"] + } + ] + } + }, + "required": ["type", "url", "token", "expiresAt"], + "additionalProperties": false + }, + "VoiceWebSocketTransportInput": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["websocket"] + } + }, + "required": ["type"], + "additionalProperties": false + }, "WebSearch.ID": { "type": "string" }, @@ -20523,6 +20737,10 @@ "name": "generate", "description": "Experimental one-shot generation routes." }, + { + "name": "voice", + "description": "Experimental server-brokered voice session routes." + }, { "name": "provider", "description": "Experimental provider routes."