feat(server): add experimental voice session endpoint (#55951)

This commit is contained in:
Kit Langton
2026-10-16 14:37:20 -07:00
committed by GitHub
Unverified
parent 180fd9d622
commit d4a6223d01
20 changed files with 1419 additions and 0 deletions
+7
View File
@@ -27,3 +27,10 @@ export type {
EvaluationResponseFor,
EvaluationRoute,
} from "./experimental/evaluation.js"
export { Realtime } from "./experimental/realtime.js"
export type {
RealtimeConnectInput,
RealtimeSession,
RealtimeTransport,
RealtimeTransportInput,
} from "./experimental/realtime.js"
+150
View File
@@ -0,0 +1,150 @@
import { Effect, type Redacted, Schema } from "effect"
import { HttpClientRequest } from "effect/http"
import { RequestExecutor } from "../route/executor.js"
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "../schema/index.js"
import { encodeJson } from "../utils/json.js"
const OPENAI_BASE_URL = "https://api.openai.com/v1"
const XAI_BASE_URL = "https://api.x.ai/v1"
export type RealtimeTransportInput =
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number }
| { readonly type: "websocket" }
export type RealtimeTransport =
| { readonly type: "webrtc"; readonly sdp: string }
| { readonly type: "websocket"; readonly url: string; readonly token: string; readonly expiresAt: number }
export type RealtimeSession = Readonly<Record<string, unknown>>
export interface RealtimeConnectInput {
readonly model: string
readonly apiKey: Redacted.Redacted
readonly session?: RealtimeSession
readonly transport: RealtimeTransportInput
}
const engines = [
{ family: "gpt-live", id: "openai-live", provider: "openai" },
{ family: "gpt-realtime", id: "openai-realtime", provider: "openai" },
{ family: "grok-voice", id: "xai-realtime", provider: "xai" },
] as const
// Matches the bare family name (`gpt-realtime`) and its versions (`gpt-realtime-2.1`, `gpt-realtime-mini`).
export const engine = (model: string) =>
engines.find((item) => model === item.family || model.startsWith(`${item.family}-`))
export const connect = Effect.fn("Realtime.connect")(function* (input: RealtimeConnectInput) {
const resolved = engine(input.model)
if (!resolved) return yield* invalidRequest(`Unsupported voice model: ${input.model}`)
const session = { ...input.session, model: input.model }
switch (resolved.id) {
case "openai-live": {
if (input.transport.type !== "webrtc")
return yield* invalidRequest("OpenAI GPT-Live only supports WebRTC transport")
const data = yield* post(
resolved.id,
`${OPENAI_BASE_URL}/live/sessions`,
input.apiKey,
{ session, transport: input.transport },
LiveSessionResponse,
)
return { type: "webrtc" as const, sdp: data.transport.sdp }
}
case "openai-realtime": {
if (input.transport.type === "websocket")
return yield* clientSecret(resolved.id, OPENAI_BASE_URL, input.model, input.apiKey, session)
const form = new FormData()
form.append("sdp", input.transport.sdp)
form.append("session", encodeJson(session))
const query = input.transport.dcid === undefined ? "" : `?dcid=${input.transport.dcid}`
const sdp = yield* post(
resolved.id,
`${OPENAI_BASE_URL}/realtime/calls${query}`,
input.apiKey,
form,
Schema.String,
)
return { type: "webrtc" as const, sdp }
}
case "xai-realtime": {
if (input.transport.type !== "websocket")
return yield* invalidRequest("xAI Grok Voice only supports WebSocket transport")
return yield* clientSecret(resolved.id, XAI_BASE_URL, input.model, input.apiKey)
}
}
})
export const Realtime = {
engine,
connect,
} as const
const LiveSessionResponse = Schema.fromJsonString(
Schema.Struct({
transport: Schema.Struct({ sdp: Schema.String }),
}),
)
// xAI's ephemeral-token docs (https://docs.x.ai/developers/model-capabilities/audio/ephemeral-tokens)
// document the request body (`{ expires_after: { seconds } }`) but not the response payload; we assume
// it mirrors OpenAI's `{ value, expires_at }` response shape.
const ClientSecretResponse = Schema.fromJsonString(
Schema.Struct({
value: Schema.String,
expires_at: Schema.Number,
}),
)
const invalidRequest = (message: string) => new AIError({ reason: new InvalidRequestError({ message }) })
const post = <A>(
route: string,
url: string,
apiKey: Redacted.Redacted,
body: Record<string, unknown> | FormData,
response: Schema.Codec<A, string>,
) =>
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
const res = yield* executor.execute(
HttpClientRequest.post(url).pipe(
HttpClientRequest.bearerToken(apiKey),
body instanceof FormData
? HttpClientRequest.bodyFormData(body)
: HttpClientRequest.bodyText(encodeJson(body), "application/json"),
),
)
const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers })
const fail = (message: string, cause?: unknown, raw?: string) =>
new AIError({ reason: new InvalidProviderOutputError({ route, message, body: raw, http, cause }) })
const text = yield* res.text.pipe(Effect.mapError((cause) => fail(`Failed to read ${route} response`, cause)))
return yield* Schema.decodeEffect(response)(text).pipe(
Effect.mapError((cause) => fail(`${route} returned an invalid response`, cause, text)),
)
})
const clientSecret = (
route: string,
baseURL: string,
model: string,
apiKey: Redacted.Redacted,
session?: Record<string, unknown>,
) =>
post(
route,
`${baseURL}/realtime/client_secrets`,
apiKey,
{ expires_after: { seconds: 60 }, session },
ClientSecretResponse,
).pipe(
Effect.map(
(data): RealtimeTransport => ({
type: "websocket",
url: `${baseURL.replace(/^http/, "ws")}/realtime?model=${encodeURIComponent(model)}`,
token: data.value,
expiresAt: data.expires_at,
}),
),
)
+127
View File
@@ -0,0 +1,127 @@
import { describe, expect, test } from "bun:test"
import { Effect, Redacted } from "effect"
import { HttpClientRequest } from "effect/http"
import { Realtime } from "../src/experimental.js"
import { it } from "./lib/effect.js"
import { dynamicResponse } from "./lib/http.js"
describe("experimental Realtime", () => {
it.effect("connects OpenAI GPT-Live, OpenAI Realtime (WebRTC + WebSocket), and xAI Grok Voice", () =>
Effect.gen(function* () {
expect(
yield* Realtime.connect({
model: "gpt-live-1",
apiKey: Redacted.make("sk-openai"),
session: { instructions: "Be concise.", voice: "marin" },
transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 },
}),
).toEqual({ type: "webrtc", sdp: "v=0\r\no=live-answer\r\n" })
expect(
yield* Realtime.connect({
model: "gpt-realtime-2.1",
apiKey: Redacted.make("sk-openai"),
session: { type: "realtime", instructions: "Brief." },
transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 },
}),
).toEqual({ type: "webrtc", sdp: "v=0\r\no=rtc-answer\r\n" })
expect(
yield* Realtime.connect({
model: "gpt-realtime-2.1",
apiKey: Redacted.make("sk-openai"),
session: { type: "realtime", instructions: "Brief." },
transport: { type: "websocket" },
}),
).toEqual({
type: "websocket",
url: "wss://api.openai.com/v1/realtime?model=gpt-realtime-2.1",
token: "ek_1",
expiresAt: 1760000060,
})
expect(
yield* Realtime.connect({
model: "grok-voice-3.0",
apiKey: Redacted.make("xai-key"),
session: { voice: "ara" },
transport: { type: "websocket" },
}),
).toEqual({
type: "websocket",
url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0",
token: "ek_2",
expiresAt: 1760000120,
})
}).pipe(
Effect.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const req = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
if (req.url === "https://api.openai.com/v1/live/sessions") {
expect(req.headers.get("authorization")).toBe("Bearer sk-openai")
expect(JSON.parse(input.text)).toEqual({
session: { instructions: "Be concise.", voice: "marin", model: "gpt-live-1" },
transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 },
})
return input.respond(
JSON.stringify({ session: { id: "s1" }, transport: { sdp: "v=0\r\no=live-answer\r\n" } }),
)
}
if (req.url === "https://api.openai.com/v1/realtime/calls?dcid=4") {
expect(req.headers.get("authorization")).toBe("Bearer sk-openai")
const boundary = input.text.slice(2, input.text.indexOf("\r\n"))
const form = yield* Effect.promise(() =>
new Response(input.text, {
headers: { "content-type": `multipart/form-data; boundary=${boundary}` },
}).formData(),
)
expect(form.get("sdp")).toBe("v=0\r\no=offer\r\n")
expect(JSON.parse(String(form.get("session")))).toEqual({
type: "realtime",
instructions: "Brief.",
model: "gpt-realtime-2.1",
})
return input.respond("v=0\r\no=rtc-answer\r\n", { headers: { "content-type": "application/sdp" } })
}
if (req.url === "https://api.openai.com/v1/realtime/client_secrets") {
expect(req.headers.get("authorization")).toBe("Bearer sk-openai")
expect(JSON.parse(input.text)).toEqual({
expires_after: { seconds: 60 },
session: { type: "realtime", instructions: "Brief.", model: "gpt-realtime-2.1" },
})
return input.respond(JSON.stringify({ value: "ek_1", expires_at: 1760000060 }))
}
expect(req.url).toBe("https://api.x.ai/v1/realtime/client_secrets")
expect(req.headers.get("authorization")).toBe("Bearer xai-key")
expect(JSON.parse(input.text)).toEqual({ expires_after: { seconds: 60 } })
return input.respond(JSON.stringify({ value: "ek_2", expires_at: 1760000120 }))
}),
),
),
),
)
test("routes bare and versioned model names to their engine", () => {
expect(
["gpt-realtime", "gpt-realtime-2.1", "gpt-live", "grok-voice-latest"].map((model) => Realtime.engine(model)?.id),
).toEqual(["openai-realtime", "openai-realtime", "openai-live", "xai-realtime"])
expect(Realtime.engine("gpt-realtimex")).toBeUndefined()
})
it.effect("rejects unsupported transports before network I/O", () =>
Effect.gen(function* () {
const liveWs = yield* Realtime.connect({
model: "gpt-live-1",
apiKey: Redacted.make("k"),
transport: { type: "websocket" },
}).pipe(Effect.flip)
const grokRtc = yield* Realtime.connect({
model: "grok-voice-3.0",
apiKey: Redacted.make("k"),
transport: { type: "webrtc", sdp: "v=0" },
}).pipe(Effect.flip)
expect([liveWs.reason._tag, grokRtc.reason._tag]).toEqual(["InvalidRequest", "InvalidRequest"])
}).pipe(Effect.provide(dynamicResponse(() => Effect.die("unsupported transport reached the network")))),
)
})
+19
View File
@@ -1402,6 +1402,24 @@ export interface GenerateApi<E = never> {
readonly text: GenerateTextOperation<E>
}
export type VoiceSessionInput = {
readonly model: string
readonly session?: { readonly [x: string]: unknown } | undefined
readonly transport:
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined }
| { readonly type: "websocket" }
}
export type VoiceSessionOutput = {
readonly transport:
| { readonly type: "webrtc"; readonly sdp: string }
| { readonly type: "websocket"; readonly url: string; readonly token: string; readonly expiresAt: number }
}
export type VoiceSessionOperation<E = never> = (input: VoiceSessionInput) => Effect.Effect<VoiceSessionOutput, E>
export interface VoiceApi<E = never> {
readonly session: VoiceSessionOperation<E>
}
export type ProviderListInput = { readonly location?: { readonly directory?: string | undefined } | undefined }
export type ProviderListOutput = { readonly location: Location.PublicRef; readonly data: ReadonlyArray<Provider.Info> }
export type ProviderListOperation<E = never> = (input?: ProviderListInput) => Effect.Effect<ProviderListOutput, E>
@@ -2272,6 +2290,7 @@ export interface AppApi<E = never> {
readonly message: MessageApi<E>
readonly model: ModelApi<E>
readonly generate: GenerateApi<E>
readonly voice: VoiceApi<E>
readonly provider: ProviderApi<E>
readonly integration: IntegrationApi<E>
readonly mcp: McpApi<E>
@@ -115,6 +115,8 @@ import type {
ModelDefaultOutput,
GenerateTextInput,
GenerateTextOutput,
VoiceSessionInput,
VoiceSessionOutput,
ProviderListInput,
ProviderListOutput,
ProviderGetInput,
@@ -863,6 +865,18 @@ const EndpointGenerateText = (raw: RawClient["server.generate"]) => (input: Gene
const adaptGroupGenerate = (raw: RawClient["server.generate"]) => ({ text: EndpointGenerateText(raw) })
const EndpointVoiceSession = (raw: RawClient["server.voice"]) => (input: VoiceSessionInput) =>
preserveEffect<VoiceSessionOutput>()(
raw["voice.session"]({
payload: { model: input["model"], session: input["session"], transport: input["transport"] },
}).pipe(
Effect.mapError(mapClientError),
Effect.map((value) => value.data),
),
)
const adaptGroupVoice = (raw: RawClient["server.voice"]) => ({ session: EndpointVoiceSession(raw) })
const EndpointProviderList = (raw: RawClient["server.provider"]) => (input?: ProviderListInput) =>
preserveEffect<ProviderListOutput>()(
raw["provider.list"]({ query: { location: input?.["location"] } }).pipe(Effect.mapError(mapClientError)),
@@ -1616,6 +1630,7 @@ const adaptClient = (raw: RawClient) => ({
message: adaptGroupMessage(raw["server.message"]),
model: adaptGroupModel(raw["server.model"]),
generate: adaptGroupGenerate(raw["server.generate"]),
voice: adaptGroupVoice(raw["server.voice"]),
provider: adaptGroupProvider(raw["server.provider"]),
integration: adaptGroupIntegration(raw["server.integration"]),
mcp: adaptGroupMcp(raw["server.mcp"]),
@@ -109,6 +109,8 @@ import type {
ModelDefaultOutput,
GenerateTextInput,
GenerateTextOutput,
VoiceSessionInput,
VoiceSessionOutput,
ProviderListInput,
ProviderListOutput,
ProviderGetInput,
@@ -1151,6 +1153,20 @@ export function make(options: ClientOptions) {
requestOptions,
).then((value) => value.data),
},
voice: {
session: (input: VoiceSessionInput, requestOptions?: RequestOptions) =>
request<{ readonly data: VoiceSessionOutput }>(
{
method: "POST",
path: `/api/experimental/voice/session`,
body: { model: input["model"], session: input["session"], transport: input["transport"] },
successStatus: 200,
declaredStatuses: [400, 401, 404, 503],
empty: false,
},
requestOptions,
).then((value) => value.data),
},
provider: {
list: (input?: ProviderListInput, requestOptions?: RequestOptions) =>
request<ProviderListOutput>(
@@ -174,6 +174,10 @@ export type MoneyUSDPerMillionTokens = number
export type GenerateTextResponse = { data: { text: string } }
export type VoiceWebRTCTransport = { type: "webrtc"; sdp: string }
export type VoiceWebSocketTransport = { type: "websocket"; url: string; token: string; expiresAt: number }
export type IntegrationID = string
export type IntegrationMethodID = string
@@ -658,6 +662,8 @@ export type ModelCost = {
cache: { read: MoneyUSDPerMillionTokens; write: MoneyUSDPerMillionTokens }
}
export type VoiceTransport = VoiceWebRTCTransport | VoiceWebSocketTransport
export type IntegrationCommandMethod = {
id: IntegrationMethodID
type: "command"
@@ -1941,6 +1947,8 @@ export type ProviderInfo = {
body?: { [x: string]: any }
}
export type VoiceSessionResponse = { data: { transport: VoiceTransport } }
export type ConnectionInfo = ConnectionCredentialInfo | ConnectionEnvInfo
export type PtyCreated = {
@@ -5780,6 +5788,32 @@ export type GenerateTextInput = {
export type GenerateTextOutput = GenerateTextResponse["data"]
export type VoiceSessionInput = {
readonly model: {
readonly model: string
readonly session?: { readonly [x: string]: unknown } | undefined
readonly transport:
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined }
| { readonly type: "websocket" }
}["model"]
readonly session?: {
readonly model: string
readonly session?: { readonly [x: string]: unknown } | undefined
readonly transport:
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined }
| { readonly type: "websocket" }
}["session"]
readonly transport: {
readonly model: string
readonly session?: { readonly [x: string]: unknown } | undefined
readonly transport:
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined }
| { readonly type: "websocket" }
}["transport"]
}
export type VoiceSessionOutput = VoiceSessionResponse["data"]
export type ProviderListInput = {
readonly location?: { readonly location?: { readonly directory?: string | undefined } | undefined }["location"]
}
+2
View File
@@ -13,6 +13,7 @@ test("exposes every standard HTTP API group", () => {
"message",
"model",
"generate",
"voice",
"provider",
"integration",
"mcp",
@@ -45,6 +46,7 @@ test("exposes every standard HTTP API group", () => {
expect(Object.keys(client.integration.oauth)).toEqual(["connect", "status", "complete", "cancel"])
expect(Object.keys(client.integration.command)).toEqual(["connect", "status", "cancel"])
expect(Object.keys(client.websearch)).toEqual(["providers", "query"])
expect(Object.keys(client.voice)).toEqual(["session"])
expect(Object.keys(client.file)).toEqual(["read", "list", "find", "write"])
expect(Object.keys(client.vcs)).toEqual(["init", "get", "base", "status", "branch", "diff"])
expect(Object.keys(client.pty)).toEqual(["list", "create", "get", "update", "remove", "connect"])
+2
View File
@@ -13,6 +13,7 @@ import { Formatter } from "./formatter.js"
import { FileSystem } from "./filesystem.js"
import { FileSystemSearch } from "./filesystem/search.js"
import { Generate } from "./generate.js"
import { Voice } from "./voice.js"
import { Form } from "./form.js"
import { Image } from "./image.js"
import { LocationWatcher } from "./filesystem/location-watcher.js"
@@ -97,6 +98,7 @@ const nodes = [
InstructionEntry.node,
Form.node,
Generate.node,
Voice.node,
ReadToolFileSystem.node,
McpTool.node,
SessionInstructions.node,
+88
View File
@@ -0,0 +1,88 @@
export * as Voice from "./voice.js"
import { type AIError } from "@opencode/ai"
import {
Realtime,
type RealtimeSession,
type RealtimeTransport,
type RealtimeTransportInput,
} from "@opencode/ai/experimental"
import { RequestExecutor } from "@opencode/ai/route"
import { makeLocationNode } from "@opencode/util/effect/app-node"
import { Context, Effect, Layer, Redacted, Schema } from "effect"
import { requestExecutor } from "./effect/app-node-platform.js"
import { Integration } from "./integration.js"
export interface SessionInput {
readonly model: string
readonly session?: RealtimeSession
readonly transport: RealtimeTransportInput
}
export class InvalidRequestError extends Schema.TaggedError<InvalidRequestError>()("Voice.InvalidRequestError", {
message: Schema.String,
}) {}
export class UnavailableError extends Schema.TaggedError<UnavailableError>()("Voice.UnavailableError", {
message: Schema.String,
service: Schema.optional(Schema.String),
}) {}
export type Error = InvalidRequestError | UnavailableError
export interface Interface {
readonly session: (input: SessionInput) => Effect.Effect<RealtimeTransport, Error>
}
export class Service extends Context.Service<Service, Interface>()("@opencode/Voice") {}
export const layer = Layer.effect(
Service,
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
const integrations = yield* Integration.Service
const session = Effect.fn("Voice.session")(function* (input: SessionInput) {
const engine = Realtime.engine(input.model)
if (!engine) return yield* new InvalidRequestError({ message: `Unsupported voice model: ${input.model}` })
// Integration lists the active credential first, then newer stored credentials, then its env vars.
const connection = (yield* integrations.get(Integration.ID.make(engine.provider)))?.connections.find(
(item) => item.type === "env" || item.method === "key",
)
const credential = connection
? yield* integrations.connection
.resolve(connection)
.pipe(
Effect.mapError((error) => new UnavailableError({ service: engine.provider, message: error.message })),
)
: undefined
if (credential?.type !== "key")
return yield* new UnavailableError({
service: engine.provider,
message:
engine.provider === "openai"
? "Voice needs an OpenAI API key on the server (a stored key or OPENAI_API_KEY). ChatGPT sign-in can't start voice sessions."
: "Voice needs an xAI API key on the server (a stored key or XAI_API_KEY).",
})
return yield* Realtime.connect({ ...input, apiKey: Redacted.make(credential.key) }).pipe(
Effect.provideService(RequestExecutor.Service, executor),
Effect.mapError(
(error: AIError): Error =>
error.reason._tag === "InvalidRequest" || error.reason._tag === "UnsupportedOperation"
? new InvalidRequestError({ message: error.message })
: new UnavailableError({ message: error.message, service: engine.provider }),
),
)
})
return Service.of({ session })
}),
)
export const node = makeLocationNode({
service: Service,
layer,
deps: [Integration.node, requestExecutor],
})
+107
View File
@@ -0,0 +1,107 @@
import { expect } from "bun:test"
import { RequestExecutor } from "@opencode/ai/route"
import { Credential } from "@opencode/core/credential"
import { Integration } from "@opencode/core/integration"
import { Voice } from "@opencode/core/voice"
import { Effect, Layer } from "effect"
import { HttpClient, type HttpClientRequest, HttpClientResponse } from "effect/http"
import { testEffect } from "./lib/effect"
const it = testEffect(Layer.empty)
const credential = (id: string, method: "key" | "oauth") => ({
type: "credential" as const,
id: Credential.ID.make(id),
label: id,
method,
})
const keys: Record<string, string> = { active: "sk-active", stored: "sk-stored", XAI_API_KEY: "xai-env-key" }
it.effect("uses the first key or env connection Integration lists and never a ChatGPT sign-in", () =>
Effect.gen(function* () {
const auths: string[] = []
const call = (model: string, connections: Integration.Info["connections"]) =>
Effect.flatMap(Voice.Service, (voice) =>
voice.session({
model,
transport: model.startsWith("grok-") ? { type: "websocket" } : { type: "webrtc", sdp: "v=0" },
}),
).pipe(
Effect.provide(
Voice.layer.pipe(
Layer.provide(
Layer.merge(
Layer.mock(Integration.Service, {
revision: () => 0,
get: (id) => Effect.succeed({ id, name: id, methods: [], connections }),
connection: {
active: () => Effect.die("unused"),
resolve: (connection) =>
connection.type === "credential" && connection.method === "oauth"
? Effect.die("a ChatGPT sign-in was resolved")
: Effect.succeed(
Credential.Key.make({
type: "key",
key: keys[connection.type === "env" ? connection.name : connection.id],
}),
),
key: () => Effect.die("unused"),
external: () => Effect.die("unused"),
activate: () => Effect.die("unused"),
update: () => Effect.die("unused"),
remove: () => Effect.die("unused"),
status: () => Effect.die("unused"),
},
oauth: {
connect: () => Effect.die("unused"),
status: () => Effect.die("unused"),
complete: () => Effect.die("unused"),
cancel: () => Effect.die("unused"),
},
command: {
connect: () => Effect.die("unused"),
status: () => Effect.die("unused"),
cancel: () => Effect.die("unused"),
},
}),
RequestExecutor.layer.pipe(
Layer.provide(
Layer.succeed(
HttpClient.HttpClient,
HttpClient.make((req: HttpClientRequest.HttpClientRequest) =>
Effect.sync(() => {
auths.push(req.headers.authorization ?? "")
return HttpClientResponse.fromWeb(
req,
model.startsWith("grok-")
? Response.json({ value: "ek_xai", expires_at: 1760000999 })
: Response.json({ transport: { sdp: "v=0\r\no=answer\r\n" } }),
)
}),
),
),
),
),
),
),
),
),
)
yield* call("gpt-live-1", [credential("active", "key"), credential("stored", "key")])
yield* call("gpt-live-1", [credential("chatgpt", "oauth"), credential("stored", "key")])
const grok = yield* call("grok-voice-3.0", [{ type: "env", name: "XAI_API_KEY" }])
expect(auths).toEqual(["Bearer sk-active", "Bearer sk-stored", "Bearer xai-env-key"])
expect(grok).toEqual({
type: "websocket",
url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0",
token: "ek_xai",
expiresAt: 1760000999,
})
const signInOnly = yield* call("gpt-live-1", [credential("chatgpt", "oauth")]).pipe(Effect.flip)
expect(signInOnly._tag).toBe("Voice.UnavailableError")
expect(signInOnly.message).toContain("OpenAI API key")
}),
)
+218
View File
@@ -5357,6 +5357,98 @@
}
}
},
"/api/experimental/voice/session": {
"post": {
"tags": ["voice"],
"operationId": "experimental.voice.session",
"parameters": [],
"security": [],
"responses": {
"200": {
"description": "VoiceSessionResponse",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/VoiceSessionResponse"
}
}
}
},
"400": {
"description": "InvalidRequestError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/InvalidRequestErrorEncoded"
}
}
}
},
"401": {
"description": "UnauthorizedError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UnauthorizedErrorEncoded"
}
}
}
},
"404": {
"description": "LocationNotFoundError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/LocationNotFoundErrorEncoded"
}
}
}
},
"503": {
"description": "ServiceUnavailableError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ServiceUnavailableErrorEncoded"
}
}
}
}
},
"description": "Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.",
"summary": "Create voice session",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"model": {
"type": "string"
},
"session": {
"anyOf": [
{
"type": "object"
},
{
"type": "null"
}
]
},
"transport": {
"$ref": "#/components/schemas/VoiceTransportInput"
}
},
"required": ["model", "transport"],
"additionalProperties": false
}
}
},
"required": true
}
}
},
"/api/provider": {
"get": {
"tags": ["provider"],
@@ -20330,6 +20422,128 @@
"required": ["_tag", "providerID", "message"],
"additionalProperties": false
},
"VoiceSessionResponse": {
"type": "object",
"properties": {
"data": {
"type": "object",
"properties": {
"transport": {
"$ref": "#/components/schemas/VoiceTransport"
}
},
"required": ["transport"],
"additionalProperties": false
}
},
"required": ["data"],
"additionalProperties": false
},
"VoiceTransport": {
"anyOf": [
{
"$ref": "#/components/schemas/VoiceWebRTCTransport"
},
{
"$ref": "#/components/schemas/VoiceWebSocketTransport"
}
]
},
"VoiceTransportInput": {
"anyOf": [
{
"$ref": "#/components/schemas/VoiceWebRTCTransportInput"
},
{
"$ref": "#/components/schemas/VoiceWebSocketTransportInput"
}
]
},
"VoiceWebRTCTransport": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["webrtc"]
},
"sdp": {
"type": "string"
}
},
"required": ["type", "sdp"],
"additionalProperties": false
},
"VoiceWebRTCTransportInput": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["webrtc"]
},
"sdp": {
"type": "string"
},
"dcid": {
"anyOf": [
{
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"enum": ["Infinity", "-Infinity", "NaN"]
}
]
},
{
"type": "null"
}
]
}
},
"required": ["type", "sdp"],
"additionalProperties": false
},
"VoiceWebSocketTransport": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["websocket"]
},
"url": {
"type": "string"
},
"token": {
"type": "string"
},
"expiresAt": {
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"enum": ["Infinity", "-Infinity", "NaN"]
}
]
}
},
"required": ["type", "url", "token", "expiresAt"],
"additionalProperties": false
},
"VoiceWebSocketTransportInput": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["websocket"]
}
},
"required": ["type"],
"additionalProperties": false
},
"WebSearch.ID": {
"type": "string"
},
@@ -20523,6 +20737,10 @@
"name": "generate",
"description": "Experimental one-shot generation routes."
},
{
"name": "voice",
"description": "Experimental server-brokered voice session routes."
},
{
"name": "provider",
"description": "Experimental provider routes."
+3
View File
@@ -2,6 +2,7 @@ import { Context } from "effect"
import { HttpApi, HttpApiGroup, HttpApiMiddleware, OpenApi } from "effect/http-api"
import { SchemaErrorMiddleware } from "./middleware/schema-error.js"
import { GenerateGroup } from "./groups/generate.js"
import { VoiceGroup } from "./groups/voice.js"
import { MessageGroup } from "./groups/message.js"
import { ModelGroup } from "./groups/model.js"
import { ProviderGroup } from "./groups/provider.js"
@@ -94,6 +95,7 @@ type ApiGroups<
| typeof MigrationGroup
| typeof WorktreeGroup
| typeof GenerateGroup
| typeof VoiceGroup
| typeof PersistentPtyGroup
| typeof CredentialGroup
| LocationGroups<LocationId>
@@ -165,6 +167,7 @@ const makeApiFromGroup = <
.add(MessageGroup)
.add(ModelGroup.middleware(locationMiddleware))
.add(GenerateGroup)
.add(VoiceGroup)
.add(ProviderGroup.middleware(locationMiddleware))
.add(IntegrationGroup.middleware(locationMiddleware))
.add(McpGroup.middleware(locationMiddleware))
+1
View File
@@ -44,6 +44,7 @@ export const groupNames = {
"server.message": "message",
"server.model": "model",
"server.generate": "generate",
"server.voice": "voice",
"server.provider": "provider",
"server.integration": "integration",
"server.websearch": "websearch",
+65
View File
@@ -0,0 +1,65 @@
import { Schema } from "effect"
import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/http-api"
import { InvalidRequestError, LocationNotFoundError, ServiceUnavailableError } from "../errors.js"
export const VoiceWebRTCTransportInput = Schema.Struct({
type: Schema.Literal("webrtc"),
sdp: Schema.String,
dcid: Schema.optional(Schema.Number),
}).annotate({ identifier: "VoiceWebRTCTransportInput" })
export const VoiceWebSocketTransportInput = Schema.Struct({
type: Schema.Literal("websocket"),
}).annotate({ identifier: "VoiceWebSocketTransportInput" })
export const VoiceTransportInput = Schema.Union([
VoiceWebRTCTransportInput,
VoiceWebSocketTransportInput,
]).annotate({ identifier: "VoiceTransportInput" })
export const VoiceWebRTCTransport = Schema.Struct({
type: Schema.Literal("webrtc"),
sdp: Schema.String,
}).annotate({ identifier: "VoiceWebRTCTransport" })
export const VoiceWebSocketTransport = Schema.Struct({
type: Schema.Literal("websocket"),
url: Schema.String,
token: Schema.String,
expiresAt: Schema.Number,
}).annotate({ identifier: "VoiceWebSocketTransport" })
export const VoiceTransport = Schema.Union([
VoiceWebRTCTransport,
VoiceWebSocketTransport,
]).annotate({ identifier: "VoiceTransport" })
export const VoiceGroup = HttpApiGroup.make("server.voice")
.add(
HttpApiEndpoint.post("voice.session", "/api/experimental/voice/session", {
payload: Schema.Struct({
model: Schema.String,
session: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
transport: VoiceTransportInput,
}),
success: Schema.Struct({
data: Schema.Struct({
transport: VoiceTransport,
}),
}).annotate({ identifier: "VoiceSessionResponse" }),
error: [InvalidRequestError, ServiceUnavailableError, LocationNotFoundError],
}).annotateMerge(
OpenApi.annotations({
identifier: "experimental.voice.session",
summary: "Create voice session",
description:
"Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.",
}),
),
)
.annotateMerge(
OpenApi.annotations({
title: "voice",
description: "Experimental server-brokered voice session routes.",
}),
)
+2
View File
@@ -1,5 +1,6 @@
import { Layer } from "effect"
import { GenerateHandler } from "./handlers/generate"
import { VoiceHandler } from "./handlers/voice"
import { MessageHandler } from "./handlers/message"
import { ModelHandler } from "./handlers/model"
import { ProviderHandler } from "./handlers/provider"
@@ -42,6 +43,7 @@ export const handlers = Layer.mergeAll(
MessageHandler,
ModelHandler,
GenerateHandler,
VoiceHandler,
ProviderHandler,
IntegrationHandler,
WebSearchHandler,
+39
View File
@@ -0,0 +1,39 @@
import { Location } from "@opencode/core/location"
import { LocationServiceMap } from "@opencode/core/location-services"
import { AbsolutePath } from "@opencode/core/schema"
import { Voice } from "@opencode/core/voice"
import { InvalidRequestError, ServiceUnavailableError } from "@opencode/protocol/errors"
import { Global } from "@opencode/util/global"
import { Effect } from "effect"
import { HttpApiBuilder } from "effect/http-api"
import { Api } from "../api"
import { awaitPlugins, locationErrors } from "../location"
export const VoiceHandler = HttpApiBuilder.group(Api, "server.voice", (handlers) =>
Effect.gen(function* () {
const global = yield* Global.Service
const locations = yield* LocationServiceMap.Service
const services = locations.get(Location.Ref.make({ directory: AbsolutePath.make(global.config) }))
return handlers.handle(
"voice.session",
Effect.fn("server.voice.session")(
function* (request) {
yield* awaitPlugins
const voice = yield* Voice.Service
const transport = yield* voice
.session(request.payload)
.pipe(
Effect.mapError((error) =>
error._tag === "Voice.InvalidRequestError"
? new InvalidRequestError({ message: error.message })
: new ServiceUnavailableError({ message: error.message, service: error.service }),
),
)
return { data: { transport } }
},
Effect.provide(services),
locationErrors,
),
)
}),
)
+88
View File
@@ -0,0 +1,88 @@
import fs from "node:fs/promises"
import path from "node:path"
import { expect } from "bun:test"
import { Voice } from "@opencode/core/voice"
import { makeLocationNode } from "@opencode/util/effect/app-node"
import { Effect, Layer } from "effect"
import { tmpdir } from "../../core/test/fixture/tmpdir"
import { it } from "../../core/test/lib/effect"
import { ServerFetch } from "../src/fetch"
const voice = makeLocationNode({
service: Voice.Service,
layer: Layer.succeed(
Voice.Service,
Voice.Service.of({
session: (input) => {
if (input.model === "invalid")
return Effect.fail(new Voice.InvalidRequestError({ message: "Unsupported voice model" }))
if (input.model === "unavailable")
return Effect.fail(new Voice.UnavailableError({ message: "OpenAI API key is required", service: "openai" }))
return Effect.succeed(
input.transport.type === "webrtc"
? { type: "webrtc" as const, sdp: `answer:${input.model}:${input.transport.sdp}` }
: {
type: "websocket" as const,
url: `wss://api.x.ai/v1/realtime?model=${input.model}`,
token: "ek_test",
expiresAt: 1760000000,
},
)
},
}),
),
deps: [],
})
it.live("routes POST /api/experimental/voice/session and maps errors", () =>
Effect.gen(function* () {
const tmp = yield* Effect.acquireDisposable(Effect.promise(() => tmpdir("opencode-voice-endpoint-")))
const global = path.join(tmp.path, "global")
yield* Effect.promise(() => fs.mkdir(global))
const handler = yield* ServerFetch.make(
{
database: { path: ":memory:" },
config: { directory: global },
fs: { filewatcher: false },
models: { fetch: false },
},
{ overrides: [Voice.node.replace(voice)] },
)
const post = (body: unknown) =>
Effect.promise(() =>
handler(
new Request("http://opencode.local/api/experimental/voice/session", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(body),
}),
),
)
const rtcRes = yield* post({
model: "gpt-live-1",
session: { voice: "marin" },
transport: { type: "webrtc", sdp: "v=0", dcid: 4 },
})
expect(rtcRes.status).toBe(200)
expect(yield* Effect.promise(() => rtcRes.json())).toEqual({
data: { transport: { type: "webrtc", sdp: "answer:gpt-live-1:v=0" } },
})
const wsRes = yield* post({ model: "grok-voice-3.0", transport: { type: "websocket" } })
expect(wsRes.status).toBe(200)
expect(yield* Effect.promise(() => wsRes.json())).toEqual({
data: {
transport: {
type: "websocket",
url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0",
token: "ek_test",
expiresAt: 1760000000,
},
},
})
expect((yield* post({ model: "invalid", transport: { type: "websocket" } })).status).toBe(400)
expect((yield* post({ model: "unavailable", transport: { type: "websocket" } })).status).toBe(503)
}),
)
+218
View File
@@ -5357,6 +5357,98 @@
}
}
},
"/api/experimental/voice/session": {
"post": {
"tags": ["voice"],
"operationId": "experimental.voice.session",
"parameters": [],
"security": [],
"responses": {
"200": {
"description": "VoiceSessionResponse",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/VoiceSessionResponse"
}
}
}
},
"400": {
"description": "InvalidRequestError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/InvalidRequestErrorEncoded"
}
}
}
},
"401": {
"description": "UnauthorizedError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UnauthorizedErrorEncoded"
}
}
}
},
"404": {
"description": "LocationNotFoundError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/LocationNotFoundErrorEncoded"
}
}
}
},
"503": {
"description": "ServiceUnavailableError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ServiceUnavailableErrorEncoded"
}
}
}
}
},
"description": "Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.",
"summary": "Create voice session",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"model": {
"type": "string"
},
"session": {
"anyOf": [
{
"type": "object"
},
{
"type": "null"
}
]
},
"transport": {
"$ref": "#/components/schemas/VoiceTransportInput"
}
},
"required": ["model", "transport"],
"additionalProperties": false
}
}
},
"required": true
}
}
},
"/api/provider": {
"get": {
"tags": ["provider"],
@@ -20330,6 +20422,128 @@
"required": ["_tag", "providerID", "message"],
"additionalProperties": false
},
"VoiceSessionResponse": {
"type": "object",
"properties": {
"data": {
"type": "object",
"properties": {
"transport": {
"$ref": "#/components/schemas/VoiceTransport"
}
},
"required": ["transport"],
"additionalProperties": false
}
},
"required": ["data"],
"additionalProperties": false
},
"VoiceTransport": {
"anyOf": [
{
"$ref": "#/components/schemas/VoiceWebRTCTransport"
},
{
"$ref": "#/components/schemas/VoiceWebSocketTransport"
}
]
},
"VoiceTransportInput": {
"anyOf": [
{
"$ref": "#/components/schemas/VoiceWebRTCTransportInput"
},
{
"$ref": "#/components/schemas/VoiceWebSocketTransportInput"
}
]
},
"VoiceWebRTCTransport": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["webrtc"]
},
"sdp": {
"type": "string"
}
},
"required": ["type", "sdp"],
"additionalProperties": false
},
"VoiceWebRTCTransportInput": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["webrtc"]
},
"sdp": {
"type": "string"
},
"dcid": {
"anyOf": [
{
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"enum": ["Infinity", "-Infinity", "NaN"]
}
]
},
{
"type": "null"
}
]
}
},
"required": ["type", "sdp"],
"additionalProperties": false
},
"VoiceWebSocketTransport": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["websocket"]
},
"url": {
"type": "string"
},
"token": {
"type": "string"
},
"expiresAt": {
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"enum": ["Infinity", "-Infinity", "NaN"]
}
]
}
},
"required": ["type", "url", "token", "expiresAt"],
"additionalProperties": false
},
"VoiceWebSocketTransportInput": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["websocket"]
}
},
"required": ["type"],
"additionalProperties": false
},
"WebSearch.ID": {
"type": "string"
},
@@ -20523,6 +20737,10 @@
"name": "generate",
"description": "Experimental one-shot generation routes."
},
{
"name": "voice",
"description": "Experimental server-brokered voice session routes."
},
{
"name": "provider",
"description": "Experimental provider routes."
+218
View File
@@ -5357,6 +5357,98 @@
}
}
},
"/api/experimental/voice/session": {
"post": {
"tags": ["voice"],
"operationId": "experimental.voice.session",
"parameters": [],
"security": [],
"responses": {
"200": {
"description": "VoiceSessionResponse",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/VoiceSessionResponse"
}
}
}
},
"400": {
"description": "InvalidRequestError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/InvalidRequestErrorEncoded"
}
}
}
},
"401": {
"description": "UnauthorizedError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UnauthorizedErrorEncoded"
}
}
}
},
"404": {
"description": "LocationNotFoundError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/LocationNotFoundErrorEncoded"
}
}
}
},
"503": {
"description": "ServiceUnavailableError",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ServiceUnavailableErrorEncoded"
}
}
}
}
},
"description": "Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.",
"summary": "Create voice session",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"model": {
"type": "string"
},
"session": {
"anyOf": [
{
"type": "object"
},
{
"type": "null"
}
]
},
"transport": {
"$ref": "#/components/schemas/VoiceTransportInput"
}
},
"required": ["model", "transport"],
"additionalProperties": false
}
}
},
"required": true
}
}
},
"/api/provider": {
"get": {
"tags": ["provider"],
@@ -20330,6 +20422,128 @@
"required": ["_tag", "providerID", "message"],
"additionalProperties": false
},
"VoiceSessionResponse": {
"type": "object",
"properties": {
"data": {
"type": "object",
"properties": {
"transport": {
"$ref": "#/components/schemas/VoiceTransport"
}
},
"required": ["transport"],
"additionalProperties": false
}
},
"required": ["data"],
"additionalProperties": false
},
"VoiceTransport": {
"anyOf": [
{
"$ref": "#/components/schemas/VoiceWebRTCTransport"
},
{
"$ref": "#/components/schemas/VoiceWebSocketTransport"
}
]
},
"VoiceTransportInput": {
"anyOf": [
{
"$ref": "#/components/schemas/VoiceWebRTCTransportInput"
},
{
"$ref": "#/components/schemas/VoiceWebSocketTransportInput"
}
]
},
"VoiceWebRTCTransport": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["webrtc"]
},
"sdp": {
"type": "string"
}
},
"required": ["type", "sdp"],
"additionalProperties": false
},
"VoiceWebRTCTransportInput": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["webrtc"]
},
"sdp": {
"type": "string"
},
"dcid": {
"anyOf": [
{
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"enum": ["Infinity", "-Infinity", "NaN"]
}
]
},
{
"type": "null"
}
]
}
},
"required": ["type", "sdp"],
"additionalProperties": false
},
"VoiceWebSocketTransport": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["websocket"]
},
"url": {
"type": "string"
},
"token": {
"type": "string"
},
"expiresAt": {
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"enum": ["Infinity", "-Infinity", "NaN"]
}
]
}
},
"required": ["type", "url", "token", "expiresAt"],
"additionalProperties": false
},
"VoiceWebSocketTransportInput": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["websocket"]
}
},
"required": ["type"],
"additionalProperties": false
},
"WebSearch.ID": {
"type": "string"
},
@@ -20523,6 +20737,10 @@
"name": "generate",
"description": "Experimental one-shot generation routes."
},
{
"name": "voice",
"description": "Experimental server-brokered voice session routes."
},
{
"name": "provider",
"description": "Experimental provider routes."