mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-19 17:01:37 +03:00
feat(server): add experimental voice session endpoint (#55951)
This commit is contained in:
committed by
GitHub
Unverified
parent
180fd9d622
commit
d4a6223d01
@@ -27,3 +27,10 @@ export type {
|
||||
EvaluationResponseFor,
|
||||
EvaluationRoute,
|
||||
} from "./experimental/evaluation.js"
|
||||
export { Realtime } from "./experimental/realtime.js"
|
||||
export type {
|
||||
RealtimeConnectInput,
|
||||
RealtimeSession,
|
||||
RealtimeTransport,
|
||||
RealtimeTransportInput,
|
||||
} from "./experimental/realtime.js"
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
import { Effect, type Redacted, Schema } from "effect"
|
||||
import { HttpClientRequest } from "effect/http"
|
||||
import { RequestExecutor } from "../route/executor.js"
|
||||
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "../schema/index.js"
|
||||
import { encodeJson } from "../utils/json.js"
|
||||
|
||||
const OPENAI_BASE_URL = "https://api.openai.com/v1"
|
||||
const XAI_BASE_URL = "https://api.x.ai/v1"
|
||||
|
||||
export type RealtimeTransportInput =
|
||||
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number }
|
||||
| { readonly type: "websocket" }
|
||||
|
||||
export type RealtimeTransport =
|
||||
| { readonly type: "webrtc"; readonly sdp: string }
|
||||
| { readonly type: "websocket"; readonly url: string; readonly token: string; readonly expiresAt: number }
|
||||
|
||||
export type RealtimeSession = Readonly<Record<string, unknown>>
|
||||
|
||||
export interface RealtimeConnectInput {
|
||||
readonly model: string
|
||||
readonly apiKey: Redacted.Redacted
|
||||
readonly session?: RealtimeSession
|
||||
readonly transport: RealtimeTransportInput
|
||||
}
|
||||
|
||||
const engines = [
|
||||
{ family: "gpt-live", id: "openai-live", provider: "openai" },
|
||||
{ family: "gpt-realtime", id: "openai-realtime", provider: "openai" },
|
||||
{ family: "grok-voice", id: "xai-realtime", provider: "xai" },
|
||||
] as const
|
||||
|
||||
// Matches the bare family name (`gpt-realtime`) and its versions (`gpt-realtime-2.1`, `gpt-realtime-mini`).
|
||||
export const engine = (model: string) =>
|
||||
engines.find((item) => model === item.family || model.startsWith(`${item.family}-`))
|
||||
|
||||
export const connect = Effect.fn("Realtime.connect")(function* (input: RealtimeConnectInput) {
|
||||
const resolved = engine(input.model)
|
||||
if (!resolved) return yield* invalidRequest(`Unsupported voice model: ${input.model}`)
|
||||
const session = { ...input.session, model: input.model }
|
||||
|
||||
switch (resolved.id) {
|
||||
case "openai-live": {
|
||||
if (input.transport.type !== "webrtc")
|
||||
return yield* invalidRequest("OpenAI GPT-Live only supports WebRTC transport")
|
||||
const data = yield* post(
|
||||
resolved.id,
|
||||
`${OPENAI_BASE_URL}/live/sessions`,
|
||||
input.apiKey,
|
||||
{ session, transport: input.transport },
|
||||
LiveSessionResponse,
|
||||
)
|
||||
return { type: "webrtc" as const, sdp: data.transport.sdp }
|
||||
}
|
||||
case "openai-realtime": {
|
||||
if (input.transport.type === "websocket")
|
||||
return yield* clientSecret(resolved.id, OPENAI_BASE_URL, input.model, input.apiKey, session)
|
||||
const form = new FormData()
|
||||
form.append("sdp", input.transport.sdp)
|
||||
form.append("session", encodeJson(session))
|
||||
const query = input.transport.dcid === undefined ? "" : `?dcid=${input.transport.dcid}`
|
||||
const sdp = yield* post(
|
||||
resolved.id,
|
||||
`${OPENAI_BASE_URL}/realtime/calls${query}`,
|
||||
input.apiKey,
|
||||
form,
|
||||
Schema.String,
|
||||
)
|
||||
return { type: "webrtc" as const, sdp }
|
||||
}
|
||||
case "xai-realtime": {
|
||||
if (input.transport.type !== "websocket")
|
||||
return yield* invalidRequest("xAI Grok Voice only supports WebSocket transport")
|
||||
return yield* clientSecret(resolved.id, XAI_BASE_URL, input.model, input.apiKey)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
export const Realtime = {
|
||||
engine,
|
||||
connect,
|
||||
} as const
|
||||
|
||||
const LiveSessionResponse = Schema.fromJsonString(
|
||||
Schema.Struct({
|
||||
transport: Schema.Struct({ sdp: Schema.String }),
|
||||
}),
|
||||
)
|
||||
|
||||
// xAI's ephemeral-token docs (https://docs.x.ai/developers/model-capabilities/audio/ephemeral-tokens)
|
||||
// document the request body (`{ expires_after: { seconds } }`) but not the response payload; we assume
|
||||
// it mirrors OpenAI's `{ value, expires_at }` response shape.
|
||||
const ClientSecretResponse = Schema.fromJsonString(
|
||||
Schema.Struct({
|
||||
value: Schema.String,
|
||||
expires_at: Schema.Number,
|
||||
}),
|
||||
)
|
||||
|
||||
const invalidRequest = (message: string) => new AIError({ reason: new InvalidRequestError({ message }) })
|
||||
|
||||
const post = <A>(
|
||||
route: string,
|
||||
url: string,
|
||||
apiKey: Redacted.Redacted,
|
||||
body: Record<string, unknown> | FormData,
|
||||
response: Schema.Codec<A, string>,
|
||||
) =>
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
const res = yield* executor.execute(
|
||||
HttpClientRequest.post(url).pipe(
|
||||
HttpClientRequest.bearerToken(apiKey),
|
||||
body instanceof FormData
|
||||
? HttpClientRequest.bodyFormData(body)
|
||||
: HttpClientRequest.bodyText(encodeJson(body), "application/json"),
|
||||
),
|
||||
)
|
||||
const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers })
|
||||
const fail = (message: string, cause?: unknown, raw?: string) =>
|
||||
new AIError({ reason: new InvalidProviderOutputError({ route, message, body: raw, http, cause }) })
|
||||
const text = yield* res.text.pipe(Effect.mapError((cause) => fail(`Failed to read ${route} response`, cause)))
|
||||
return yield* Schema.decodeEffect(response)(text).pipe(
|
||||
Effect.mapError((cause) => fail(`${route} returned an invalid response`, cause, text)),
|
||||
)
|
||||
})
|
||||
|
||||
const clientSecret = (
|
||||
route: string,
|
||||
baseURL: string,
|
||||
model: string,
|
||||
apiKey: Redacted.Redacted,
|
||||
session?: Record<string, unknown>,
|
||||
) =>
|
||||
post(
|
||||
route,
|
||||
`${baseURL}/realtime/client_secrets`,
|
||||
apiKey,
|
||||
{ expires_after: { seconds: 60 }, session },
|
||||
ClientSecretResponse,
|
||||
).pipe(
|
||||
Effect.map(
|
||||
(data): RealtimeTransport => ({
|
||||
type: "websocket",
|
||||
url: `${baseURL.replace(/^http/, "ws")}/realtime?model=${encodeURIComponent(model)}`,
|
||||
token: data.value,
|
||||
expiresAt: data.expires_at,
|
||||
}),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,127 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Effect, Redacted } from "effect"
|
||||
import { HttpClientRequest } from "effect/http"
|
||||
import { Realtime } from "../src/experimental.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
import { dynamicResponse } from "./lib/http.js"
|
||||
|
||||
describe("experimental Realtime", () => {
|
||||
it.effect("connects OpenAI GPT-Live, OpenAI Realtime (WebRTC + WebSocket), and xAI Grok Voice", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(
|
||||
yield* Realtime.connect({
|
||||
model: "gpt-live-1",
|
||||
apiKey: Redacted.make("sk-openai"),
|
||||
session: { instructions: "Be concise.", voice: "marin" },
|
||||
transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 },
|
||||
}),
|
||||
).toEqual({ type: "webrtc", sdp: "v=0\r\no=live-answer\r\n" })
|
||||
|
||||
expect(
|
||||
yield* Realtime.connect({
|
||||
model: "gpt-realtime-2.1",
|
||||
apiKey: Redacted.make("sk-openai"),
|
||||
session: { type: "realtime", instructions: "Brief." },
|
||||
transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 },
|
||||
}),
|
||||
).toEqual({ type: "webrtc", sdp: "v=0\r\no=rtc-answer\r\n" })
|
||||
|
||||
expect(
|
||||
yield* Realtime.connect({
|
||||
model: "gpt-realtime-2.1",
|
||||
apiKey: Redacted.make("sk-openai"),
|
||||
session: { type: "realtime", instructions: "Brief." },
|
||||
transport: { type: "websocket" },
|
||||
}),
|
||||
).toEqual({
|
||||
type: "websocket",
|
||||
url: "wss://api.openai.com/v1/realtime?model=gpt-realtime-2.1",
|
||||
token: "ek_1",
|
||||
expiresAt: 1760000060,
|
||||
})
|
||||
|
||||
expect(
|
||||
yield* Realtime.connect({
|
||||
model: "grok-voice-3.0",
|
||||
apiKey: Redacted.make("xai-key"),
|
||||
session: { voice: "ara" },
|
||||
transport: { type: "websocket" },
|
||||
}),
|
||||
).toEqual({
|
||||
type: "websocket",
|
||||
url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0",
|
||||
token: "ek_2",
|
||||
expiresAt: 1760000120,
|
||||
})
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const req = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
if (req.url === "https://api.openai.com/v1/live/sessions") {
|
||||
expect(req.headers.get("authorization")).toBe("Bearer sk-openai")
|
||||
expect(JSON.parse(input.text)).toEqual({
|
||||
session: { instructions: "Be concise.", voice: "marin", model: "gpt-live-1" },
|
||||
transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 },
|
||||
})
|
||||
return input.respond(
|
||||
JSON.stringify({ session: { id: "s1" }, transport: { sdp: "v=0\r\no=live-answer\r\n" } }),
|
||||
)
|
||||
}
|
||||
if (req.url === "https://api.openai.com/v1/realtime/calls?dcid=4") {
|
||||
expect(req.headers.get("authorization")).toBe("Bearer sk-openai")
|
||||
const boundary = input.text.slice(2, input.text.indexOf("\r\n"))
|
||||
const form = yield* Effect.promise(() =>
|
||||
new Response(input.text, {
|
||||
headers: { "content-type": `multipart/form-data; boundary=${boundary}` },
|
||||
}).formData(),
|
||||
)
|
||||
expect(form.get("sdp")).toBe("v=0\r\no=offer\r\n")
|
||||
expect(JSON.parse(String(form.get("session")))).toEqual({
|
||||
type: "realtime",
|
||||
instructions: "Brief.",
|
||||
model: "gpt-realtime-2.1",
|
||||
})
|
||||
return input.respond("v=0\r\no=rtc-answer\r\n", { headers: { "content-type": "application/sdp" } })
|
||||
}
|
||||
if (req.url === "https://api.openai.com/v1/realtime/client_secrets") {
|
||||
expect(req.headers.get("authorization")).toBe("Bearer sk-openai")
|
||||
expect(JSON.parse(input.text)).toEqual({
|
||||
expires_after: { seconds: 60 },
|
||||
session: { type: "realtime", instructions: "Brief.", model: "gpt-realtime-2.1" },
|
||||
})
|
||||
return input.respond(JSON.stringify({ value: "ek_1", expires_at: 1760000060 }))
|
||||
}
|
||||
expect(req.url).toBe("https://api.x.ai/v1/realtime/client_secrets")
|
||||
expect(req.headers.get("authorization")).toBe("Bearer xai-key")
|
||||
expect(JSON.parse(input.text)).toEqual({ expires_after: { seconds: 60 } })
|
||||
return input.respond(JSON.stringify({ value: "ek_2", expires_at: 1760000120 }))
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
test("routes bare and versioned model names to their engine", () => {
|
||||
expect(
|
||||
["gpt-realtime", "gpt-realtime-2.1", "gpt-live", "grok-voice-latest"].map((model) => Realtime.engine(model)?.id),
|
||||
).toEqual(["openai-realtime", "openai-realtime", "openai-live", "xai-realtime"])
|
||||
expect(Realtime.engine("gpt-realtimex")).toBeUndefined()
|
||||
})
|
||||
|
||||
it.effect("rejects unsupported transports before network I/O", () =>
|
||||
Effect.gen(function* () {
|
||||
const liveWs = yield* Realtime.connect({
|
||||
model: "gpt-live-1",
|
||||
apiKey: Redacted.make("k"),
|
||||
transport: { type: "websocket" },
|
||||
}).pipe(Effect.flip)
|
||||
const grokRtc = yield* Realtime.connect({
|
||||
model: "grok-voice-3.0",
|
||||
apiKey: Redacted.make("k"),
|
||||
transport: { type: "webrtc", sdp: "v=0" },
|
||||
}).pipe(Effect.flip)
|
||||
expect([liveWs.reason._tag, grokRtc.reason._tag]).toEqual(["InvalidRequest", "InvalidRequest"])
|
||||
}).pipe(Effect.provide(dynamicResponse(() => Effect.die("unsupported transport reached the network")))),
|
||||
)
|
||||
})
|
||||
@@ -1402,6 +1402,24 @@ export interface GenerateApi<E = never> {
|
||||
readonly text: GenerateTextOperation<E>
|
||||
}
|
||||
|
||||
export type VoiceSessionInput = {
|
||||
readonly model: string
|
||||
readonly session?: { readonly [x: string]: unknown } | undefined
|
||||
readonly transport:
|
||||
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined }
|
||||
| { readonly type: "websocket" }
|
||||
}
|
||||
export type VoiceSessionOutput = {
|
||||
readonly transport:
|
||||
| { readonly type: "webrtc"; readonly sdp: string }
|
||||
| { readonly type: "websocket"; readonly url: string; readonly token: string; readonly expiresAt: number }
|
||||
}
|
||||
export type VoiceSessionOperation<E = never> = (input: VoiceSessionInput) => Effect.Effect<VoiceSessionOutput, E>
|
||||
|
||||
export interface VoiceApi<E = never> {
|
||||
readonly session: VoiceSessionOperation<E>
|
||||
}
|
||||
|
||||
export type ProviderListInput = { readonly location?: { readonly directory?: string | undefined } | undefined }
|
||||
export type ProviderListOutput = { readonly location: Location.PublicRef; readonly data: ReadonlyArray<Provider.Info> }
|
||||
export type ProviderListOperation<E = never> = (input?: ProviderListInput) => Effect.Effect<ProviderListOutput, E>
|
||||
@@ -2272,6 +2290,7 @@ export interface AppApi<E = never> {
|
||||
readonly message: MessageApi<E>
|
||||
readonly model: ModelApi<E>
|
||||
readonly generate: GenerateApi<E>
|
||||
readonly voice: VoiceApi<E>
|
||||
readonly provider: ProviderApi<E>
|
||||
readonly integration: IntegrationApi<E>
|
||||
readonly mcp: McpApi<E>
|
||||
|
||||
@@ -115,6 +115,8 @@ import type {
|
||||
ModelDefaultOutput,
|
||||
GenerateTextInput,
|
||||
GenerateTextOutput,
|
||||
VoiceSessionInput,
|
||||
VoiceSessionOutput,
|
||||
ProviderListInput,
|
||||
ProviderListOutput,
|
||||
ProviderGetInput,
|
||||
@@ -863,6 +865,18 @@ const EndpointGenerateText = (raw: RawClient["server.generate"]) => (input: Gene
|
||||
|
||||
const adaptGroupGenerate = (raw: RawClient["server.generate"]) => ({ text: EndpointGenerateText(raw) })
|
||||
|
||||
const EndpointVoiceSession = (raw: RawClient["server.voice"]) => (input: VoiceSessionInput) =>
|
||||
preserveEffect<VoiceSessionOutput>()(
|
||||
raw["voice.session"]({
|
||||
payload: { model: input["model"], session: input["session"], transport: input["transport"] },
|
||||
}).pipe(
|
||||
Effect.mapError(mapClientError),
|
||||
Effect.map((value) => value.data),
|
||||
),
|
||||
)
|
||||
|
||||
const adaptGroupVoice = (raw: RawClient["server.voice"]) => ({ session: EndpointVoiceSession(raw) })
|
||||
|
||||
const EndpointProviderList = (raw: RawClient["server.provider"]) => (input?: ProviderListInput) =>
|
||||
preserveEffect<ProviderListOutput>()(
|
||||
raw["provider.list"]({ query: { location: input?.["location"] } }).pipe(Effect.mapError(mapClientError)),
|
||||
@@ -1616,6 +1630,7 @@ const adaptClient = (raw: RawClient) => ({
|
||||
message: adaptGroupMessage(raw["server.message"]),
|
||||
model: adaptGroupModel(raw["server.model"]),
|
||||
generate: adaptGroupGenerate(raw["server.generate"]),
|
||||
voice: adaptGroupVoice(raw["server.voice"]),
|
||||
provider: adaptGroupProvider(raw["server.provider"]),
|
||||
integration: adaptGroupIntegration(raw["server.integration"]),
|
||||
mcp: adaptGroupMcp(raw["server.mcp"]),
|
||||
|
||||
@@ -109,6 +109,8 @@ import type {
|
||||
ModelDefaultOutput,
|
||||
GenerateTextInput,
|
||||
GenerateTextOutput,
|
||||
VoiceSessionInput,
|
||||
VoiceSessionOutput,
|
||||
ProviderListInput,
|
||||
ProviderListOutput,
|
||||
ProviderGetInput,
|
||||
@@ -1151,6 +1153,20 @@ export function make(options: ClientOptions) {
|
||||
requestOptions,
|
||||
).then((value) => value.data),
|
||||
},
|
||||
voice: {
|
||||
session: (input: VoiceSessionInput, requestOptions?: RequestOptions) =>
|
||||
request<{ readonly data: VoiceSessionOutput }>(
|
||||
{
|
||||
method: "POST",
|
||||
path: `/api/experimental/voice/session`,
|
||||
body: { model: input["model"], session: input["session"], transport: input["transport"] },
|
||||
successStatus: 200,
|
||||
declaredStatuses: [400, 401, 404, 503],
|
||||
empty: false,
|
||||
},
|
||||
requestOptions,
|
||||
).then((value) => value.data),
|
||||
},
|
||||
provider: {
|
||||
list: (input?: ProviderListInput, requestOptions?: RequestOptions) =>
|
||||
request<ProviderListOutput>(
|
||||
|
||||
@@ -174,6 +174,10 @@ export type MoneyUSDPerMillionTokens = number
|
||||
|
||||
export type GenerateTextResponse = { data: { text: string } }
|
||||
|
||||
export type VoiceWebRTCTransport = { type: "webrtc"; sdp: string }
|
||||
|
||||
export type VoiceWebSocketTransport = { type: "websocket"; url: string; token: string; expiresAt: number }
|
||||
|
||||
export type IntegrationID = string
|
||||
|
||||
export type IntegrationMethodID = string
|
||||
@@ -658,6 +662,8 @@ export type ModelCost = {
|
||||
cache: { read: MoneyUSDPerMillionTokens; write: MoneyUSDPerMillionTokens }
|
||||
}
|
||||
|
||||
export type VoiceTransport = VoiceWebRTCTransport | VoiceWebSocketTransport
|
||||
|
||||
export type IntegrationCommandMethod = {
|
||||
id: IntegrationMethodID
|
||||
type: "command"
|
||||
@@ -1941,6 +1947,8 @@ export type ProviderInfo = {
|
||||
body?: { [x: string]: any }
|
||||
}
|
||||
|
||||
export type VoiceSessionResponse = { data: { transport: VoiceTransport } }
|
||||
|
||||
export type ConnectionInfo = ConnectionCredentialInfo | ConnectionEnvInfo
|
||||
|
||||
export type PtyCreated = {
|
||||
@@ -5780,6 +5788,32 @@ export type GenerateTextInput = {
|
||||
|
||||
export type GenerateTextOutput = GenerateTextResponse["data"]
|
||||
|
||||
export type VoiceSessionInput = {
|
||||
readonly model: {
|
||||
readonly model: string
|
||||
readonly session?: { readonly [x: string]: unknown } | undefined
|
||||
readonly transport:
|
||||
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined }
|
||||
| { readonly type: "websocket" }
|
||||
}["model"]
|
||||
readonly session?: {
|
||||
readonly model: string
|
||||
readonly session?: { readonly [x: string]: unknown } | undefined
|
||||
readonly transport:
|
||||
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined }
|
||||
| { readonly type: "websocket" }
|
||||
}["session"]
|
||||
readonly transport: {
|
||||
readonly model: string
|
||||
readonly session?: { readonly [x: string]: unknown } | undefined
|
||||
readonly transport:
|
||||
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined }
|
||||
| { readonly type: "websocket" }
|
||||
}["transport"]
|
||||
}
|
||||
|
||||
export type VoiceSessionOutput = VoiceSessionResponse["data"]
|
||||
|
||||
export type ProviderListInput = {
|
||||
readonly location?: { readonly location?: { readonly directory?: string | undefined } | undefined }["location"]
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ test("exposes every standard HTTP API group", () => {
|
||||
"message",
|
||||
"model",
|
||||
"generate",
|
||||
"voice",
|
||||
"provider",
|
||||
"integration",
|
||||
"mcp",
|
||||
@@ -45,6 +46,7 @@ test("exposes every standard HTTP API group", () => {
|
||||
expect(Object.keys(client.integration.oauth)).toEqual(["connect", "status", "complete", "cancel"])
|
||||
expect(Object.keys(client.integration.command)).toEqual(["connect", "status", "cancel"])
|
||||
expect(Object.keys(client.websearch)).toEqual(["providers", "query"])
|
||||
expect(Object.keys(client.voice)).toEqual(["session"])
|
||||
expect(Object.keys(client.file)).toEqual(["read", "list", "find", "write"])
|
||||
expect(Object.keys(client.vcs)).toEqual(["init", "get", "base", "status", "branch", "diff"])
|
||||
expect(Object.keys(client.pty)).toEqual(["list", "create", "get", "update", "remove", "connect"])
|
||||
|
||||
@@ -13,6 +13,7 @@ import { Formatter } from "./formatter.js"
|
||||
import { FileSystem } from "./filesystem.js"
|
||||
import { FileSystemSearch } from "./filesystem/search.js"
|
||||
import { Generate } from "./generate.js"
|
||||
import { Voice } from "./voice.js"
|
||||
import { Form } from "./form.js"
|
||||
import { Image } from "./image.js"
|
||||
import { LocationWatcher } from "./filesystem/location-watcher.js"
|
||||
@@ -97,6 +98,7 @@ const nodes = [
|
||||
InstructionEntry.node,
|
||||
Form.node,
|
||||
Generate.node,
|
||||
Voice.node,
|
||||
ReadToolFileSystem.node,
|
||||
McpTool.node,
|
||||
SessionInstructions.node,
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
export * as Voice from "./voice.js"
|
||||
|
||||
import { type AIError } from "@opencode/ai"
|
||||
import {
|
||||
Realtime,
|
||||
type RealtimeSession,
|
||||
type RealtimeTransport,
|
||||
type RealtimeTransportInput,
|
||||
} from "@opencode/ai/experimental"
|
||||
import { RequestExecutor } from "@opencode/ai/route"
|
||||
import { makeLocationNode } from "@opencode/util/effect/app-node"
|
||||
import { Context, Effect, Layer, Redacted, Schema } from "effect"
|
||||
import { requestExecutor } from "./effect/app-node-platform.js"
|
||||
import { Integration } from "./integration.js"
|
||||
|
||||
export interface SessionInput {
|
||||
readonly model: string
|
||||
readonly session?: RealtimeSession
|
||||
readonly transport: RealtimeTransportInput
|
||||
}
|
||||
|
||||
export class InvalidRequestError extends Schema.TaggedError<InvalidRequestError>()("Voice.InvalidRequestError", {
|
||||
message: Schema.String,
|
||||
}) {}
|
||||
|
||||
export class UnavailableError extends Schema.TaggedError<UnavailableError>()("Voice.UnavailableError", {
|
||||
message: Schema.String,
|
||||
service: Schema.optional(Schema.String),
|
||||
}) {}
|
||||
|
||||
export type Error = InvalidRequestError | UnavailableError
|
||||
|
||||
export interface Interface {
|
||||
readonly session: (input: SessionInput) => Effect.Effect<RealtimeTransport, Error>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/Voice") {}
|
||||
|
||||
export const layer = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
const integrations = yield* Integration.Service
|
||||
|
||||
const session = Effect.fn("Voice.session")(function* (input: SessionInput) {
|
||||
const engine = Realtime.engine(input.model)
|
||||
if (!engine) return yield* new InvalidRequestError({ message: `Unsupported voice model: ${input.model}` })
|
||||
|
||||
// Integration lists the active credential first, then newer stored credentials, then its env vars.
|
||||
const connection = (yield* integrations.get(Integration.ID.make(engine.provider)))?.connections.find(
|
||||
(item) => item.type === "env" || item.method === "key",
|
||||
)
|
||||
const credential = connection
|
||||
? yield* integrations.connection
|
||||
.resolve(connection)
|
||||
.pipe(
|
||||
Effect.mapError((error) => new UnavailableError({ service: engine.provider, message: error.message })),
|
||||
)
|
||||
: undefined
|
||||
if (credential?.type !== "key")
|
||||
return yield* new UnavailableError({
|
||||
service: engine.provider,
|
||||
message:
|
||||
engine.provider === "openai"
|
||||
? "Voice needs an OpenAI API key on the server (a stored key or OPENAI_API_KEY). ChatGPT sign-in can't start voice sessions."
|
||||
: "Voice needs an xAI API key on the server (a stored key or XAI_API_KEY).",
|
||||
})
|
||||
|
||||
return yield* Realtime.connect({ ...input, apiKey: Redacted.make(credential.key) }).pipe(
|
||||
Effect.provideService(RequestExecutor.Service, executor),
|
||||
Effect.mapError(
|
||||
(error: AIError): Error =>
|
||||
error.reason._tag === "InvalidRequest" || error.reason._tag === "UnsupportedOperation"
|
||||
? new InvalidRequestError({ message: error.message })
|
||||
: new UnavailableError({ message: error.message, service: engine.provider }),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
return Service.of({ session })
|
||||
}),
|
||||
)
|
||||
|
||||
export const node = makeLocationNode({
|
||||
service: Service,
|
||||
layer,
|
||||
deps: [Integration.node, requestExecutor],
|
||||
})
|
||||
@@ -0,0 +1,107 @@
|
||||
import { expect } from "bun:test"
|
||||
import { RequestExecutor } from "@opencode/ai/route"
|
||||
import { Credential } from "@opencode/core/credential"
|
||||
import { Integration } from "@opencode/core/integration"
|
||||
import { Voice } from "@opencode/core/voice"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClient, type HttpClientRequest, HttpClientResponse } from "effect/http"
|
||||
import { testEffect } from "./lib/effect"
|
||||
|
||||
const it = testEffect(Layer.empty)
|
||||
|
||||
const credential = (id: string, method: "key" | "oauth") => ({
|
||||
type: "credential" as const,
|
||||
id: Credential.ID.make(id),
|
||||
label: id,
|
||||
method,
|
||||
})
|
||||
|
||||
const keys: Record<string, string> = { active: "sk-active", stored: "sk-stored", XAI_API_KEY: "xai-env-key" }
|
||||
|
||||
it.effect("uses the first key or env connection Integration lists and never a ChatGPT sign-in", () =>
|
||||
Effect.gen(function* () {
|
||||
const auths: string[] = []
|
||||
const call = (model: string, connections: Integration.Info["connections"]) =>
|
||||
Effect.flatMap(Voice.Service, (voice) =>
|
||||
voice.session({
|
||||
model,
|
||||
transport: model.startsWith("grok-") ? { type: "websocket" } : { type: "webrtc", sdp: "v=0" },
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
Voice.layer.pipe(
|
||||
Layer.provide(
|
||||
Layer.merge(
|
||||
Layer.mock(Integration.Service, {
|
||||
revision: () => 0,
|
||||
get: (id) => Effect.succeed({ id, name: id, methods: [], connections }),
|
||||
connection: {
|
||||
active: () => Effect.die("unused"),
|
||||
resolve: (connection) =>
|
||||
connection.type === "credential" && connection.method === "oauth"
|
||||
? Effect.die("a ChatGPT sign-in was resolved")
|
||||
: Effect.succeed(
|
||||
Credential.Key.make({
|
||||
type: "key",
|
||||
key: keys[connection.type === "env" ? connection.name : connection.id],
|
||||
}),
|
||||
),
|
||||
key: () => Effect.die("unused"),
|
||||
external: () => Effect.die("unused"),
|
||||
activate: () => Effect.die("unused"),
|
||||
update: () => Effect.die("unused"),
|
||||
remove: () => Effect.die("unused"),
|
||||
status: () => Effect.die("unused"),
|
||||
},
|
||||
oauth: {
|
||||
connect: () => Effect.die("unused"),
|
||||
status: () => Effect.die("unused"),
|
||||
complete: () => Effect.die("unused"),
|
||||
cancel: () => Effect.die("unused"),
|
||||
},
|
||||
command: {
|
||||
connect: () => Effect.die("unused"),
|
||||
status: () => Effect.die("unused"),
|
||||
cancel: () => Effect.die("unused"),
|
||||
},
|
||||
}),
|
||||
RequestExecutor.layer.pipe(
|
||||
Layer.provide(
|
||||
Layer.succeed(
|
||||
HttpClient.HttpClient,
|
||||
HttpClient.make((req: HttpClientRequest.HttpClientRequest) =>
|
||||
Effect.sync(() => {
|
||||
auths.push(req.headers.authorization ?? "")
|
||||
return HttpClientResponse.fromWeb(
|
||||
req,
|
||||
model.startsWith("grok-")
|
||||
? Response.json({ value: "ek_xai", expires_at: 1760000999 })
|
||||
: Response.json({ transport: { sdp: "v=0\r\no=answer\r\n" } }),
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
yield* call("gpt-live-1", [credential("active", "key"), credential("stored", "key")])
|
||||
yield* call("gpt-live-1", [credential("chatgpt", "oauth"), credential("stored", "key")])
|
||||
const grok = yield* call("grok-voice-3.0", [{ type: "env", name: "XAI_API_KEY" }])
|
||||
expect(auths).toEqual(["Bearer sk-active", "Bearer sk-stored", "Bearer xai-env-key"])
|
||||
expect(grok).toEqual({
|
||||
type: "websocket",
|
||||
url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0",
|
||||
token: "ek_xai",
|
||||
expiresAt: 1760000999,
|
||||
})
|
||||
|
||||
const signInOnly = yield* call("gpt-live-1", [credential("chatgpt", "oauth")]).pipe(Effect.flip)
|
||||
expect(signInOnly._tag).toBe("Voice.UnavailableError")
|
||||
expect(signInOnly.message).toContain("OpenAI API key")
|
||||
}),
|
||||
)
|
||||
@@ -5357,6 +5357,98 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/experimental/voice/session": {
|
||||
"post": {
|
||||
"tags": ["voice"],
|
||||
"operationId": "experimental.voice.session",
|
||||
"parameters": [],
|
||||
"security": [],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "VoiceSessionResponse",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/VoiceSessionResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "InvalidRequestError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/InvalidRequestErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "UnauthorizedError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/UnauthorizedErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "LocationNotFoundError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/LocationNotFoundErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "ServiceUnavailableError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ServiceUnavailableErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.",
|
||||
"summary": "Create voice session",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"session": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"transport": {
|
||||
"$ref": "#/components/schemas/VoiceTransportInput"
|
||||
}
|
||||
},
|
||||
"required": ["model", "transport"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/provider": {
|
||||
"get": {
|
||||
"tags": ["provider"],
|
||||
@@ -20330,6 +20422,128 @@
|
||||
"required": ["_tag", "providerID", "message"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceSessionResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"transport": {
|
||||
"$ref": "#/components/schemas/VoiceTransport"
|
||||
}
|
||||
},
|
||||
"required": ["transport"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": ["data"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceTransport": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebRTCTransport"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebSocketTransport"
|
||||
}
|
||||
]
|
||||
},
|
||||
"VoiceTransportInput": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebRTCTransportInput"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebSocketTransportInput"
|
||||
}
|
||||
]
|
||||
},
|
||||
"VoiceWebRTCTransport": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["webrtc"]
|
||||
},
|
||||
"sdp": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["type", "sdp"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceWebRTCTransportInput": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["webrtc"]
|
||||
},
|
||||
"sdp": {
|
||||
"type": "string"
|
||||
},
|
||||
"dcid": {
|
||||
"anyOf": [
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"enum": ["Infinity", "-Infinity", "NaN"]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": ["type", "sdp"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceWebSocketTransport": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["websocket"]
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
},
|
||||
"token": {
|
||||
"type": "string"
|
||||
},
|
||||
"expiresAt": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"enum": ["Infinity", "-Infinity", "NaN"]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": ["type", "url", "token", "expiresAt"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceWebSocketTransportInput": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["websocket"]
|
||||
}
|
||||
},
|
||||
"required": ["type"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"WebSearch.ID": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -20523,6 +20737,10 @@
|
||||
"name": "generate",
|
||||
"description": "Experimental one-shot generation routes."
|
||||
},
|
||||
{
|
||||
"name": "voice",
|
||||
"description": "Experimental server-brokered voice session routes."
|
||||
},
|
||||
{
|
||||
"name": "provider",
|
||||
"description": "Experimental provider routes."
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Context } from "effect"
|
||||
import { HttpApi, HttpApiGroup, HttpApiMiddleware, OpenApi } from "effect/http-api"
|
||||
import { SchemaErrorMiddleware } from "./middleware/schema-error.js"
|
||||
import { GenerateGroup } from "./groups/generate.js"
|
||||
import { VoiceGroup } from "./groups/voice.js"
|
||||
import { MessageGroup } from "./groups/message.js"
|
||||
import { ModelGroup } from "./groups/model.js"
|
||||
import { ProviderGroup } from "./groups/provider.js"
|
||||
@@ -94,6 +95,7 @@ type ApiGroups<
|
||||
| typeof MigrationGroup
|
||||
| typeof WorktreeGroup
|
||||
| typeof GenerateGroup
|
||||
| typeof VoiceGroup
|
||||
| typeof PersistentPtyGroup
|
||||
| typeof CredentialGroup
|
||||
| LocationGroups<LocationId>
|
||||
@@ -165,6 +167,7 @@ const makeApiFromGroup = <
|
||||
.add(MessageGroup)
|
||||
.add(ModelGroup.middleware(locationMiddleware))
|
||||
.add(GenerateGroup)
|
||||
.add(VoiceGroup)
|
||||
.add(ProviderGroup.middleware(locationMiddleware))
|
||||
.add(IntegrationGroup.middleware(locationMiddleware))
|
||||
.add(McpGroup.middleware(locationMiddleware))
|
||||
|
||||
@@ -44,6 +44,7 @@ export const groupNames = {
|
||||
"server.message": "message",
|
||||
"server.model": "model",
|
||||
"server.generate": "generate",
|
||||
"server.voice": "voice",
|
||||
"server.provider": "provider",
|
||||
"server.integration": "integration",
|
||||
"server.websearch": "websearch",
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { Schema } from "effect"
|
||||
import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/http-api"
|
||||
import { InvalidRequestError, LocationNotFoundError, ServiceUnavailableError } from "../errors.js"
|
||||
|
||||
export const VoiceWebRTCTransportInput = Schema.Struct({
|
||||
type: Schema.Literal("webrtc"),
|
||||
sdp: Schema.String,
|
||||
dcid: Schema.optional(Schema.Number),
|
||||
}).annotate({ identifier: "VoiceWebRTCTransportInput" })
|
||||
|
||||
export const VoiceWebSocketTransportInput = Schema.Struct({
|
||||
type: Schema.Literal("websocket"),
|
||||
}).annotate({ identifier: "VoiceWebSocketTransportInput" })
|
||||
|
||||
export const VoiceTransportInput = Schema.Union([
|
||||
VoiceWebRTCTransportInput,
|
||||
VoiceWebSocketTransportInput,
|
||||
]).annotate({ identifier: "VoiceTransportInput" })
|
||||
|
||||
export const VoiceWebRTCTransport = Schema.Struct({
|
||||
type: Schema.Literal("webrtc"),
|
||||
sdp: Schema.String,
|
||||
}).annotate({ identifier: "VoiceWebRTCTransport" })
|
||||
|
||||
export const VoiceWebSocketTransport = Schema.Struct({
|
||||
type: Schema.Literal("websocket"),
|
||||
url: Schema.String,
|
||||
token: Schema.String,
|
||||
expiresAt: Schema.Number,
|
||||
}).annotate({ identifier: "VoiceWebSocketTransport" })
|
||||
|
||||
export const VoiceTransport = Schema.Union([
|
||||
VoiceWebRTCTransport,
|
||||
VoiceWebSocketTransport,
|
||||
]).annotate({ identifier: "VoiceTransport" })
|
||||
|
||||
export const VoiceGroup = HttpApiGroup.make("server.voice")
|
||||
.add(
|
||||
HttpApiEndpoint.post("voice.session", "/api/experimental/voice/session", {
|
||||
payload: Schema.Struct({
|
||||
model: Schema.String,
|
||||
session: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
transport: VoiceTransportInput,
|
||||
}),
|
||||
success: Schema.Struct({
|
||||
data: Schema.Struct({
|
||||
transport: VoiceTransport,
|
||||
}),
|
||||
}).annotate({ identifier: "VoiceSessionResponse" }),
|
||||
error: [InvalidRequestError, ServiceUnavailableError, LocationNotFoundError],
|
||||
}).annotateMerge(
|
||||
OpenApi.annotations({
|
||||
identifier: "experimental.voice.session",
|
||||
summary: "Create voice session",
|
||||
description:
|
||||
"Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.",
|
||||
}),
|
||||
),
|
||||
)
|
||||
.annotateMerge(
|
||||
OpenApi.annotations({
|
||||
title: "voice",
|
||||
description: "Experimental server-brokered voice session routes.",
|
||||
}),
|
||||
)
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Layer } from "effect"
|
||||
import { GenerateHandler } from "./handlers/generate"
|
||||
import { VoiceHandler } from "./handlers/voice"
|
||||
import { MessageHandler } from "./handlers/message"
|
||||
import { ModelHandler } from "./handlers/model"
|
||||
import { ProviderHandler } from "./handlers/provider"
|
||||
@@ -42,6 +43,7 @@ export const handlers = Layer.mergeAll(
|
||||
MessageHandler,
|
||||
ModelHandler,
|
||||
GenerateHandler,
|
||||
VoiceHandler,
|
||||
ProviderHandler,
|
||||
IntegrationHandler,
|
||||
WebSearchHandler,
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { Location } from "@opencode/core/location"
|
||||
import { LocationServiceMap } from "@opencode/core/location-services"
|
||||
import { AbsolutePath } from "@opencode/core/schema"
|
||||
import { Voice } from "@opencode/core/voice"
|
||||
import { InvalidRequestError, ServiceUnavailableError } from "@opencode/protocol/errors"
|
||||
import { Global } from "@opencode/util/global"
|
||||
import { Effect } from "effect"
|
||||
import { HttpApiBuilder } from "effect/http-api"
|
||||
import { Api } from "../api"
|
||||
import { awaitPlugins, locationErrors } from "../location"
|
||||
|
||||
export const VoiceHandler = HttpApiBuilder.group(Api, "server.voice", (handlers) =>
|
||||
Effect.gen(function* () {
|
||||
const global = yield* Global.Service
|
||||
const locations = yield* LocationServiceMap.Service
|
||||
const services = locations.get(Location.Ref.make({ directory: AbsolutePath.make(global.config) }))
|
||||
return handlers.handle(
|
||||
"voice.session",
|
||||
Effect.fn("server.voice.session")(
|
||||
function* (request) {
|
||||
yield* awaitPlugins
|
||||
const voice = yield* Voice.Service
|
||||
const transport = yield* voice
|
||||
.session(request.payload)
|
||||
.pipe(
|
||||
Effect.mapError((error) =>
|
||||
error._tag === "Voice.InvalidRequestError"
|
||||
? new InvalidRequestError({ message: error.message })
|
||||
: new ServiceUnavailableError({ message: error.message, service: error.service }),
|
||||
),
|
||||
)
|
||||
return { data: { transport } }
|
||||
},
|
||||
Effect.provide(services),
|
||||
locationErrors,
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
@@ -0,0 +1,88 @@
|
||||
import fs from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import { expect } from "bun:test"
|
||||
import { Voice } from "@opencode/core/voice"
|
||||
import { makeLocationNode } from "@opencode/util/effect/app-node"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { tmpdir } from "../../core/test/fixture/tmpdir"
|
||||
import { it } from "../../core/test/lib/effect"
|
||||
import { ServerFetch } from "../src/fetch"
|
||||
|
||||
const voice = makeLocationNode({
|
||||
service: Voice.Service,
|
||||
layer: Layer.succeed(
|
||||
Voice.Service,
|
||||
Voice.Service.of({
|
||||
session: (input) => {
|
||||
if (input.model === "invalid")
|
||||
return Effect.fail(new Voice.InvalidRequestError({ message: "Unsupported voice model" }))
|
||||
if (input.model === "unavailable")
|
||||
return Effect.fail(new Voice.UnavailableError({ message: "OpenAI API key is required", service: "openai" }))
|
||||
return Effect.succeed(
|
||||
input.transport.type === "webrtc"
|
||||
? { type: "webrtc" as const, sdp: `answer:${input.model}:${input.transport.sdp}` }
|
||||
: {
|
||||
type: "websocket" as const,
|
||||
url: `wss://api.x.ai/v1/realtime?model=${input.model}`,
|
||||
token: "ek_test",
|
||||
expiresAt: 1760000000,
|
||||
},
|
||||
)
|
||||
},
|
||||
}),
|
||||
),
|
||||
deps: [],
|
||||
})
|
||||
|
||||
it.live("routes POST /api/experimental/voice/session and maps errors", () =>
|
||||
Effect.gen(function* () {
|
||||
const tmp = yield* Effect.acquireDisposable(Effect.promise(() => tmpdir("opencode-voice-endpoint-")))
|
||||
const global = path.join(tmp.path, "global")
|
||||
yield* Effect.promise(() => fs.mkdir(global))
|
||||
const handler = yield* ServerFetch.make(
|
||||
{
|
||||
database: { path: ":memory:" },
|
||||
config: { directory: global },
|
||||
fs: { filewatcher: false },
|
||||
models: { fetch: false },
|
||||
},
|
||||
{ overrides: [Voice.node.replace(voice)] },
|
||||
)
|
||||
const post = (body: unknown) =>
|
||||
Effect.promise(() =>
|
||||
handler(
|
||||
new Request("http://opencode.local/api/experimental/voice/session", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
const rtcRes = yield* post({
|
||||
model: "gpt-live-1",
|
||||
session: { voice: "marin" },
|
||||
transport: { type: "webrtc", sdp: "v=0", dcid: 4 },
|
||||
})
|
||||
expect(rtcRes.status).toBe(200)
|
||||
expect(yield* Effect.promise(() => rtcRes.json())).toEqual({
|
||||
data: { transport: { type: "webrtc", sdp: "answer:gpt-live-1:v=0" } },
|
||||
})
|
||||
|
||||
const wsRes = yield* post({ model: "grok-voice-3.0", transport: { type: "websocket" } })
|
||||
expect(wsRes.status).toBe(200)
|
||||
expect(yield* Effect.promise(() => wsRes.json())).toEqual({
|
||||
data: {
|
||||
transport: {
|
||||
type: "websocket",
|
||||
url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0",
|
||||
token: "ek_test",
|
||||
expiresAt: 1760000000,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect((yield* post({ model: "invalid", transport: { type: "websocket" } })).status).toBe(400)
|
||||
expect((yield* post({ model: "unavailable", transport: { type: "websocket" } })).status).toBe(503)
|
||||
}),
|
||||
)
|
||||
@@ -5357,6 +5357,98 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/experimental/voice/session": {
|
||||
"post": {
|
||||
"tags": ["voice"],
|
||||
"operationId": "experimental.voice.session",
|
||||
"parameters": [],
|
||||
"security": [],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "VoiceSessionResponse",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/VoiceSessionResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "InvalidRequestError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/InvalidRequestErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "UnauthorizedError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/UnauthorizedErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "LocationNotFoundError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/LocationNotFoundErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "ServiceUnavailableError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ServiceUnavailableErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.",
|
||||
"summary": "Create voice session",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"session": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"transport": {
|
||||
"$ref": "#/components/schemas/VoiceTransportInput"
|
||||
}
|
||||
},
|
||||
"required": ["model", "transport"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/provider": {
|
||||
"get": {
|
||||
"tags": ["provider"],
|
||||
@@ -20330,6 +20422,128 @@
|
||||
"required": ["_tag", "providerID", "message"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceSessionResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"transport": {
|
||||
"$ref": "#/components/schemas/VoiceTransport"
|
||||
}
|
||||
},
|
||||
"required": ["transport"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": ["data"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceTransport": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebRTCTransport"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebSocketTransport"
|
||||
}
|
||||
]
|
||||
},
|
||||
"VoiceTransportInput": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebRTCTransportInput"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebSocketTransportInput"
|
||||
}
|
||||
]
|
||||
},
|
||||
"VoiceWebRTCTransport": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["webrtc"]
|
||||
},
|
||||
"sdp": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["type", "sdp"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceWebRTCTransportInput": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["webrtc"]
|
||||
},
|
||||
"sdp": {
|
||||
"type": "string"
|
||||
},
|
||||
"dcid": {
|
||||
"anyOf": [
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"enum": ["Infinity", "-Infinity", "NaN"]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": ["type", "sdp"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceWebSocketTransport": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["websocket"]
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
},
|
||||
"token": {
|
||||
"type": "string"
|
||||
},
|
||||
"expiresAt": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"enum": ["Infinity", "-Infinity", "NaN"]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": ["type", "url", "token", "expiresAt"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceWebSocketTransportInput": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["websocket"]
|
||||
}
|
||||
},
|
||||
"required": ["type"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"WebSearch.ID": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -20523,6 +20737,10 @@
|
||||
"name": "generate",
|
||||
"description": "Experimental one-shot generation routes."
|
||||
},
|
||||
{
|
||||
"name": "voice",
|
||||
"description": "Experimental server-brokered voice session routes."
|
||||
},
|
||||
{
|
||||
"name": "provider",
|
||||
"description": "Experimental provider routes."
|
||||
|
||||
@@ -5357,6 +5357,98 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/experimental/voice/session": {
|
||||
"post": {
|
||||
"tags": ["voice"],
|
||||
"operationId": "experimental.voice.session",
|
||||
"parameters": [],
|
||||
"security": [],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "VoiceSessionResponse",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/VoiceSessionResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "InvalidRequestError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/InvalidRequestErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "UnauthorizedError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/UnauthorizedErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "LocationNotFoundError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/LocationNotFoundErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "ServiceUnavailableError",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ServiceUnavailableErrorEncoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Create a server-brokered voice session using the server's configured provider credentials and return the WebRTC SDP answer or WebSocket ephemeral token.",
|
||||
"summary": "Create voice session",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"session": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"transport": {
|
||||
"$ref": "#/components/schemas/VoiceTransportInput"
|
||||
}
|
||||
},
|
||||
"required": ["model", "transport"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/provider": {
|
||||
"get": {
|
||||
"tags": ["provider"],
|
||||
@@ -20330,6 +20422,128 @@
|
||||
"required": ["_tag", "providerID", "message"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceSessionResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"transport": {
|
||||
"$ref": "#/components/schemas/VoiceTransport"
|
||||
}
|
||||
},
|
||||
"required": ["transport"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": ["data"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceTransport": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebRTCTransport"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebSocketTransport"
|
||||
}
|
||||
]
|
||||
},
|
||||
"VoiceTransportInput": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebRTCTransportInput"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/VoiceWebSocketTransportInput"
|
||||
}
|
||||
]
|
||||
},
|
||||
"VoiceWebRTCTransport": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["webrtc"]
|
||||
},
|
||||
"sdp": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["type", "sdp"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceWebRTCTransportInput": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["webrtc"]
|
||||
},
|
||||
"sdp": {
|
||||
"type": "string"
|
||||
},
|
||||
"dcid": {
|
||||
"anyOf": [
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"enum": ["Infinity", "-Infinity", "NaN"]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": ["type", "sdp"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceWebSocketTransport": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["websocket"]
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
},
|
||||
"token": {
|
||||
"type": "string"
|
||||
},
|
||||
"expiresAt": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"enum": ["Infinity", "-Infinity", "NaN"]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": ["type", "url", "token", "expiresAt"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceWebSocketTransportInput": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["websocket"]
|
||||
}
|
||||
},
|
||||
"required": ["type"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"WebSearch.ID": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -20523,6 +20737,10 @@
|
||||
"name": "generate",
|
||||
"description": "Experimental one-shot generation routes."
|
||||
},
|
||||
{
|
||||
"name": "voice",
|
||||
"description": "Experimental server-brokered voice session routes."
|
||||
},
|
||||
{
|
||||
"name": "provider",
|
||||
"description": "Experimental provider routes."
|
||||
|
||||
Reference in New Issue
Block a user