mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
feat(voice): add Gemini Live ephemeral token support (#56888)
This commit is contained in:
committed by
GitHub
Unverified
parent
c718e6b9e4
commit
31dc762147
@@ -1,4 +1,4 @@
|
||||
import { Effect, type Redacted, Schema } from "effect"
|
||||
import { Clock, Effect, Redacted, Schema } from "effect"
|
||||
import { HttpClientRequest } from "effect/http"
|
||||
import { RequestExecutor } from "../route/executor.js"
|
||||
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "../schema/index.js"
|
||||
@@ -6,6 +6,7 @@ import { encodeJson } from "../utils/json.js"
|
||||
|
||||
const OPENAI_BASE_URL = "https://api.openai.com/v1"
|
||||
const XAI_BASE_URL = "https://api.x.ai/v1"
|
||||
const GOOGLE_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
||||
export type RealtimeTransportInput =
|
||||
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number }
|
||||
@@ -28,6 +29,7 @@ const engines = [
|
||||
{ family: "gpt-live", id: "openai-live", provider: "openai" },
|
||||
{ family: "gpt-realtime", id: "openai-realtime", provider: "openai" },
|
||||
{ family: "grok-voice", id: "xai-realtime", provider: "xai" },
|
||||
{ family: "gemini-3.8-live", id: "gemini-live", provider: "google" },
|
||||
] as const
|
||||
|
||||
// Matches the bare family name (`gpt-realtime`) and its versions (`gpt-realtime-2.1`, `gpt-realtime-mini`).
|
||||
@@ -73,6 +75,31 @@ export const connect = Effect.fn("Realtime.connect")(function* (input: RealtimeC
|
||||
return yield* invalidRequest("xAI Grok Voice only supports WebSocket transport")
|
||||
return yield* clientSecret(resolved.id, XAI_BASE_URL, input.model, input.apiKey)
|
||||
}
|
||||
case "gemini-live": {
|
||||
if (input.transport.type !== "websocket")
|
||||
return yield* invalidRequest("Google Gemini Live only supports WebSocket transport")
|
||||
const now = yield* Clock.currentTimeMillis
|
||||
const expiresAt = now + 60_000
|
||||
const data = yield* post(
|
||||
resolved.id,
|
||||
`${GOOGLE_BASE_URL}/auth_tokens`,
|
||||
input.apiKey,
|
||||
{
|
||||
uses: 1,
|
||||
expireTime: new Date(now + 30 * 60_000).toISOString(),
|
||||
newSessionExpireTime: new Date(expiresAt).toISOString(),
|
||||
bidiGenerateContentSetup: { ...input.session, model: `models/${input.model}` },
|
||||
},
|
||||
AuthTokenResponse,
|
||||
"x-goog-api-key",
|
||||
)
|
||||
return {
|
||||
type: "websocket" as const,
|
||||
url: "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained",
|
||||
token: data.name,
|
||||
expiresAt: Math.floor(expiresAt / 1000),
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
@@ -97,6 +124,12 @@ const ClientSecretResponse = Schema.fromJsonString(
|
||||
}),
|
||||
)
|
||||
|
||||
const AuthTokenResponse = Schema.fromJsonString(
|
||||
Schema.Struct({
|
||||
name: Schema.String,
|
||||
}),
|
||||
)
|
||||
|
||||
const invalidRequest = (message: string) => new AIError({ reason: new InvalidRequestError({ message }) })
|
||||
|
||||
const post = <A>(
|
||||
@@ -105,12 +138,15 @@ const post = <A>(
|
||||
apiKey: Redacted.Redacted,
|
||||
body: Record<string, unknown> | FormData,
|
||||
response: Schema.Codec<A, string>,
|
||||
authHeader?: string,
|
||||
) =>
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
const res = yield* executor.execute(
|
||||
HttpClientRequest.post(url).pipe(
|
||||
HttpClientRequest.bearerToken(apiKey),
|
||||
authHeader
|
||||
? HttpClientRequest.setHeader(authHeader, Redacted.value(apiKey))
|
||||
: HttpClientRequest.bearerToken(apiKey),
|
||||
body instanceof FormData
|
||||
? HttpClientRequest.bodyFormData(body)
|
||||
: HttpClientRequest.bodyText(encodeJson(body), "application/json"),
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Effect, Redacted } from "effect"
|
||||
import { HttpClientRequest } from "effect/http"
|
||||
import { TestClock } from "effect/testing"
|
||||
import { Realtime } from "../src/experimental.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
import { dynamicResponse } from "./lib/http.js"
|
||||
|
||||
describe("experimental Realtime", () => {
|
||||
it.effect("connects OpenAI GPT-Live, OpenAI Realtime (WebRTC + WebSocket), and xAI Grok Voice", () =>
|
||||
it.effect("connects OpenAI GPT-Live, OpenAI Realtime, xAI Grok Voice, and Google Gemini Live", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* TestClock.setTime(1760000000_000)
|
||||
|
||||
expect(
|
||||
yield* Realtime.connect({
|
||||
model: "gpt-live-1",
|
||||
@@ -53,6 +56,23 @@ describe("experimental Realtime", () => {
|
||||
token: "ek_2",
|
||||
expiresAt: 1760000120,
|
||||
})
|
||||
|
||||
expect(
|
||||
yield* Realtime.connect({
|
||||
model: "gemini-3.8-live",
|
||||
apiKey: Redacted.make("gemini-key"),
|
||||
session: {
|
||||
generationConfig: { responseModalities: ["AUDIO"] },
|
||||
systemInstruction: { parts: [{ text: "Brief." }] },
|
||||
},
|
||||
transport: { type: "websocket" },
|
||||
}),
|
||||
).toEqual({
|
||||
type: "websocket",
|
||||
url: "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained",
|
||||
token: "auth_tokens/ek_3",
|
||||
expiresAt: 1760000060,
|
||||
})
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
@@ -92,6 +112,21 @@ describe("experimental Realtime", () => {
|
||||
})
|
||||
return input.respond(JSON.stringify({ value: "ek_1", expires_at: 1760000060 }))
|
||||
}
|
||||
if (req.url === "https://generativelanguage.googleapis.com/v1beta/auth_tokens") {
|
||||
expect(req.headers.get("x-goog-api-key")).toBe("gemini-key")
|
||||
expect(req.headers.get("authorization")).toBeNull()
|
||||
expect(JSON.parse(input.text)).toEqual({
|
||||
uses: 1,
|
||||
expireTime: "2025-10-09T09:23:20.000Z",
|
||||
newSessionExpireTime: "2025-10-09T08:54:20.000Z",
|
||||
bidiGenerateContentSetup: {
|
||||
generationConfig: { responseModalities: ["AUDIO"] },
|
||||
systemInstruction: { parts: [{ text: "Brief." }] },
|
||||
model: "models/gemini-3.8-live",
|
||||
},
|
||||
})
|
||||
return input.respond(JSON.stringify({ name: "auth_tokens/ek_3" }))
|
||||
}
|
||||
expect(req.url).toBe("https://api.x.ai/v1/realtime/client_secrets")
|
||||
expect(req.headers.get("authorization")).toBe("Bearer xai-key")
|
||||
expect(JSON.parse(input.text)).toEqual({ expires_after: { seconds: 60 } })
|
||||
@@ -104,9 +139,18 @@ describe("experimental Realtime", () => {
|
||||
|
||||
test("routes bare and versioned model names to their engine", () => {
|
||||
expect(
|
||||
["gpt-realtime", "gpt-realtime-2.1", "gpt-live", "grok-voice-latest"].map((model) => Realtime.engine(model)?.id),
|
||||
).toEqual(["openai-realtime", "openai-realtime", "openai-live", "xai-realtime"])
|
||||
[
|
||||
"gpt-realtime",
|
||||
"gpt-realtime-2.1",
|
||||
"gpt-live",
|
||||
"grok-voice-latest",
|
||||
"gemini-3.8-live",
|
||||
"gemini-3.8-live-extended-thinking",
|
||||
].map((model) => Realtime.engine(model)?.id),
|
||||
).toEqual(["openai-realtime", "openai-realtime", "openai-live", "xai-realtime", "gemini-live", "gemini-live"])
|
||||
expect(Realtime.engine("gpt-realtimex")).toBeUndefined()
|
||||
expect(Realtime.engine("gemini-3.8-flash")).toBeUndefined()
|
||||
expect(Realtime.engine("gemini-3.5-live")).toBeUndefined()
|
||||
})
|
||||
|
||||
it.effect("rejects unsupported transports before network I/O", () =>
|
||||
@@ -121,7 +165,16 @@ describe("experimental Realtime", () => {
|
||||
apiKey: Redacted.make("k"),
|
||||
transport: { type: "webrtc", sdp: "v=0" },
|
||||
}).pipe(Effect.flip)
|
||||
expect([liveWs.reason._tag, grokRtc.reason._tag]).toEqual(["InvalidRequest", "InvalidRequest"])
|
||||
const geminiRtc = yield* Realtime.connect({
|
||||
model: "gemini-3.8-live",
|
||||
apiKey: Redacted.make("k"),
|
||||
transport: { type: "webrtc", sdp: "v=0" },
|
||||
}).pipe(Effect.flip)
|
||||
expect([liveWs.reason._tag, grokRtc.reason._tag, geminiRtc.reason._tag]).toEqual([
|
||||
"InvalidRequest",
|
||||
"InvalidRequest",
|
||||
"InvalidRequest",
|
||||
])
|
||||
}).pipe(Effect.provide(dynamicResponse(() => Effect.die("unsupported transport reached the network")))),
|
||||
)
|
||||
})
|
||||
|
||||
@@ -60,10 +60,7 @@ export const layer = Layer.effect(
|
||||
if (credential?.type !== "key")
|
||||
return yield* new UnavailableError({
|
||||
service: engine.provider,
|
||||
message:
|
||||
engine.provider === "openai"
|
||||
? "Voice needs an OpenAI API key on the server (a stored key or OPENAI_API_KEY). ChatGPT sign-in can't start voice sessions."
|
||||
: "Voice needs an xAI API key on the server (a stored key or XAI_API_KEY).",
|
||||
message: missingKeyMessage[engine.provider],
|
||||
})
|
||||
|
||||
return yield* Realtime.connect({ ...input, apiKey: Redacted.make(credential.key) }).pipe(
|
||||
@@ -86,3 +83,11 @@ export const node = makeLocationNode({
|
||||
layer,
|
||||
deps: [Integration.node, requestExecutor],
|
||||
})
|
||||
|
||||
const missingKeyMessage = {
|
||||
openai:
|
||||
"Voice needs an OpenAI API key on the server (a stored key or OPENAI_API_KEY). ChatGPT sign-in can't start voice sessions.",
|
||||
xai: "Voice needs an xAI API key on the server (a stored key or XAI_API_KEY).",
|
||||
google:
|
||||
"Voice needs a Google API key on the server (a stored key, GEMINI_API_KEY, GOOGLE_GENERATIVE_AI_API_KEY, or GOOGLE_API_KEY).",
|
||||
} as const
|
||||
|
||||
@@ -5,6 +5,7 @@ import { Integration } from "@opencode/core/integration"
|
||||
import { Voice } from "@opencode/core/voice"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClient, type HttpClientRequest, HttpClientResponse } from "effect/http"
|
||||
import { TestClock } from "effect/testing"
|
||||
import { testEffect } from "./lib/effect"
|
||||
|
||||
const it = testEffect(Layer.empty)
|
||||
@@ -16,16 +17,25 @@ const credential = (id: string, method: "key" | "oauth") => ({
|
||||
method,
|
||||
})
|
||||
|
||||
const keys: Record<string, string> = { active: "sk-active", stored: "sk-stored", XAI_API_KEY: "xai-env-key" }
|
||||
const keys: Record<string, string> = {
|
||||
active: "sk-active",
|
||||
stored: "sk-stored",
|
||||
XAI_API_KEY: "xai-env-key",
|
||||
GEMINI_API_KEY: "gemini-env-key",
|
||||
}
|
||||
|
||||
it.effect("uses the first key or env connection Integration lists and never a ChatGPT sign-in", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* TestClock.setTime(1760000000_000)
|
||||
const auths: string[] = []
|
||||
const call = (model: string, connections: Integration.Info["connections"]) =>
|
||||
Effect.flatMap(Voice.Service, (voice) =>
|
||||
voice.session({
|
||||
model,
|
||||
transport: model.startsWith("grok-") ? { type: "websocket" } : { type: "webrtc", sdp: "v=0" },
|
||||
transport:
|
||||
model.startsWith("grok-") || model.startsWith("gemini-")
|
||||
? { type: "websocket" }
|
||||
: { type: "webrtc", sdp: "v=0" },
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
@@ -71,12 +81,14 @@ it.effect("uses the first key or env connection Integration lists and never a Ch
|
||||
HttpClient.HttpClient,
|
||||
HttpClient.make((req: HttpClientRequest.HttpClientRequest) =>
|
||||
Effect.sync(() => {
|
||||
auths.push(req.headers.authorization ?? "")
|
||||
auths.push(req.headers.authorization ?? req.headers["x-goog-api-key"] ?? "")
|
||||
return HttpClientResponse.fromWeb(
|
||||
req,
|
||||
model.startsWith("grok-")
|
||||
? Response.json({ value: "ek_xai", expires_at: 1760000999 })
|
||||
: Response.json({ transport: { sdp: "v=0\r\no=answer\r\n" } }),
|
||||
model.startsWith("gemini-")
|
||||
? Response.json({ name: "auth_tokens/ek_gemini" })
|
||||
: model.startsWith("grok-")
|
||||
? Response.json({ value: "ek_xai", expires_at: 1760000999 })
|
||||
: Response.json({ transport: { sdp: "v=0\r\no=answer\r\n" } }),
|
||||
)
|
||||
}),
|
||||
),
|
||||
@@ -92,16 +104,27 @@ it.effect("uses the first key or env connection Integration lists and never a Ch
|
||||
yield* call("gpt-live-1", [credential("active", "key"), credential("stored", "key")])
|
||||
yield* call("gpt-live-1", [credential("chatgpt", "oauth"), credential("stored", "key")])
|
||||
const grok = yield* call("grok-voice-3.0", [{ type: "env", name: "XAI_API_KEY" }])
|
||||
expect(auths).toEqual(["Bearer sk-active", "Bearer sk-stored", "Bearer xai-env-key"])
|
||||
const gemini = yield* call("gemini-3.8-live", [{ type: "env", name: "GEMINI_API_KEY" }])
|
||||
expect(auths).toEqual(["Bearer sk-active", "Bearer sk-stored", "Bearer xai-env-key", "gemini-env-key"])
|
||||
expect(grok).toEqual({
|
||||
type: "websocket",
|
||||
url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0",
|
||||
token: "ek_xai",
|
||||
expiresAt: 1760000999,
|
||||
})
|
||||
expect(gemini).toEqual({
|
||||
type: "websocket",
|
||||
url: "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained",
|
||||
token: "auth_tokens/ek_gemini",
|
||||
expiresAt: 1760000060,
|
||||
})
|
||||
|
||||
const signInOnly = yield* call("gpt-live-1", [credential("chatgpt", "oauth")]).pipe(Effect.flip)
|
||||
expect(signInOnly._tag).toBe("Voice.UnavailableError")
|
||||
expect(signInOnly.message).toContain("OpenAI API key")
|
||||
|
||||
const missingGoogle = yield* call("gemini-3.8-live", []).pipe(Effect.flip)
|
||||
expect(missingGoogle).toMatchObject({ _tag: "Voice.UnavailableError", service: "google" })
|
||||
expect(missingGoogle.message).toContain("GEMINI_API_KEY")
|
||||
}),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user