feat(voice): add Gemini Live ephemeral token support (#56888)

This commit is contained in:
Kit Langton
2026-10-19 13:04:04 -07:00
committed by GitHub
Unverified
parent c718e6b9e4
commit 31dc762147
4 changed files with 134 additions and 17 deletions
+38 -2
View File
@@ -1,4 +1,4 @@
import { Effect, type Redacted, Schema } from "effect"
import { Clock, Effect, Redacted, Schema } from "effect"
import { HttpClientRequest } from "effect/http"
import { RequestExecutor } from "../route/executor.js"
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "../schema/index.js"
@@ -6,6 +6,7 @@ import { encodeJson } from "../utils/json.js"
const OPENAI_BASE_URL = "https://api.openai.com/v1"
const XAI_BASE_URL = "https://api.x.ai/v1"
const GOOGLE_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
export type RealtimeTransportInput =
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number }
@@ -28,6 +29,7 @@ const engines = [
{ family: "gpt-live", id: "openai-live", provider: "openai" },
{ family: "gpt-realtime", id: "openai-realtime", provider: "openai" },
{ family: "grok-voice", id: "xai-realtime", provider: "xai" },
{ family: "gemini-3.8-live", id: "gemini-live", provider: "google" },
] as const
// Matches the bare family name (`gpt-realtime`) and its versions (`gpt-realtime-2.1`, `gpt-realtime-mini`).
@@ -73,6 +75,31 @@ export const connect = Effect.fn("Realtime.connect")(function* (input: RealtimeC
return yield* invalidRequest("xAI Grok Voice only supports WebSocket transport")
return yield* clientSecret(resolved.id, XAI_BASE_URL, input.model, input.apiKey)
}
case "gemini-live": {
if (input.transport.type !== "websocket")
return yield* invalidRequest("Google Gemini Live only supports WebSocket transport")
const now = yield* Clock.currentTimeMillis
const expiresAt = now + 60_000
const data = yield* post(
resolved.id,
`${GOOGLE_BASE_URL}/auth_tokens`,
input.apiKey,
{
uses: 1,
expireTime: new Date(now + 30 * 60_000).toISOString(),
newSessionExpireTime: new Date(expiresAt).toISOString(),
bidiGenerateContentSetup: { ...input.session, model: `models/${input.model}` },
},
AuthTokenResponse,
"x-goog-api-key",
)
return {
type: "websocket" as const,
url: "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained",
token: data.name,
expiresAt: Math.floor(expiresAt / 1000),
}
}
}
})
@@ -97,6 +124,12 @@ const ClientSecretResponse = Schema.fromJsonString(
}),
)
const AuthTokenResponse = Schema.fromJsonString(
Schema.Struct({
name: Schema.String,
}),
)
const invalidRequest = (message: string) => new AIError({ reason: new InvalidRequestError({ message }) })
const post = <A>(
@@ -105,12 +138,15 @@ const post = <A>(
apiKey: Redacted.Redacted,
body: Record<string, unknown> | FormData,
response: Schema.Codec<A, string>,
authHeader?: string,
) =>
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
const res = yield* executor.execute(
HttpClientRequest.post(url).pipe(
HttpClientRequest.bearerToken(apiKey),
authHeader
? HttpClientRequest.setHeader(authHeader, Redacted.value(apiKey))
: HttpClientRequest.bearerToken(apiKey),
body instanceof FormData
? HttpClientRequest.bodyFormData(body)
: HttpClientRequest.bodyText(encodeJson(body), "application/json"),
+57 -4
View File
@@ -1,13 +1,16 @@
import { describe, expect, test } from "bun:test"
import { Effect, Redacted } from "effect"
import { HttpClientRequest } from "effect/http"
import { TestClock } from "effect/testing"
import { Realtime } from "../src/experimental.js"
import { it } from "./lib/effect.js"
import { dynamicResponse } from "./lib/http.js"
describe("experimental Realtime", () => {
it.effect("connects OpenAI GPT-Live, OpenAI Realtime (WebRTC + WebSocket), and xAI Grok Voice", () =>
it.effect("connects OpenAI GPT-Live, OpenAI Realtime, xAI Grok Voice, and Google Gemini Live", () =>
Effect.gen(function* () {
yield* TestClock.setTime(1760000000_000)
expect(
yield* Realtime.connect({
model: "gpt-live-1",
@@ -53,6 +56,23 @@ describe("experimental Realtime", () => {
token: "ek_2",
expiresAt: 1760000120,
})
expect(
yield* Realtime.connect({
model: "gemini-3.8-live",
apiKey: Redacted.make("gemini-key"),
session: {
generationConfig: { responseModalities: ["AUDIO"] },
systemInstruction: { parts: [{ text: "Brief." }] },
},
transport: { type: "websocket" },
}),
).toEqual({
type: "websocket",
url: "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained",
token: "auth_tokens/ek_3",
expiresAt: 1760000060,
})
}).pipe(
Effect.provide(
dynamicResponse((input) =>
@@ -92,6 +112,21 @@ describe("experimental Realtime", () => {
})
return input.respond(JSON.stringify({ value: "ek_1", expires_at: 1760000060 }))
}
if (req.url === "https://generativelanguage.googleapis.com/v1beta/auth_tokens") {
expect(req.headers.get("x-goog-api-key")).toBe("gemini-key")
expect(req.headers.get("authorization")).toBeNull()
expect(JSON.parse(input.text)).toEqual({
uses: 1,
expireTime: "2025-10-09T09:23:20.000Z",
newSessionExpireTime: "2025-10-09T08:54:20.000Z",
bidiGenerateContentSetup: {
generationConfig: { responseModalities: ["AUDIO"] },
systemInstruction: { parts: [{ text: "Brief." }] },
model: "models/gemini-3.8-live",
},
})
return input.respond(JSON.stringify({ name: "auth_tokens/ek_3" }))
}
expect(req.url).toBe("https://api.x.ai/v1/realtime/client_secrets")
expect(req.headers.get("authorization")).toBe("Bearer xai-key")
expect(JSON.parse(input.text)).toEqual({ expires_after: { seconds: 60 } })
@@ -104,9 +139,18 @@ describe("experimental Realtime", () => {
test("routes bare and versioned model names to their engine", () => {
expect(
["gpt-realtime", "gpt-realtime-2.1", "gpt-live", "grok-voice-latest"].map((model) => Realtime.engine(model)?.id),
).toEqual(["openai-realtime", "openai-realtime", "openai-live", "xai-realtime"])
[
"gpt-realtime",
"gpt-realtime-2.1",
"gpt-live",
"grok-voice-latest",
"gemini-3.8-live",
"gemini-3.8-live-extended-thinking",
].map((model) => Realtime.engine(model)?.id),
).toEqual(["openai-realtime", "openai-realtime", "openai-live", "xai-realtime", "gemini-live", "gemini-live"])
expect(Realtime.engine("gpt-realtimex")).toBeUndefined()
expect(Realtime.engine("gemini-3.8-flash")).toBeUndefined()
expect(Realtime.engine("gemini-3.5-live")).toBeUndefined()
})
it.effect("rejects unsupported transports before network I/O", () =>
@@ -121,7 +165,16 @@ describe("experimental Realtime", () => {
apiKey: Redacted.make("k"),
transport: { type: "webrtc", sdp: "v=0" },
}).pipe(Effect.flip)
expect([liveWs.reason._tag, grokRtc.reason._tag]).toEqual(["InvalidRequest", "InvalidRequest"])
const geminiRtc = yield* Realtime.connect({
model: "gemini-3.8-live",
apiKey: Redacted.make("k"),
transport: { type: "webrtc", sdp: "v=0" },
}).pipe(Effect.flip)
expect([liveWs.reason._tag, grokRtc.reason._tag, geminiRtc.reason._tag]).toEqual([
"InvalidRequest",
"InvalidRequest",
"InvalidRequest",
])
}).pipe(Effect.provide(dynamicResponse(() => Effect.die("unsupported transport reached the network")))),
)
})
+9 -4
View File
@@ -60,10 +60,7 @@ export const layer = Layer.effect(
if (credential?.type !== "key")
return yield* new UnavailableError({
service: engine.provider,
message:
engine.provider === "openai"
? "Voice needs an OpenAI API key on the server (a stored key or OPENAI_API_KEY). ChatGPT sign-in can't start voice sessions."
: "Voice needs an xAI API key on the server (a stored key or XAI_API_KEY).",
message: missingKeyMessage[engine.provider],
})
return yield* Realtime.connect({ ...input, apiKey: Redacted.make(credential.key) }).pipe(
@@ -86,3 +83,11 @@ export const node = makeLocationNode({
layer,
deps: [Integration.node, requestExecutor],
})
const missingKeyMessage = {
openai:
"Voice needs an OpenAI API key on the server (a stored key or OPENAI_API_KEY). ChatGPT sign-in can't start voice sessions.",
xai: "Voice needs an xAI API key on the server (a stored key or XAI_API_KEY).",
google:
"Voice needs a Google API key on the server (a stored key, GEMINI_API_KEY, GOOGLE_GENERATIVE_AI_API_KEY, or GOOGLE_API_KEY).",
} as const
+30 -7
View File
@@ -5,6 +5,7 @@ import { Integration } from "@opencode/core/integration"
import { Voice } from "@opencode/core/voice"
import { Effect, Layer } from "effect"
import { HttpClient, type HttpClientRequest, HttpClientResponse } from "effect/http"
import { TestClock } from "effect/testing"
import { testEffect } from "./lib/effect"
const it = testEffect(Layer.empty)
@@ -16,16 +17,25 @@ const credential = (id: string, method: "key" | "oauth") => ({
method,
})
const keys: Record<string, string> = { active: "sk-active", stored: "sk-stored", XAI_API_KEY: "xai-env-key" }
const keys: Record<string, string> = {
active: "sk-active",
stored: "sk-stored",
XAI_API_KEY: "xai-env-key",
GEMINI_API_KEY: "gemini-env-key",
}
it.effect("uses the first key or env connection Integration lists and never a ChatGPT sign-in", () =>
Effect.gen(function* () {
yield* TestClock.setTime(1760000000_000)
const auths: string[] = []
const call = (model: string, connections: Integration.Info["connections"]) =>
Effect.flatMap(Voice.Service, (voice) =>
voice.session({
model,
transport: model.startsWith("grok-") ? { type: "websocket" } : { type: "webrtc", sdp: "v=0" },
transport:
model.startsWith("grok-") || model.startsWith("gemini-")
? { type: "websocket" }
: { type: "webrtc", sdp: "v=0" },
}),
).pipe(
Effect.provide(
@@ -71,12 +81,14 @@ it.effect("uses the first key or env connection Integration lists and never a Ch
HttpClient.HttpClient,
HttpClient.make((req: HttpClientRequest.HttpClientRequest) =>
Effect.sync(() => {
auths.push(req.headers.authorization ?? "")
auths.push(req.headers.authorization ?? req.headers["x-goog-api-key"] ?? "")
return HttpClientResponse.fromWeb(
req,
model.startsWith("grok-")
? Response.json({ value: "ek_xai", expires_at: 1760000999 })
: Response.json({ transport: { sdp: "v=0\r\no=answer\r\n" } }),
model.startsWith("gemini-")
? Response.json({ name: "auth_tokens/ek_gemini" })
: model.startsWith("grok-")
? Response.json({ value: "ek_xai", expires_at: 1760000999 })
: Response.json({ transport: { sdp: "v=0\r\no=answer\r\n" } }),
)
}),
),
@@ -92,16 +104,27 @@ it.effect("uses the first key or env connection Integration lists and never a Ch
yield* call("gpt-live-1", [credential("active", "key"), credential("stored", "key")])
yield* call("gpt-live-1", [credential("chatgpt", "oauth"), credential("stored", "key")])
const grok = yield* call("grok-voice-3.0", [{ type: "env", name: "XAI_API_KEY" }])
expect(auths).toEqual(["Bearer sk-active", "Bearer sk-stored", "Bearer xai-env-key"])
const gemini = yield* call("gemini-3.8-live", [{ type: "env", name: "GEMINI_API_KEY" }])
expect(auths).toEqual(["Bearer sk-active", "Bearer sk-stored", "Bearer xai-env-key", "gemini-env-key"])
expect(grok).toEqual({
type: "websocket",
url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0",
token: "ek_xai",
expiresAt: 1760000999,
})
expect(gemini).toEqual({
type: "websocket",
url: "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained",
token: "auth_tokens/ek_gemini",
expiresAt: 1760000060,
})
const signInOnly = yield* call("gpt-live-1", [credential("chatgpt", "oauth")]).pipe(Effect.flip)
expect(signInOnly._tag).toBe("Voice.UnavailableError")
expect(signInOnly.message).toContain("OpenAI API key")
const missingGoogle = yield* call("gemini-3.8-live", []).pipe(Effect.flip)
expect(missingGoogle).toMatchObject({ _tag: "Voice.UnavailableError", service: "google" })
expect(missingGoogle.message).toContain("GEMINI_API_KEY")
}),
)