From 31dc762147190f1803dd52bb0909c5163e6cc467 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Mon, 19 Oct 2026 13:04:04 -0700 Subject: [PATCH] feat(voice): add Gemini Live ephemeral token support (#56888) --- packages/ai/src/experimental/realtime.ts | 40 +++++++++++++++- packages/ai/test/realtime.test.ts | 61 ++++++++++++++++++++++-- packages/core/src/voice.ts | 13 +++-- packages/core/test/voice.test.ts | 37 +++++++++++--- 4 files changed, 134 insertions(+), 17 deletions(-) diff --git a/packages/ai/src/experimental/realtime.ts b/packages/ai/src/experimental/realtime.ts index 42191fedd40..4af26e34701 100644 --- a/packages/ai/src/experimental/realtime.ts +++ b/packages/ai/src/experimental/realtime.ts @@ -1,4 +1,4 @@ -import { Effect, type Redacted, Schema } from "effect" +import { Clock, Effect, Redacted, Schema } from "effect" import { HttpClientRequest } from "effect/http" import { RequestExecutor } from "../route/executor.js" import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "../schema/index.js" @@ -6,6 +6,7 @@ import { encodeJson } from "../utils/json.js" const OPENAI_BASE_URL = "https://api.openai.com/v1" const XAI_BASE_URL = "https://api.x.ai/v1" +const GOOGLE_BASE_URL = "https://generativelanguage.googleapis.com/v1beta" export type RealtimeTransportInput = | { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number } @@ -28,6 +29,7 @@ const engines = [ { family: "gpt-live", id: "openai-live", provider: "openai" }, { family: "gpt-realtime", id: "openai-realtime", provider: "openai" }, { family: "grok-voice", id: "xai-realtime", provider: "xai" }, + { family: "gemini-3.8-live", id: "gemini-live", provider: "google" }, ] as const // Matches the bare family name (`gpt-realtime`) and its versions (`gpt-realtime-2.1`, `gpt-realtime-mini`). @@ -73,6 +75,31 @@ export const connect = Effect.fn("Realtime.connect")(function* (input: RealtimeC return yield* invalidRequest("xAI Grok Voice only supports WebSocket transport") return yield* clientSecret(resolved.id, XAI_BASE_URL, input.model, input.apiKey) } + case "gemini-live": { + if (input.transport.type !== "websocket") + return yield* invalidRequest("Google Gemini Live only supports WebSocket transport") + const now = yield* Clock.currentTimeMillis + const expiresAt = now + 60_000 + const data = yield* post( + resolved.id, + `${GOOGLE_BASE_URL}/auth_tokens`, + input.apiKey, + { + uses: 1, + expireTime: new Date(now + 30 * 60_000).toISOString(), + newSessionExpireTime: new Date(expiresAt).toISOString(), + bidiGenerateContentSetup: { ...input.session, model: `models/${input.model}` }, + }, + AuthTokenResponse, + "x-goog-api-key", + ) + return { + type: "websocket" as const, + url: "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained", + token: data.name, + expiresAt: Math.floor(expiresAt / 1000), + } + } } }) @@ -97,6 +124,12 @@ const ClientSecretResponse = Schema.fromJsonString( }), ) +const AuthTokenResponse = Schema.fromJsonString( + Schema.Struct({ + name: Schema.String, + }), +) + const invalidRequest = (message: string) => new AIError({ reason: new InvalidRequestError({ message }) }) const post = ( @@ -105,12 +138,15 @@ const post = ( apiKey: Redacted.Redacted, body: Record | FormData, response: Schema.Codec, + authHeader?: string, ) => Effect.gen(function* () { const executor = yield* RequestExecutor.Service const res = yield* executor.execute( HttpClientRequest.post(url).pipe( - HttpClientRequest.bearerToken(apiKey), + authHeader + ? HttpClientRequest.setHeader(authHeader, Redacted.value(apiKey)) + : HttpClientRequest.bearerToken(apiKey), body instanceof FormData ? HttpClientRequest.bodyFormData(body) : HttpClientRequest.bodyText(encodeJson(body), "application/json"), diff --git a/packages/ai/test/realtime.test.ts b/packages/ai/test/realtime.test.ts index 61b8be24fbb..2f1a663a5ab 100644 --- a/packages/ai/test/realtime.test.ts +++ b/packages/ai/test/realtime.test.ts @@ -1,13 +1,16 @@ import { describe, expect, test } from "bun:test" import { Effect, Redacted } from "effect" import { HttpClientRequest } from "effect/http" +import { TestClock } from "effect/testing" import { Realtime } from "../src/experimental.js" import { it } from "./lib/effect.js" import { dynamicResponse } from "./lib/http.js" describe("experimental Realtime", () => { - it.effect("connects OpenAI GPT-Live, OpenAI Realtime (WebRTC + WebSocket), and xAI Grok Voice", () => + it.effect("connects OpenAI GPT-Live, OpenAI Realtime, xAI Grok Voice, and Google Gemini Live", () => Effect.gen(function* () { + yield* TestClock.setTime(1760000000_000) + expect( yield* Realtime.connect({ model: "gpt-live-1", @@ -53,6 +56,23 @@ describe("experimental Realtime", () => { token: "ek_2", expiresAt: 1760000120, }) + + expect( + yield* Realtime.connect({ + model: "gemini-3.8-live", + apiKey: Redacted.make("gemini-key"), + session: { + generationConfig: { responseModalities: ["AUDIO"] }, + systemInstruction: { parts: [{ text: "Brief." }] }, + }, + transport: { type: "websocket" }, + }), + ).toEqual({ + type: "websocket", + url: "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained", + token: "auth_tokens/ek_3", + expiresAt: 1760000060, + }) }).pipe( Effect.provide( dynamicResponse((input) => @@ -92,6 +112,21 @@ describe("experimental Realtime", () => { }) return input.respond(JSON.stringify({ value: "ek_1", expires_at: 1760000060 })) } + if (req.url === "https://generativelanguage.googleapis.com/v1beta/auth_tokens") { + expect(req.headers.get("x-goog-api-key")).toBe("gemini-key") + expect(req.headers.get("authorization")).toBeNull() + expect(JSON.parse(input.text)).toEqual({ + uses: 1, + expireTime: "2025-10-09T09:23:20.000Z", + newSessionExpireTime: "2025-10-09T08:54:20.000Z", + bidiGenerateContentSetup: { + generationConfig: { responseModalities: ["AUDIO"] }, + systemInstruction: { parts: [{ text: "Brief." }] }, + model: "models/gemini-3.8-live", + }, + }) + return input.respond(JSON.stringify({ name: "auth_tokens/ek_3" })) + } expect(req.url).toBe("https://api.x.ai/v1/realtime/client_secrets") expect(req.headers.get("authorization")).toBe("Bearer xai-key") expect(JSON.parse(input.text)).toEqual({ expires_after: { seconds: 60 } }) @@ -104,9 +139,18 @@ describe("experimental Realtime", () => { test("routes bare and versioned model names to their engine", () => { expect( - ["gpt-realtime", "gpt-realtime-2.1", "gpt-live", "grok-voice-latest"].map((model) => Realtime.engine(model)?.id), - ).toEqual(["openai-realtime", "openai-realtime", "openai-live", "xai-realtime"]) + [ + "gpt-realtime", + "gpt-realtime-2.1", + "gpt-live", + "grok-voice-latest", + "gemini-3.8-live", + "gemini-3.8-live-extended-thinking", + ].map((model) => Realtime.engine(model)?.id), + ).toEqual(["openai-realtime", "openai-realtime", "openai-live", "xai-realtime", "gemini-live", "gemini-live"]) expect(Realtime.engine("gpt-realtimex")).toBeUndefined() + expect(Realtime.engine("gemini-3.8-flash")).toBeUndefined() + expect(Realtime.engine("gemini-3.5-live")).toBeUndefined() }) it.effect("rejects unsupported transports before network I/O", () => @@ -121,7 +165,16 @@ describe("experimental Realtime", () => { apiKey: Redacted.make("k"), transport: { type: "webrtc", sdp: "v=0" }, }).pipe(Effect.flip) - expect([liveWs.reason._tag, grokRtc.reason._tag]).toEqual(["InvalidRequest", "InvalidRequest"]) + const geminiRtc = yield* Realtime.connect({ + model: "gemini-3.8-live", + apiKey: Redacted.make("k"), + transport: { type: "webrtc", sdp: "v=0" }, + }).pipe(Effect.flip) + expect([liveWs.reason._tag, grokRtc.reason._tag, geminiRtc.reason._tag]).toEqual([ + "InvalidRequest", + "InvalidRequest", + "InvalidRequest", + ]) }).pipe(Effect.provide(dynamicResponse(() => Effect.die("unsupported transport reached the network")))), ) }) diff --git a/packages/core/src/voice.ts b/packages/core/src/voice.ts index 811ea10f691..fdfd664bf65 100644 --- a/packages/core/src/voice.ts +++ b/packages/core/src/voice.ts @@ -60,10 +60,7 @@ export const layer = Layer.effect( if (credential?.type !== "key") return yield* new UnavailableError({ service: engine.provider, - message: - engine.provider === "openai" - ? "Voice needs an OpenAI API key on the server (a stored key or OPENAI_API_KEY). ChatGPT sign-in can't start voice sessions." - : "Voice needs an xAI API key on the server (a stored key or XAI_API_KEY).", + message: missingKeyMessage[engine.provider], }) return yield* Realtime.connect({ ...input, apiKey: Redacted.make(credential.key) }).pipe( @@ -86,3 +83,11 @@ export const node = makeLocationNode({ layer, deps: [Integration.node, requestExecutor], }) + +const missingKeyMessage = { + openai: + "Voice needs an OpenAI API key on the server (a stored key or OPENAI_API_KEY). ChatGPT sign-in can't start voice sessions.", + xai: "Voice needs an xAI API key on the server (a stored key or XAI_API_KEY).", + google: + "Voice needs a Google API key on the server (a stored key, GEMINI_API_KEY, GOOGLE_GENERATIVE_AI_API_KEY, or GOOGLE_API_KEY).", +} as const diff --git a/packages/core/test/voice.test.ts b/packages/core/test/voice.test.ts index 82b045db996..14970f94bbb 100644 --- a/packages/core/test/voice.test.ts +++ b/packages/core/test/voice.test.ts @@ -5,6 +5,7 @@ import { Integration } from "@opencode/core/integration" import { Voice } from "@opencode/core/voice" import { Effect, Layer } from "effect" import { HttpClient, type HttpClientRequest, HttpClientResponse } from "effect/http" +import { TestClock } from "effect/testing" import { testEffect } from "./lib/effect" const it = testEffect(Layer.empty) @@ -16,16 +17,25 @@ const credential = (id: string, method: "key" | "oauth") => ({ method, }) -const keys: Record = { active: "sk-active", stored: "sk-stored", XAI_API_KEY: "xai-env-key" } +const keys: Record = { + active: "sk-active", + stored: "sk-stored", + XAI_API_KEY: "xai-env-key", + GEMINI_API_KEY: "gemini-env-key", +} it.effect("uses the first key or env connection Integration lists and never a ChatGPT sign-in", () => Effect.gen(function* () { + yield* TestClock.setTime(1760000000_000) const auths: string[] = [] const call = (model: string, connections: Integration.Info["connections"]) => Effect.flatMap(Voice.Service, (voice) => voice.session({ model, - transport: model.startsWith("grok-") ? { type: "websocket" } : { type: "webrtc", sdp: "v=0" }, + transport: + model.startsWith("grok-") || model.startsWith("gemini-") + ? { type: "websocket" } + : { type: "webrtc", sdp: "v=0" }, }), ).pipe( Effect.provide( @@ -71,12 +81,14 @@ it.effect("uses the first key or env connection Integration lists and never a Ch HttpClient.HttpClient, HttpClient.make((req: HttpClientRequest.HttpClientRequest) => Effect.sync(() => { - auths.push(req.headers.authorization ?? "") + auths.push(req.headers.authorization ?? req.headers["x-goog-api-key"] ?? "") return HttpClientResponse.fromWeb( req, - model.startsWith("grok-") - ? Response.json({ value: "ek_xai", expires_at: 1760000999 }) - : Response.json({ transport: { sdp: "v=0\r\no=answer\r\n" } }), + model.startsWith("gemini-") + ? Response.json({ name: "auth_tokens/ek_gemini" }) + : model.startsWith("grok-") + ? Response.json({ value: "ek_xai", expires_at: 1760000999 }) + : Response.json({ transport: { sdp: "v=0\r\no=answer\r\n" } }), ) }), ), @@ -92,16 +104,27 @@ it.effect("uses the first key or env connection Integration lists and never a Ch yield* call("gpt-live-1", [credential("active", "key"), credential("stored", "key")]) yield* call("gpt-live-1", [credential("chatgpt", "oauth"), credential("stored", "key")]) const grok = yield* call("grok-voice-3.0", [{ type: "env", name: "XAI_API_KEY" }]) - expect(auths).toEqual(["Bearer sk-active", "Bearer sk-stored", "Bearer xai-env-key"]) + const gemini = yield* call("gemini-3.8-live", [{ type: "env", name: "GEMINI_API_KEY" }]) + expect(auths).toEqual(["Bearer sk-active", "Bearer sk-stored", "Bearer xai-env-key", "gemini-env-key"]) expect(grok).toEqual({ type: "websocket", url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0", token: "ek_xai", expiresAt: 1760000999, }) + expect(gemini).toEqual({ + type: "websocket", + url: "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained", + token: "auth_tokens/ek_gemini", + expiresAt: 1760000060, + }) const signInOnly = yield* call("gpt-live-1", [credential("chatgpt", "oauth")]).pipe(Effect.flip) expect(signInOnly._tag).toBe("Voice.UnavailableError") expect(signInOnly.message).toContain("OpenAI API key") + + const missingGoogle = yield* call("gemini-3.8-live", []).pipe(Effect.flip) + expect(missingGoogle).toMatchObject({ _tag: "Voice.UnavailableError", service: "google" }) + expect(missingGoogle.message).toContain("GEMINI_API_KEY") }), )