mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-15 04:27:20 +03:00
feat(ai): add Google embedContent embeddings (#62843)
This commit is contained in:
committed by
GitHub
Unverified
parent
2a9a3ed97c
commit
fc7e9223de
@@ -409,6 +409,31 @@ OpenAICompatible.configure({ provider: "nvidia", baseURL: "https://integrate.api
|
||||
Every host here uses OpenAI's `/embeddings` API; Cohere through its compatibility API
|
||||
(`https://api.cohere.ai/compatibility/v1`). Package entry points are `@opencode/ai/providers/<provider>/embeddings`.
|
||||
|
||||
Google embeds through `embedContent`, which also takes images, audio, video and PDFs on `gemini-embedding-2`. Each
|
||||
value is its own content and gets its own vector (several parts in one content would be averaged into one). `dimensions`
|
||||
is `outputDimensionality`, `inputType` becomes `taskType` `RETRIEVAL_QUERY` or `RETRIEVAL_DOCUMENT`, and `taskType` or
|
||||
`title` in `providerOptions` replace or add to it. `gemini-embedding-2` ignores task types; Google recommends writing the
|
||||
task into the text instead (`task: search result | query: …`, `title: none | text: …`).
|
||||
|
||||
```ts
|
||||
import { Google, GoogleVertex } from "@opencode/ai/providers"
|
||||
|
||||
const response = await ai.embedding.generate({
|
||||
model: Google.configure().embedding("gemini-embedding-2"),
|
||||
values: [await ai.file("./receipt.png"), "A receipt for two coffees."],
|
||||
dimensions: 768,
|
||||
})
|
||||
|
||||
GoogleVertex.configure({ project, location: "global" }).embedding("gemini-embedding-2")
|
||||
```
|
||||
|
||||
The Gemini API sends up to 100 values per `batchEmbedContents` request. Vertex `embedContent` takes one value per
|
||||
request, so a call sends one request per value, four at a time, and needs project credentials: Vertex Express Mode API
|
||||
keys can't call `embedContent`. `gemini-embedding-2` runs in the `global`, `us` and `eu` Vertex locations. When a call
|
||||
is split into several requests, the vectors stay in value order, `usage.inputTokens` is their sum, and each request's
|
||||
metadata is listed in order under `providerMetadata.<provider>.responses`. Google has no `/embeddings` entry point yet;
|
||||
use `embeddingModel` from `@opencode/ai/providers/google` or `@opencode/ai/providers/google-vertex`.
|
||||
|
||||
## Alibaba Cloud Model Studio
|
||||
|
||||
`Alibaba` provides standard Model Studio inference. Configure a region explicitly, then select
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { EmbeddingResponse, EmbeddingUsage, type EmbeddingRequest } from "../embedding.js"
|
||||
import { EmbeddingRoute } from "../route/embedding.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { mergeJsonRecords, type AIError, type OpenString } from "../schema/index.js"
|
||||
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
|
||||
|
||||
const ID = "google-embed-content"
|
||||
const NAME = "Google embedContent"
|
||||
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
/** `batchEmbedContents` takes at most 100 requests. */
|
||||
export const MAX_BATCH = 100
|
||||
|
||||
export type TaskType = OpenString<
|
||||
| "RETRIEVAL_QUERY"
|
||||
| "RETRIEVAL_DOCUMENT"
|
||||
| "SEMANTIC_SIMILARITY"
|
||||
| "CLASSIFICATION"
|
||||
| "CLUSTERING"
|
||||
| "QUESTION_ANSWERING"
|
||||
| "FACT_VERIFICATION"
|
||||
| "CODE_RETRIEVAL_QUERY"
|
||||
>
|
||||
|
||||
/**
|
||||
* Per-value embedding fields, sent with each value. `taskType` replaces the one `inputType` maps to; Gemini Embedding 2
|
||||
* ignores task types and documents task prefixes in the text instead (`task: search result | query: …`).
|
||||
*/
|
||||
export interface OptionsInput {
|
||||
readonly [key: string]: unknown
|
||||
readonly taskType?: TaskType
|
||||
readonly title?: string
|
||||
}
|
||||
|
||||
/** Vertex `embedContentConfig` fields; Vertex alone takes `autoTruncate`, `documentOcr` and `audioTrackExtraction`. */
|
||||
export interface VertexOptionsInput extends OptionsInput {
|
||||
readonly autoTruncate?: boolean
|
||||
readonly documentOcr?: boolean
|
||||
readonly audioTrackExtraction?: boolean
|
||||
}
|
||||
|
||||
const Values = Schema.Struct({ values: Schema.Array(Schema.Number) })
|
||||
|
||||
const UsageMetadata = Schema.StructWithRest(Schema.Struct({ promptTokenCount: Schema.optional(Schema.Number) }), [
|
||||
Schema.Record(Schema.String, Schema.Unknown),
|
||||
])
|
||||
|
||||
const BatchResponse = Schema.Struct({
|
||||
embeddings: Schema.Array(Values),
|
||||
usageMetadata: Schema.optional(UsageMetadata),
|
||||
})
|
||||
|
||||
const SingleResponse = Schema.Struct({
|
||||
embedding: Values,
|
||||
usageMetadata: Schema.optional(UsageMetadata),
|
||||
truncated: Schema.optional(Schema.Boolean),
|
||||
})
|
||||
|
||||
const TASK_TYPES = { query: "RETRIEVAL_QUERY", document: "RETRIEVAL_DOCUMENT" } as const
|
||||
|
||||
// Each value is its own content: several parts in one content embed into one aggregated vector.
|
||||
const content = (value: EmbeddingRequest["values"][number]): Effect.Effect<{ readonly parts: [unknown] }, AIError> =>
|
||||
typeof value === "string"
|
||||
? Effect.succeed({ parts: [{ text: value }] })
|
||||
: GeminiGenerateContent.mediaPart(NAME, value).pipe(Effect.map((part) => ({ parts: [part] })))
|
||||
|
||||
const config = (request: EmbeddingRequest) => ({
|
||||
...(request.inputType === undefined ? {} : { taskType: TASK_TYPES[request.inputType] }),
|
||||
...(request.dimensions === undefined ? {} : { outputDimensionality: request.dimensions }),
|
||||
...request.providerOptions,
|
||||
})
|
||||
|
||||
const identity = MediaProtocol.identity({ id: ID, name: NAME, provider: "google" })
|
||||
const decodeBatch = identity.decodeJson(BatchResponse)
|
||||
const decodeSingle = identity.decodeJson(SingleResponse)
|
||||
|
||||
const usage = (metadata: Schema.Schema.Type<typeof UsageMetadata> | undefined) =>
|
||||
metadata?.promptTokenCount === undefined ? undefined : new EmbeddingUsage({ inputTokens: metadata.promptTokenCount })
|
||||
|
||||
/** Gemini API `batchEmbedContents`: one request per value, so every value gets its own vector. */
|
||||
export const protocol = EmbeddingRoute.protocol({
|
||||
id: ID,
|
||||
name: NAME,
|
||||
body: {
|
||||
from: Effect.fn("GoogleEmbedContent.fromRequest")(function* (request: EmbeddingRequest) {
|
||||
const contents = yield* Effect.forEach(request.values, content)
|
||||
const fields = config(request)
|
||||
return (
|
||||
mergeJsonRecords(
|
||||
{ requests: contents.map((item) => ({ model: `models/${request.model.id}`, content: item, ...fields })) },
|
||||
request.http?.body,
|
||||
) ?? {}
|
||||
)
|
||||
}),
|
||||
},
|
||||
response: {
|
||||
decode: Effect.fn("GoogleEmbedContent.decodeResponse")(function* (response, context) {
|
||||
const output = yield* decodeBatch(response)
|
||||
return new EmbeddingResponse({
|
||||
embeddings: output.value.embeddings.map((item) => item.values),
|
||||
usage: usage(output.value.usageMetadata),
|
||||
providerMetadata: {
|
||||
[context.providerMetadataKey]:
|
||||
output.value.usageMetadata === undefined ? {} : { usageMetadata: output.value.usageMetadata },
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
/**
|
||||
* Vertex `embedContent`: one content per request, with options under `embedContentConfig`. Routes using it set
|
||||
* `maxValues: 1`.
|
||||
*/
|
||||
export const vertexProtocol = EmbeddingRoute.protocol({
|
||||
id: "google-vertex-embed-content",
|
||||
name: "Google Vertex embedContent",
|
||||
body: {
|
||||
from: Effect.fn("GoogleEmbedContent.vertexFromRequest")(function* (request: EmbeddingRequest) {
|
||||
const contents = yield* Effect.forEach(request.values, content)
|
||||
const fields = config(request)
|
||||
return (
|
||||
mergeJsonRecords(
|
||||
{
|
||||
content: contents[0],
|
||||
...(Object.keys(fields).length === 0 ? {} : { embedContentConfig: fields }),
|
||||
},
|
||||
request.http?.body,
|
||||
) ?? {}
|
||||
)
|
||||
}),
|
||||
},
|
||||
response: {
|
||||
decode: Effect.fn("GoogleEmbedContent.vertexDecodeResponse")(function* (response, context) {
|
||||
const output = yield* decodeSingle(response)
|
||||
return new EmbeddingResponse({
|
||||
embeddings: [output.value.embedding.values],
|
||||
usage: usage(output.value.usageMetadata),
|
||||
providerMetadata: {
|
||||
[context.providerMetadataKey]: {
|
||||
...(output.value.usageMetadata === undefined ? {} : { usageMetadata: output.value.usageMetadata }),
|
||||
...(output.value.truncated === undefined ? {} : { truncated: output.value.truncated }),
|
||||
},
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
export const route = EmbeddingRoute.make({
|
||||
id: ID,
|
||||
provider: "google",
|
||||
protocol,
|
||||
endpoint: Endpoint.path(({ request }) => `/models/${request.model.id}:batchEmbedContents`, {
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
}),
|
||||
maxValues: MAX_BATCH,
|
||||
})
|
||||
|
||||
export const GoogleEmbedContent = {
|
||||
protocol,
|
||||
vertexProtocol,
|
||||
route,
|
||||
} as const
|
||||
@@ -3,6 +3,7 @@ export * as BedrockConverse from "./bedrock-converse.js"
|
||||
export * as CloudflareRunDecisions from "./cloudflare-run-decisions.js"
|
||||
export * as CohereChat from "./cohere-chat.js"
|
||||
export * as Gemini from "./gemini.js"
|
||||
export * as GoogleEmbedContent from "./google-embed-content.js"
|
||||
export * as GoogleInteractions from "./google-interactions.js"
|
||||
export * as MistralChat from "./mistral-chat.js"
|
||||
export * as OpenAIChat from "./openai-chat.js"
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { Effect } from "effect"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { Gemini } from "../protocols/gemini.js"
|
||||
import { GoogleEmbedContent, type VertexOptionsInput } from "../protocols/google-embed-content.js"
|
||||
import { GoogleImages } from "../protocols/google-images.js"
|
||||
import { ProviderShared } from "../protocols/shared.js"
|
||||
import { Auth } from "../route/auth.js"
|
||||
import { Route } from "../route/client.js"
|
||||
import { EmbeddingRoute } from "../route/embedding.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import {
|
||||
@@ -82,6 +84,18 @@ const route = Route.make({
|
||||
|
||||
export const routes = [route]
|
||||
|
||||
export type { VertexOptionsInput as GoogleVertexEmbeddingOptions } from "../protocols/google-embed-content.js"
|
||||
|
||||
const embeddingRoute = EmbeddingRoute.make({
|
||||
id: "google-vertex-embed-content",
|
||||
provider: id,
|
||||
providerMetadataKey: "vertex",
|
||||
protocol: GoogleEmbedContent.vertexProtocol,
|
||||
endpoint: Endpoint.path(({ request }) => `/models/${request.model.id}:embedContent`),
|
||||
// Vertex embedContent embeds one content per request.
|
||||
maxValues: 1,
|
||||
})
|
||||
|
||||
const configuredRoute = (input: Config, modelID: string | ModelID) => {
|
||||
const {
|
||||
accessToken: _accessToken,
|
||||
@@ -129,6 +143,18 @@ export const configure = (input: Config = {}) => {
|
||||
providerOptions,
|
||||
})
|
||||
},
|
||||
/** `gemini-embedding-2` through Vertex `embedContent`; it needs project credentials, not an Express Mode key. */
|
||||
embedding: (modelID: string | ModelID, providerOptions?: ProviderOptions) => {
|
||||
const target = connection(input, modelID)
|
||||
if (GoogleVertexShared.apiKey(input) !== undefined && input.baseURL === undefined)
|
||||
throw new ProviderConfigurationError({
|
||||
provider: id,
|
||||
message: "Google Vertex embedContent does not support Express Mode API keys; use project credentials",
|
||||
})
|
||||
return embeddingRoute
|
||||
.with({ endpoint: target.endpoint, auth: target.auth, headers: input.headers, http: input.http })
|
||||
.model<VertexOptionsInput>({ id: modelID, providerOptions })
|
||||
},
|
||||
configure,
|
||||
}
|
||||
}
|
||||
@@ -143,6 +169,10 @@ export const imageModel = (modelID: string, settings?: Settings) => {
|
||||
const input = config(settings)
|
||||
return configure(input).image(modelID, input.providerOptions)
|
||||
}
|
||||
export const embeddingModel = (modelID: string, settings?: Settings) => {
|
||||
const input = config(settings)
|
||||
return configure(input).embedding(modelID, input.providerOptions)
|
||||
}
|
||||
|
||||
function config({ accessToken, apiKey, baseURL, body, headers, location, project, ...providerOptions }: Settings = {}) {
|
||||
if (apiKey !== undefined && accessToken !== undefined)
|
||||
|
||||
@@ -4,6 +4,10 @@ import { MediaRoute } from "../route/media.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { ProviderID, type ModelID, type ProviderOptions, type RequestOptions } from "../schema/index.js"
|
||||
import { Gemini } from "../protocols/gemini.js"
|
||||
import {
|
||||
GoogleEmbedContent,
|
||||
type OptionsInput as GoogleEmbeddingOptionsInput,
|
||||
} from "../protocols/google-embed-content.js"
|
||||
import { GoogleInteractions } from "../protocols/google-interactions.js"
|
||||
import { GoogleImages } from "../protocols/google-images.js"
|
||||
import { GoogleSpeech } from "../protocols/google-speech.js"
|
||||
@@ -11,6 +15,7 @@ import { GoogleTranscription } from "../protocols/google-transcription.js"
|
||||
import { GoogleVideo } from "../protocols/google-video.js"
|
||||
import { toConfig } from "./settings.js"
|
||||
|
||||
export type { OptionsInput as GoogleEmbeddingOptions } from "../protocols/google-embed-content.js"
|
||||
export type { GoogleImageOptions } from "../protocols/google-images.js"
|
||||
export type { GoogleSpeechOptions } from "../protocols/google-speech.js"
|
||||
export type { GoogleTranscriptionOptions } from "../protocols/google-transcription.js"
|
||||
@@ -52,6 +57,13 @@ export const configure = (input: Config = {}) => {
|
||||
const route = configuredRoute(input)
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
const gemini = (modelID: string | ModelID) => route.model<Gemini.ProviderOptionsInput>({ id: modelID })
|
||||
const embeddingRoute = GoogleEmbedContent.route.with({
|
||||
provider: id,
|
||||
endpoint: { baseURL: input.baseURL },
|
||||
auth: auth(input),
|
||||
headers: input.headers,
|
||||
http: input.http,
|
||||
})
|
||||
return {
|
||||
id,
|
||||
model: gemini,
|
||||
@@ -63,6 +75,8 @@ export const configure = (input: Config = {}) => {
|
||||
video: (modelID: string | ModelID) => GoogleVideo.model({ ...media, id: modelID }),
|
||||
speech: (modelID: string | ModelID) => GoogleSpeech.model({ ...media, id: modelID }),
|
||||
transcription: (modelID: string | ModelID) => GoogleTranscription.model({ ...media, id: modelID }),
|
||||
embedding: (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
|
||||
embeddingRoute.model<GoogleEmbeddingOptionsInput>({ id: modelID, providerOptions }),
|
||||
configure,
|
||||
}
|
||||
}
|
||||
@@ -74,8 +88,13 @@ export const imageModel = (modelID: string, settings?: Settings) => {
|
||||
const input = toConfig(settings)
|
||||
return configure(input).image(modelID, input.providerOptions)
|
||||
}
|
||||
export const embeddingModel = (modelID: string, settings?: Settings) => {
|
||||
const input = toConfig(settings)
|
||||
return configure(input).embedding(modelID, input.providerOptions)
|
||||
}
|
||||
|
||||
export const image = provider.image
|
||||
export const embedding = provider.embedding
|
||||
export const gemini = provider.gemini
|
||||
export const interactions = provider.interactions
|
||||
export const video = provider.video
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { Effect } from "effect"
|
||||
import type { HttpClientResponse } from "effect/http"
|
||||
import { EmbeddingModel, type EmbeddingOptions, type EmbeddingRequest, type EmbeddingResponse } from "../embedding.js"
|
||||
import {
|
||||
EmbeddingModel,
|
||||
EmbeddingResponse,
|
||||
EmbeddingUsage,
|
||||
type EmbeddingOptions,
|
||||
type EmbeddingRequest,
|
||||
} from "../embedding.js"
|
||||
import { ProviderID, type HttpOptions, type AIError, type ProviderOptions } from "../schema/index.js"
|
||||
import { Auth } from "./auth.js"
|
||||
import { Endpoint } from "./endpoint.js"
|
||||
@@ -31,6 +37,11 @@ export interface Patch {
|
||||
readonly auth?: Auth.Definition
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions
|
||||
/**
|
||||
* Most values one HTTP request may carry. Longer requests are split into requests of this size, sent a few at a
|
||||
* time, and joined back in value order.
|
||||
*/
|
||||
readonly maxValues?: number
|
||||
}
|
||||
|
||||
export interface Route {
|
||||
@@ -42,6 +53,7 @@ export interface Route {
|
||||
readonly auth: Auth.Definition
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions
|
||||
readonly maxValues?: number
|
||||
readonly with: (patch: Patch) => Route
|
||||
readonly model: <Options extends EmbeddingOptions = EmbeddingOptions>(input: {
|
||||
readonly id: string
|
||||
@@ -67,6 +79,7 @@ export const make = (input: MakeInput): Route => {
|
||||
auth: input.auth ?? Auth.none,
|
||||
headers: input.headers,
|
||||
http: input.http,
|
||||
maxValues: input.maxValues,
|
||||
}
|
||||
return {
|
||||
...route,
|
||||
@@ -80,6 +93,7 @@ export const make = (input: MakeInput): Route => {
|
||||
endpoint: patch.endpoint === undefined ? route.endpoint : Endpoint.merge(route.endpoint, patch.endpoint),
|
||||
headers: patch.headers === undefined ? route.headers : { ...route.headers, ...patch.headers },
|
||||
http: patch.http ?? route.http,
|
||||
maxValues: patch.maxValues ?? route.maxValues,
|
||||
}),
|
||||
model: <Options extends EmbeddingOptions>(model: {
|
||||
readonly id: string
|
||||
@@ -90,11 +104,51 @@ export const make = (input: MakeInput): Route => {
|
||||
provider,
|
||||
http: route.http,
|
||||
providerOptions: model.providerOptions,
|
||||
route: { id: route.id, generate: transport(route).generate },
|
||||
route: { id: route.id, generate: split(route.maxValues, transport(route).generate) },
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
const CONCURRENCY = 4
|
||||
|
||||
/**
|
||||
* Splits a request whose values exceed `maxValues` and joins the parts: vectors in value order, `inputTokens` summed
|
||||
* when every part reports it, and each part's metadata in order under `providerMetadata[key].responses`.
|
||||
*/
|
||||
const split =
|
||||
(maxValues: number | undefined, generate: Generate): Generate =>
|
||||
(request, execute) => {
|
||||
if (maxValues === undefined || request.values.length <= maxValues) return generate(request, execute)
|
||||
const chunks = Array.from({ length: Math.ceil(request.values.length / maxValues) }, (_, index) =>
|
||||
request.values.slice(index * maxValues, (index + 1) * maxValues),
|
||||
)
|
||||
return Effect.forEach(chunks, (values) => generate({ ...request, values }, execute), {
|
||||
concurrency: CONCURRENCY,
|
||||
}).pipe(
|
||||
Effect.map((responses) => {
|
||||
const tokens = responses.map((response) => response.usage?.inputTokens)
|
||||
const keys = [...new Set(responses.flatMap((response) => Object.keys(response.providerMetadata ?? {})))]
|
||||
return new EmbeddingResponse({
|
||||
embeddings: responses.flatMap((response) => response.embeddings),
|
||||
usage: tokens.every((count) => count !== undefined)
|
||||
? new EmbeddingUsage({ inputTokens: tokens.reduce((sum, count) => sum + count, 0) })
|
||||
: undefined,
|
||||
providerMetadata:
|
||||
keys.length === 0
|
||||
? undefined
|
||||
: Object.fromEntries(
|
||||
keys.map((key) => [
|
||||
key,
|
||||
{ responses: responses.map((response) => response.providerMetadata?.[key] ?? {}) },
|
||||
]),
|
||||
),
|
||||
})
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
type Generate = EmbeddingModel["route"]["generate"]
|
||||
|
||||
// The HTTP plumbing (URL, query, auth, headers, `http` overlays, surrogate sanitizing) is the one media routes use.
|
||||
const transport = (route: Omit<Route, "with" | "model">) =>
|
||||
MediaRoute.inline({
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { ai, EmbeddingClient, EmbeddingUsage, Media } from "../src/effect.js"
|
||||
import { OpenAI, OpenAICompatible } from "../src/providers.js"
|
||||
import { Google, GoogleVertex, OpenAI, OpenAICompatible } from "../src/providers.js"
|
||||
import { embeddingModel as googleEmbeddingModel } from "../src/providers/google.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
import { dynamicResponse, json, type Handler } from "./lib/http.js"
|
||||
|
||||
@@ -130,4 +131,181 @@ describe("Embedding", () => {
|
||||
expect(error.message).toContain("Expected 2 embeddings, got 1")
|
||||
}).pipe(Effect.provide(client((input) => Effect.succeed(json(input, { data: [{ index: 0, embedding: [0.1] }] }))))),
|
||||
)
|
||||
|
||||
it.effect("batches Google values as one content each and maps inputType to taskType", () =>
|
||||
ai.embedding
|
||||
.generate({
|
||||
model: Google.configure({ apiKey: "google-key" }).embedding("gemini-embedding-2"),
|
||||
values: ["a caption", Media.base64("iVBORw0KGgo=", "image/png")],
|
||||
dimensions: 256,
|
||||
inputType: "document",
|
||||
providerOptions: { title: "Restroom" },
|
||||
})
|
||||
.pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => {
|
||||
expect(response.embeddings).toEqual([
|
||||
[0.1, 0.2],
|
||||
[0.3, 0.4],
|
||||
])
|
||||
expect(response.usage).toEqual(new EmbeddingUsage({ inputTokens: 260 }))
|
||||
expect(response.providerMetadata).toEqual({ google: { usageMetadata: { promptTokenCount: 260 } } })
|
||||
}),
|
||||
),
|
||||
Effect.provide(
|
||||
client((input) => {
|
||||
expect(input.request.url).toBe(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-2:batchEmbedContents",
|
||||
)
|
||||
expect(input.request.headers["x-goog-api-key"]).toBe("google-key")
|
||||
const fields = { taskType: "RETRIEVAL_DOCUMENT", outputDimensionality: 256, title: "Restroom" }
|
||||
expect(JSON.parse(input.text)).toEqual({
|
||||
requests: [
|
||||
{ model: "models/gemini-embedding-2", content: { parts: [{ text: "a caption" }] }, ...fields },
|
||||
{
|
||||
model: "models/gemini-embedding-2",
|
||||
content: { parts: [{ inlineData: { mimeType: "image/png", data: "iVBORw0KGgo=" } }] },
|
||||
...fields,
|
||||
},
|
||||
],
|
||||
})
|
||||
return Effect.succeed(
|
||||
json(input, {
|
||||
embeddings: [{ values: [0.1, 0.2] }, { values: [0.3, 0.4] }],
|
||||
usageMetadata: { promptTokenCount: 260 },
|
||||
}),
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("lets a Google taskType option replace the inputType mapping and sends settings", () =>
|
||||
ai.embedding
|
||||
.generate({
|
||||
model: googleEmbeddingModel("gemini-embedding-001", {
|
||||
apiKey: "google-key",
|
||||
headers: { "x-test": "yes" },
|
||||
body: { extra: true },
|
||||
}),
|
||||
values: ["query"],
|
||||
inputType: "query",
|
||||
providerOptions: { taskType: "CODE_RETRIEVAL_QUERY" },
|
||||
})
|
||||
.pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => {
|
||||
expect(response.usage).toBeUndefined()
|
||||
expect(response.providerMetadata).toEqual({ google: {} })
|
||||
}),
|
||||
),
|
||||
Effect.provide(
|
||||
client((input) => {
|
||||
expect(input.request.headers["x-test"]).toBe("yes")
|
||||
expect(JSON.parse(input.text)).toEqual({
|
||||
requests: [
|
||||
{
|
||||
model: "models/gemini-embedding-001",
|
||||
content: { parts: [{ text: "query" }] },
|
||||
taskType: "CODE_RETRIEVAL_QUERY",
|
||||
},
|
||||
],
|
||||
extra: true,
|
||||
})
|
||||
return Effect.succeed(json(input, { embeddings: [{ values: [1] }] }))
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("splits more than 100 Google values across batches and joins them in order", () =>
|
||||
Effect.gen(function* () {
|
||||
const sizes: Array<number> = []
|
||||
const response = yield* ai.embedding
|
||||
.generate({
|
||||
model: Google.configure({ apiKey: "google-key" }).embedding("gemini-embedding-2"),
|
||||
values: Array.from({ length: 105 }, (_, index) => `${index}`),
|
||||
})
|
||||
.pipe(
|
||||
Effect.provide(
|
||||
client((input) => {
|
||||
const requests: Array<{ content: { parts: Array<{ text: string }> } }> = JSON.parse(input.text).requests
|
||||
sizes.push(requests.length)
|
||||
return Effect.succeed(
|
||||
json(input, {
|
||||
embeddings: requests.map((request) => ({ values: [Number(request.content.parts[0].text)] })),
|
||||
usageMetadata: { promptTokenCount: requests.length },
|
||||
}),
|
||||
)
|
||||
}),
|
||||
),
|
||||
)
|
||||
expect(sizes.toSorted((a, b) => a - b)).toEqual([5, 100])
|
||||
expect(response.embeddings).toEqual(Array.from({ length: 105 }, (_, index) => [index]))
|
||||
expect(response.usage).toEqual(new EmbeddingUsage({ inputTokens: 105 }))
|
||||
expect(response.providerMetadata).toEqual({
|
||||
google: {
|
||||
responses: [{ usageMetadata: { promptTokenCount: 100 } }, { usageMetadata: { promptTokenCount: 5 } }],
|
||||
},
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("sends one Vertex embedContent request per value with embedContentConfig", () =>
|
||||
Effect.gen(function* () {
|
||||
const bodies: Array<unknown> = []
|
||||
const response = yield* ai.embedding
|
||||
.generate({
|
||||
model: GoogleVertex.configure({ accessToken: "vertex-token", project: "proj", location: "us" }).embedding(
|
||||
"gemini-embedding-2",
|
||||
),
|
||||
values: ["first", Media.base64("JVBERi0=", "application/pdf")],
|
||||
dimensions: 128,
|
||||
providerOptions: { documentOcr: true },
|
||||
})
|
||||
.pipe(
|
||||
Effect.provide(
|
||||
client((input) => {
|
||||
expect(input.request.url).toBe(
|
||||
"https://aiplatform.us.rep.googleapis.com/v1beta1/projects/proj/locations/us/publishers/google/models/gemini-embedding-2:embedContent",
|
||||
)
|
||||
expect(input.request.headers.authorization).toBe("Bearer vertex-token")
|
||||
const body = JSON.parse(input.text)
|
||||
bodies.push(body)
|
||||
return Effect.succeed(
|
||||
json(input, {
|
||||
embedding: { values: ["text" in body.content.parts[0] ? 1 : 2] },
|
||||
usageMetadata: { promptTokenCount: 3 },
|
||||
truncated: false,
|
||||
}),
|
||||
)
|
||||
}),
|
||||
),
|
||||
)
|
||||
expect(bodies).toContainEqual({
|
||||
content: { parts: [{ text: "first" }] },
|
||||
embedContentConfig: { outputDimensionality: 128, documentOcr: true },
|
||||
})
|
||||
expect(bodies).toContainEqual({
|
||||
content: { parts: [{ inlineData: { mimeType: "application/pdf", data: "JVBERi0=" } }] },
|
||||
embedContentConfig: { outputDimensionality: 128, documentOcr: true },
|
||||
})
|
||||
expect(response.embeddings).toEqual([[1], [2]])
|
||||
expect(response.usage).toEqual(new EmbeddingUsage({ inputTokens: 6 }))
|
||||
expect(response.providerMetadata).toEqual({
|
||||
vertex: {
|
||||
responses: [
|
||||
{ usageMetadata: { promptTokenCount: 3 }, truncated: false },
|
||||
{ usageMetadata: { promptTokenCount: 3 }, truncated: false },
|
||||
],
|
||||
},
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
test("rejects Vertex Express Mode keys for embedContent", () => {
|
||||
expect(() => GoogleVertex.configure({ apiKey: "vertex-key" }).embedding("gemini-embedding-2")).toThrow(
|
||||
"does not support Express Mode API keys",
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
+32
File diff suppressed because one or more lines are too long
Vendored
+32
File diff suppressed because one or more lines are too long
+50
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:google-embedding",
|
||||
"provider:google",
|
||||
"protocol:google-embed-content"
|
||||
],
|
||||
"name": "google-embedding/maps-inputtype-to-a-task-type-for-gemini-embedding-001",
|
||||
"recordedAt": "2026-11-09T06:56:10.031Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"requests\":[{\"model\":\"models/gemini-embedding-001\",\"content\":{\"parts\":[{\"text\":\"The invoice was charged twice.\"}]},\"taskType\":\"RETRIEVAL_QUERY\",\"outputDimensionality\":64}]}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "application/json; charset=UTF-8"
|
||||
},
|
||||
"body": "{\n \"embeddings\": [\n {\n \"values\": [\n 0.04013612,\n -0.01537641,\n -0.00894363,\n -0.045427274,\n 0.015747963,\n -0.009066882,\n 0.034106165,\n 0.008299592,\n -0.029671155,\n 0.0073614777,\n 0.011705497,\n -0.008064136,\n -0.012201166,\n -0.023549316,\n 0.12342585,\n 0.011068751,\n -0.003649386,\n -0.0046033487,\n -0.011378856,\n 0.008659822,\n -0.024897752,\n 0.001241405,\n -0.0016047506,\n 0.032963928,\n 0.014408621,\n -0.011485396,\n 0.011547241,\n -0.0087663345,\n 0.029018002,\n 0.01399287,\n -0.0005734975,\n 0.009806568,\n -0.033623748,\n 0.013897123,\n 0.027592098,\n -0.001526048,\n 0.024996622,\n -0.0028200862,\n -0.0016486795,\n -0.019535935,\n -0.0060296683,\n 0.020810796,\n -0.014712436,\n 0.0027340506,\n -0.021106219,\n -0.022350833,\n 0.0039701443,\n 0.010802266,\n -0.0124504585,\n -0.0040887143,\n -0.0062737963,\n -0.018470604,\n 0.021161897,\n -0.16764373,\n 0.018159335,\n -0.015530227,\n -0.025055906,\n -0.026749635,\n 0.002768487,\n -0.03824891,\n -0.010511027,\n 0.014685062,\n -0.015881633,\n -0.042950973\n ]\n }\n ]\n}\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"requests\":[{\"model\":\"models/gemini-embedding-001\",\"content\":{\"parts\":[{\"text\":\"The invoice was charged twice.\"}]},\"taskType\":\"RETRIEVAL_DOCUMENT\",\"outputDimensionality\":64}]}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "application/json; charset=UTF-8"
|
||||
},
|
||||
"body": "{\n \"embeddings\": [\n {\n \"values\": [\n 0.0067660585,\n -0.008564701,\n 0.006918798,\n -0.0653632,\n 0.010584768,\n 0.0019771366,\n 0.024187071,\n 0.00468459,\n -0.022482483,\n -0.0011979828,\n 0.01025464,\n -0.012994005,\n 0.010501545,\n -0.008400968,\n 0.1354296,\n -0.003842612,\n -0.009868842,\n 0.0045481804,\n -0.012495539,\n 0.010945229,\n -0.012774331,\n 0.0018108432,\n 0.004564442,\n 0.024119604,\n 0.0096224435,\n 0.0008170349,\n 0.013087967,\n 0.0028655464,\n 0.038301487,\n 0.013128475,\n 0.008370898,\n 0.01710832,\n -0.02379502,\n 0.006764786,\n 0.028454835,\n 0.01613345,\n 0.013660678,\n 0.0013027737,\n 0.003783348,\n -0.0053537926,\n -0.0019457041,\n 0.022436744,\n -0.021374583,\n 0.0048431614,\n 0.0014370559,\n -0.0071425945,\n -0.0075106425,\n 0.0035781835,\n -0.002726382,\n -0.0017613088,\n 0.00242477,\n -0.012590718,\n 0.0015917314,\n -0.23083314,\n 0.006775061,\n -0.013197291,\n -0.020559328,\n -0.01887775,\n 0.011951808,\n -0.03615789,\n -0.017971825,\n 0.011554768,\n -0.017077072,\n -0.033402666\n ]\n }\n ]\n}\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { ai, type EmbeddingModel, type EmbeddingOptions } from "../../src/effect.js"
|
||||
import { Cohere, OpenAI } from "../../src/providers.js"
|
||||
import { ai, Media, type EmbeddingModel, type EmbeddingOptions } from "../../src/effect.js"
|
||||
import { Cohere, Google, OpenAI } from "../../src/providers.js"
|
||||
import { recordedTests } from "../recorded-test.js"
|
||||
|
||||
const values = ["The invoice was charged twice.", "Refund the duplicate payment."]
|
||||
@@ -22,6 +22,15 @@ const cohere = recordedTests({
|
||||
metadata: { model: "embed-v4.0" },
|
||||
})
|
||||
|
||||
const google = recordedTests({
|
||||
prefix: "google-embedding",
|
||||
provider: "google",
|
||||
protocol: "google-embed-content",
|
||||
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
|
||||
})
|
||||
|
||||
const googleProvider = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture" })
|
||||
|
||||
describe("Embedding recorded", () => {
|
||||
openai.effect("embeds text with requested dimensions", () =>
|
||||
assertEmbedding(
|
||||
@@ -32,6 +41,44 @@ describe("Embedding recorded", () => {
|
||||
cohere.effect("embeds text through the compatibility API", () =>
|
||||
assertEmbedding(Cohere.configure({ apiKey: process.env.COHERE_API_KEY ?? "fixture" }).embedding("embed-v4.0")),
|
||||
)
|
||||
|
||||
google.effect("embeds text through batchEmbedContents", () =>
|
||||
assertEmbedding(googleProvider.embedding("gemini-embedding-2")),
|
||||
)
|
||||
|
||||
google.effect("embeds an image and text into one vector each", () =>
|
||||
Effect.gen(function* () {
|
||||
const image = Media.bytes(
|
||||
new Uint8Array(
|
||||
yield* Effect.promise(() => Bun.file(`${import.meta.dir}/../fixtures/images/edit-source.jpg`).arrayBuffer()),
|
||||
),
|
||||
"image/jpeg",
|
||||
)
|
||||
const response = yield* ai.embedding.generate({
|
||||
model: googleProvider.embedding("gemini-embedding-2"),
|
||||
values: [image, "A small square image."],
|
||||
dimensions: 128,
|
||||
})
|
||||
expect(response.embeddings).toHaveLength(2)
|
||||
for (const vector of response.embeddings) expect(vector).toHaveLength(128)
|
||||
expect(response.usage?.inputTokens).toBeGreaterThan(258)
|
||||
}),
|
||||
)
|
||||
|
||||
google.effect("maps inputType to a task type for gemini-embedding-001", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = googleProvider.embedding("gemini-embedding-001")
|
||||
const query = yield* ai.embedding.generate({ model, values: [values[0]], dimensions: 64, inputType: "query" })
|
||||
const document = yield* ai.embedding.generate({
|
||||
model,
|
||||
values: [values[0]],
|
||||
dimensions: 64,
|
||||
inputType: "document",
|
||||
})
|
||||
expect(query.embeddings[0]).toHaveLength(64)
|
||||
expect(query.embeddings[0]).not.toEqual(document.embeddings[0])
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
const assertEmbedding = <Options extends EmbeddingOptions>(model: EmbeddingModel<Options>) =>
|
||||
|
||||
Reference in New Issue
Block a user