diff --git a/packages/ai/README.md b/packages/ai/README.md index 66e2a6384ff..72c77a73fd1 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -198,6 +198,15 @@ OpenAI reads `OPENAI_API_KEY` and calls `/v1/decisions`. The common API uses `boolean`; System One routes lower it to native `noul`, and OpenAI lowers it to `predicate`. OpenAI predicates take no criteria, so boolean criteria fail before the request, and a refused question fails the call with a `ContentPolicy` error. +Cloudflare serves its Clef models through Workers AI and TypeSafe Jev through AI Gateway, both on `/ai/run`: + +```ts +import { CloudflareAIGateway, CloudflareWorkersAI } from "@opencode/ai/providers" + +CloudflareWorkersAI.configure({ accountId }).experimental.evaluation("@cf/cloudflare/clef") +CloudflareAIGateway.configure({ accountId }).experimental.evaluation("typesafe/jev") +``` + Choice and score answers include `confidence` when the provider returns it, such as `response.answers.department.confidence`. Score legends remain available in provider metadata, and the provider's rounded probabilities are returned unchanged. diff --git a/packages/ai/src/experimental/cloudflare-run-evaluation.ts b/packages/ai/src/experimental/cloudflare-run-evaluation.ts new file mode 100644 index 00000000000..fd14859fdf2 --- /dev/null +++ b/packages/ai/src/experimental/cloudflare-run-evaluation.ts @@ -0,0 +1,65 @@ +import { Effect, Schema } from "effect" +import { EvaluationModel, type EvaluationRequestFor, type EvaluationResponse } from "./evaluation.js" +import { SystemOne } from "./system-one.js" +import { Endpoint } from "../route/endpoint.js" +import { MediaProtocol } from "../route/media-protocol.js" +import { MediaRoute } from "../route/media.js" +import type { ProviderID } from "../schema/index.js" + +// Workers AI models answer with the System One response; third-party models such as TypeSafe Jev wrap it in a +// completed run record. +const Envelope = Schema.Struct({ + result: Schema.Union([ + SystemOne.Response, + Schema.Struct({ state: Schema.Literal("Completed"), result: SystemOne.Response }), + ]), +}) + +export interface ModelInput extends MediaRoute.ModelInput { + /** Also the provider metadata key. */ + readonly provider: ProviderID + /** The Run API base, `.../client/v4/accounts/{account}/ai/run`. */ + readonly baseURL: string +} + +/** Posts `{ model, input }` to Cloudflare's `/ai/run` with the System One request body as `input`. */ +export const model = (input: ModelInput) => { + const route = MediaProtocol.identity({ + id: "cloudflare-run-evaluation", + name: "Cloudflare Run", + provider: input.provider, + }) + const decodeEnvelope = route.decodeJson(Envelope) + const composed = MediaRoute.inline({ + protocol: MediaProtocol.inline(route, { + body: { + from: (request) => + SystemOne.encodeInput(request).pipe( + Effect.map((body) => MediaProtocol.json({ model: request.model.id, input: body })), + ), + }, + response: { + decode: (response) => + decodeEnvelope(response).pipe( + Effect.map((output) => + SystemOne.toResponse( + "answers" in output.value.result ? output.value.result : output.value.result.result, + input.provider, + ), + ), + ), + }, + }), + endpoint: Endpoint.path("", { baseURL: input.baseURL }), + auth: input.auth, + headers: input.headers, + }) + return EvaluationModel.make({ + id: input.id, + provider: input.provider, + http: input.http, + route: { id: composed.id, evaluate: composed.generate }, + }) +} + +export const CloudflareRunEvaluation = { model } as const diff --git a/packages/ai/src/experimental/system-one.ts b/packages/ai/src/experimental/system-one.ts index e3be4e109cb..74394d03f0b 100644 --- a/packages/ai/src/experimental/system-one.ts +++ b/packages/ai/src/experimental/system-one.ts @@ -9,6 +9,7 @@ import { ScoreQuestion, type EvaluationAnswer, type EvaluationOptions, + type EvaluationRequestFor, } from "./evaluation.js" import { Auth, type Definition as AuthDefinition } from "../route/auth.js" import { RequestExecutor } from "../route/executor.js" @@ -23,6 +24,7 @@ import { Usage, mergeJsonRecords, } from "../schema/index.js" +import { encodeJson } from "../utils/json.js" const Noul = Schema.Struct({ type: Schema.Literal("noul"), @@ -47,9 +49,8 @@ const Question = Schema.Union([ ), Noul, ]) -const Request = Schema.StructWithRest( +const Input = Schema.StructWithRest( Schema.Struct({ - model: Schema.String, state: EvaluationInput, questions: Schema.Record(Schema.String, Question), }), @@ -79,7 +80,7 @@ const NativeUsage = Schema.StructWithRest( }), [JsonObject], ) -const Response = Schema.Struct({ +export const Response = Schema.Struct({ model: Schema.String, answers: Schema.Record(Schema.String, Answer), usage: Schema.optional(NativeUsage), @@ -98,6 +99,68 @@ export interface ModelInput { readonly http?: HttpOptions } +/** The System One request body without `model`, with `boolean` questions lowered to native `noul`. */ +export const encodeInput = (req: EvaluationRequestFor) => + Schema.encodeUnknownEffect(Input)({ + ...mergeJsonRecords(req.options, req.http?.body), + state: req.state, + questions: Object.fromEntries( + Object.entries(req.questions).map(([id, x]) => [id, x.type === "boolean" ? { ...x, type: "noul" } : x]), + ), + }).pipe( + Effect.mapError((cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) })), + ) + +export const toResponse = (data: typeof Response.Type, providerMetadataKey: string) => { + const legend: Record> = {} + const answers = Object.fromEntries( + Object.entries(data.answers).map(([id, answer]): [string, EvaluationAnswer] => { + if (answer.type === "noul") return [id, { type: "boolean", probability: answer.noul }] + if (answer.type === "choice") { + return [ + id, + { + type: "choice", + choice: answer.choice, + probabilities: answer.probabilities, + ...(answer.confidence === undefined ? {} : { confidence: answer.confidence }), + }, + ] + } + if (answer.legend !== undefined) legend[id] = answer.legend + return [ + id, + { + type: "score", + score: answer.score, + probabilities: answer.probabilities, + ...(answer.confidence === undefined ? {} : { confidence: answer.confidence }), + }, + ] + }), + ) + const meta = { + ...(data.id === undefined ? {} : { responseId: data.id }), + ...(data.provider === undefined ? {} : { provider: data.provider }), + ...data.provider_metadata?.[providerMetadataKey], + ...(Object.keys(legend).length === 0 ? {} : { legend }), + } + return new EvaluationResponse({ + model: ModelID.make(data.model), + answers, + usage: data.usage + ? new Usage({ + inputTokens: data.usage.input_tokens, + outputTokens: data.usage.output_tokens, + totalTokens: ProviderShared.totalTokens(data.usage.input_tokens, data.usage.output_tokens, undefined), + providerMetadata: { [providerMetadataKey]: data.usage }, + }) + : undefined, + rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }), + providerMetadata: Object.keys(meta).length === 0 ? undefined : { [providerMetadataKey]: meta }, + }) +} + export const model = (cfg: ModelInput) => EvaluationModel.make({ id: cfg.id, @@ -109,18 +172,7 @@ export const model = (cfg Effect.gen(function* () { const url = new URL(`${cfg.baseURL.replace(/\/$/, "")}/systemone`) Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value)) - const body = yield* Schema.encodeUnknownEffect(Schema.fromJsonString(Request))({ - ...mergeJsonRecords(req.options, req.http?.body), - model: req.model.id, - state: req.state, - questions: Object.fromEntries( - Object.entries(req.questions).map(([id, x]) => [id, x.type === "boolean" ? { ...x, type: "noul" } : x]), - ), - }).pipe( - Effect.mapError( - (cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }), - ), - ) + const body = encodeJson({ ...(yield* encodeInput(req)), model: req.model.id }) const headers = yield* Auth.toEffect(cfg.auth)({ request: req, method: "POST", @@ -143,60 +195,9 @@ export const model = (cfg const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(text).pipe( Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)), ) - - const legend: Record> = {} - const answers = Object.fromEntries( - Object.entries(data.answers).map(([id, answer]): [string, EvaluationAnswer] => { - if (answer.type === "noul") return [id, { type: "boolean", probability: answer.noul }] - if (answer.type === "choice") { - return [ - id, - { - type: "choice", - choice: answer.choice, - probabilities: answer.probabilities, - ...(answer.confidence === undefined ? {} : { confidence: answer.confidence }), - }, - ] - } - if (answer.legend !== undefined) legend[id] = answer.legend - return [ - id, - { - type: "score", - score: answer.score, - probabilities: answer.probabilities, - ...(answer.confidence === undefined ? {} : { confidence: answer.confidence }), - }, - ] - }), - ) - const meta = { - ...(data.id === undefined ? {} : { responseId: data.id }), - ...(data.provider === undefined ? {} : { provider: data.provider }), - ...data.provider_metadata?.[cfg.providerMetadataKey], - ...(Object.keys(legend).length === 0 ? {} : { legend }), - } - return new EvaluationResponse({ - model: ModelID.make(data.model), - answers, - usage: data.usage - ? new Usage({ - inputTokens: data.usage.input_tokens, - outputTokens: data.usage.output_tokens, - totalTokens: ProviderShared.totalTokens( - data.usage.input_tokens, - data.usage.output_tokens, - undefined, - ), - providerMetadata: { [cfg.providerMetadataKey]: data.usage }, - }) - : undefined, - rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }), - providerMetadata: Object.keys(meta).length === 0 ? undefined : { [cfg.providerMetadataKey]: meta }, - }) + return toResponse(data, cfg.providerMetadataKey) }), }, }) -export const SystemOne = { model } as const +export const SystemOne = { Response, encodeInput, toResponse, model } as const diff --git a/packages/ai/src/providers/cloudflare-ai-gateway.ts b/packages/ai/src/providers/cloudflare-ai-gateway.ts index 7ca071656e9..d9323b89ead 100644 --- a/packages/ai/src/providers/cloudflare-ai-gateway.ts +++ b/packages/ai/src/providers/cloudflare-ai-gateway.ts @@ -1,10 +1,12 @@ import type { Config, Redacted } from "effect" import type { ProviderPackage } from "../provider-package.js" +import { CloudflareRunEvaluation } from "../experimental/cloudflare-run-evaluation.js" import { AnthropicMessages, OpenAIChat, OpenAIResponses } from "../protocols/index.js" import { Auth } from "../route/auth.js" +import { Endpoint } from "../route/endpoint.js" import type { AtLeastOne, ProviderAuthOption } from "../route/auth-options.js" import type { RouteDefaultsInput } from "../route/client.js" -import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js" +import { HttpOptions, ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js" import { encodeJson } from "../utils/json.js" import type { OpenAIProviderOptionsInput } from "./openai-options.js" @@ -46,6 +48,17 @@ export const baseURL = (input: GatewayURL) => { return `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(requireAccountId(input))}/ai/v1` } +/** The Run API (`/ai/run`) next to the OpenAI-compatible `/ai/v1` base. */ +export const runBaseURL = (input: GatewayURL) => { + const url = Endpoint.trimBaseUrl(baseURL(input)) + if (!url.endsWith("/ai/v1")) + throw new ProviderConfigurationError({ + provider: id, + message: "CloudflareAIGateway needs accountId or a baseURL ending in /ai/v1 to derive the Run API base", + }) + return `${url.slice(0, -"/v1".length)}/run` +} + // Cloudflare's REST API rejects parts of the Anthropic Messages and OpenAI Responses request shapes, such as // system prompt blocks and `minimal` reasoning effort, so these models use the provider-native gateway endpoints. const passthroughURL = (input: GatewayURL & GatewayOptions, provider: "anthropic/v1" | "openai") => @@ -118,6 +131,15 @@ export const configure = (input: LanguageModelOptions) => { const responses = responsesRoute.with(passthrough("openai")) const messages = messagesRoute.with(passthrough("anthropic/v1")) const chat = route.with(rest) + const evaluation = (value: string | ModelID) => + CloudflareRunEvaluation.model({ + id: value, + provider: id, + auth: rest.auth, + baseURL: runBaseURL(input), + headers: rest.headers, + http: HttpOptions.make(input.http), + }) return { id, model: (value: string | ModelID) => { @@ -131,6 +153,7 @@ export const configure = (input: LanguageModelOptions) => { return chat.model({ id: value.slice("workers-ai/".length) }) return chat.model({ id: value }) }, + experimental: { evaluation }, configure, } } diff --git a/packages/ai/src/providers/cloudflare-workers-ai.ts b/packages/ai/src/providers/cloudflare-workers-ai.ts index 160093a1de4..935aaa31777 100644 --- a/packages/ai/src/providers/cloudflare-workers-ai.ts +++ b/packages/ai/src/providers/cloudflare-workers-ai.ts @@ -1,9 +1,10 @@ import type { ProviderPackage } from "../provider-package.js" +import { CloudflareRunEvaluation } from "../experimental/cloudflare-run-evaluation.js" import { OpenAIChat } from "../protocols/openai-chat.js" import { AuthOptions, type AtLeastOne, type ProviderAuthOption } from "../route/auth-options.js" import { Route, type RouteDefaultsInput } from "../route/client.js" import { Endpoint } from "../route/endpoint.js" -import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js" +import { HttpOptions, ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js" import type { OpenAIProviderOptionsInput } from "./openai-options.js" export const id = ProviderID.make("cloudflare-workers-ai") @@ -36,6 +37,17 @@ export const baseURL = (input: WorkersAIURL) => { return `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(input.accountId)}/ai/v1` } +/** The Run API (`/ai/run`) next to the OpenAI-compatible `/ai/v1` base. */ +export const runBaseURL = (input: WorkersAIURL) => { + const url = Endpoint.trimBaseUrl(baseURL(input)) + if (!url.endsWith("/ai/v1")) + throw new ProviderConfigurationError({ + provider: id, + message: "CloudflareWorkersAI needs accountId or a baseURL ending in /ai/v1 to derive the Run API base", + }) + return `${url.slice(0, -"/v1".length)}/run` +} + export const route = Route.make({ id: "cloudflare-workers-ai", provider: id, @@ -49,14 +61,21 @@ export const routes = [route] export const configure = (input: LanguageModelOptions) => { const { accountId: _accountId, apiKey: _apiKey, auth: _auth, baseURL: _baseURL, ...defaults } = input - const configured = route.with({ - ...defaults, - endpoint: { baseURL: baseURL(input) }, - auth: AuthOptions.bearer(input, authEnvVars), - }) + const auth = AuthOptions.bearer(input, authEnvVars) + const configured = route.with({ ...defaults, endpoint: { baseURL: baseURL(input) }, auth }) + const evaluation = (modelID: string | ModelID) => + CloudflareRunEvaluation.model({ + id: modelID, + provider: id, + auth, + baseURL: runBaseURL(input), + headers: input.headers, + http: HttpOptions.make(input.http), + }) return { id, model: (modelID: string | ModelID) => configured.model({ id: modelID }), + experimental: { evaluation }, configure, } } diff --git a/packages/ai/test/evaluation.test.ts b/packages/ai/test/evaluation.test.ts index 722de600b5a..83bf295ecb1 100644 --- a/packages/ai/test/evaluation.test.ts +++ b/packages/ai/test/evaluation.test.ts @@ -2,7 +2,7 @@ import { describe, expect } from "bun:test" import { Effect, Layer } from "effect" import { HttpClientRequest } from "effect/http" import { Evaluation, EvaluationClient } from "../src/experimental.js" -import { OpenAI, OpenCode, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js" +import { CloudflareWorkersAI, OpenAI, OpenCode, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js" import { it } from "./lib/effect.js" import { dynamicResponse } from "./lib/http.js" @@ -118,9 +118,10 @@ describe("experimental Evaluation", () => { it.effect("configures the OpenCode System One endpoint", () => Evaluation.run({ - model: OpenCode.configure({ apiKey: "opencode-key", baseURL: "https://opencode.test/v1" }).experimental.evaluation( - "jev-1.13", - ), + model: OpenCode.configure({ + apiKey: "opencode-key", + baseURL: "https://opencode.test/v1", + }).experimental.evaluation("jev-1.13"), state: "hello", questions: { greeting: { type: "boolean", instructions: "Is this a greeting?" } }, }).pipe( @@ -425,6 +426,59 @@ describe("experimental Evaluation", () => { ), ) + it.effect("surfaces Cloudflare Run errors", () => + Evaluation.run({ + model: CloudflareWorkersAI.configure({ + accountId: "test-account", + apiKey: "cloudflare-key", + }).experimental.evaluation("@cf/cloudflare/clef"), + state: "refund", + questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } }, + }).pipe( + Effect.flip, + Effect.tap((error) => + Effect.sync(() => { + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("At least one question is required") + }), + ), + Effect.provide( + EvaluationClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => { + expect(input.request.url).toBe("https://api.cloudflare.com/client/v4/accounts/test-account/ai/run") + expect(input.request.headers.authorization).toBe("Bearer cloudflare-key") + expect(JSON.parse(input.text)).toEqual({ + model: "@cf/cloudflare/clef", + input: { + state: "refund", + questions: { refund: { type: "noul", instructions: "Is a refund requested?" } }, + }, + }) + return Effect.succeed( + input.respond( + JSON.stringify({ + errors: [ + { + message: + "Model execution failed (User Input Error): Validation error at questions: At least one question is required", + code: 7003, + }, + ], + success: false, + result: {}, + messages: [], + }), + { status: 400, headers: { "content-type": "application/json" } }, + ), + ) + }), + ), + ), + ), + ), + ) + it.effect("rejects answers that do not match their questions", () => Evaluation.run({ model: VercelAIGateway.configure({ diff --git a/packages/ai/test/fixtures/recordings/cloudflare-ai-gateway-evaluation/evaluates-choice-score-and-boolean-questions.json b/packages/ai/test/fixtures/recordings/cloudflare-ai-gateway-evaluation/evaluates-choice-score-and-boolean-questions.json new file mode 100644 index 00000000000..fec94484183 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/cloudflare-ai-gateway-evaluation/evaluates-choice-score-and-boolean-questions.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "metadata": { + "model": "typesafe/jev", + "tags": [ + "prefix:cloudflare-ai-gateway-evaluation", + "provider:cloudflare-ai-gateway", + "protocol:system-one" + ], + "name": "cloudflare-ai-gateway-evaluation/evaluates-choice-score-and-boolean-questions", + "recordedAt": "2026-10-23T07:46:54.512Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.cloudflare.com/client/v4/accounts/{account}/ai/run", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"typesafe/jev\",\"input\":{\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "{\"result\":{\"state\":\"Completed\",\"result\":{\"model\":\"jev-1.13.0\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"probabilities\":{\"billing\":1,\"technical\":0,\"sales\":0},\"confidence\":1},\"urgency\":{\"type\":\"score\",\"score\":1.02,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0,\"1\":0.97,\"2\":0.03},\"confidence\":0.95},\"refund\":{\"type\":\"noul\",\"noul\":0.99}},\"usage\":{\"input_tokens\":422,\"output_tokens\":69}}},\"success\":true,\"errors\":[],\"messages\":[]}" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/cloudflare-workers-ai-evaluation/evaluates-choice-score-and-boolean-questions.json b/packages/ai/test/fixtures/recordings/cloudflare-workers-ai-evaluation/evaluates-choice-score-and-boolean-questions.json new file mode 100644 index 00000000000..471aeccefad --- /dev/null +++ b/packages/ai/test/fixtures/recordings/cloudflare-workers-ai-evaluation/evaluates-choice-score-and-boolean-questions.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "metadata": { + "model": "@cf/cloudflare/clef", + "tags": [ + "prefix:cloudflare-workers-ai-evaluation", + "provider:cloudflare-workers-ai", + "protocol:system-one" + ], + "name": "cloudflare-workers-ai-evaluation/evaluates-choice-score-and-boolean-questions", + "recordedAt": "2026-10-23T08:02:48.344Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.cloudflare.com/client/v4/accounts/{account}/ai/run", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"@cf/cloudflare/clef\",\"input\":{\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "{\"result\":{\"model\":\"clef\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"probabilities\":{\"billing\":0.9936,\"technical\":0.0036,\"sales\":0.0028},\"confidence\":0.9807},\"urgency\":{\"type\":\"score\",\"score\":1.3063,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0.0476,\"1\":0.5985,\"2\":0.3539},\"confidence\":0.2286},\"refund\":{\"type\":\"noul\",\"noul\":0.9946}},\"usage\":{\"input_tokens\":355,\"output_tokens\":0}},\"success\":true,\"errors\":[],\"messages\":[]}" + } + } + ] +} diff --git a/packages/ai/test/provider/evaluation.recorded.test.ts b/packages/ai/test/provider/evaluation.recorded.test.ts index 49b3098c45c..e58aff24103 100644 --- a/packages/ai/test/provider/evaluation.recorded.test.ts +++ b/packages/ai/test/provider/evaluation.recorded.test.ts @@ -1,7 +1,14 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" import { Evaluation, EvaluationModel, type EvaluationOptions } from "../../src/experimental.js" -import { OpenAI, OpenCode, OpenRouter, TypeSafeAI } from "../../src/providers.js" +import { + CloudflareAIGateway, + CloudflareWorkersAI, + OpenAI, + OpenCode, + OpenRouter, + TypeSafeAI, +} from "../../src/providers.js" import { recordedTests } from "../recorded-test.js" const questions = { @@ -56,6 +63,28 @@ const openai = recordedTests({ metadata: { model: "gpt-6-luna" }, }) +const cloudflareOptions = { + redact: { url: (url: string) => url.replace(/\/client\/v4\/accounts\/[^/]+\//, "/client/v4/accounts/{account}/") }, +} + +const cloudflareWorkersAI = recordedTests({ + prefix: "cloudflare-workers-ai-evaluation", + provider: "cloudflare-workers-ai", + protocol: "system-one", + requires: ["CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_KEY"], + metadata: { model: "@cf/cloudflare/clef" }, + options: cloudflareOptions, +}) + +const cloudflareAIGateway = recordedTests({ + prefix: "cloudflare-ai-gateway-evaluation", + provider: "cloudflare-ai-gateway", + protocol: "system-one", + requires: ["CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_TOKEN"], + metadata: { model: "typesafe/jev" }, + options: cloudflareOptions, +}) + describe("experimental Evaluation recorded", () => { typesafe.effect("evaluates choice score and boolean questions", () => assertEvaluation( @@ -100,12 +129,31 @@ describe("experimental Evaluation recorded", () => { expect(response.usage?.outputTokens).toBe(0) }), ) + + cloudflareWorkersAI.effect("evaluates choice score and boolean questions", () => + assertEvaluation( + CloudflareWorkersAI.configure({ + accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account", + apiKey: process.env.CLOUDFLARE_API_KEY ?? "fixture", + }).experimental.evaluation("@cf/cloudflare/clef"), + "clef", + ), + ) + + cloudflareAIGateway.effect("evaluates choice score and boolean questions", () => + assertEvaluation( + CloudflareAIGateway.configure({ + accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account", + gatewayApiKey: process.env.CLOUDFLARE_API_TOKEN ?? "fixture", + }).experimental.evaluation("typesafe/jev"), + ), + ) }) -const assertEvaluation = (model: EvaluationModel) => +const assertEvaluation = (model: EvaluationModel, name = "jev-") => Effect.gen(function* () { const response = yield* Evaluation.run({ model, state, questions }) - expect(response.model).toContain("jev-") + expect(response.model).toContain(name) expect(response.answers.department.type).toBe("choice") expect(response.answers.department.choice).toBe("billing") expect(response.answers.department.probabilities?.billing).toBeGreaterThan(0.5) @@ -115,7 +163,7 @@ const assertEvaluation = (model: EvaluationMo expect(response.answers.refund.type).toBe("boolean") expect(response.answers.refund.probability).toBeGreaterThan(0.5) expect(response.usage?.inputTokens).toBeGreaterThan(0) - expect(response.usage?.outputTokens).toBeGreaterThan(0) + expect(response.usage?.outputTokens).toBeDefined() expect(response.answers.department.confidence).toBeGreaterThan(0) expect(response.answers.urgency.confidence).toBeGreaterThan(0) })