diff --git a/packages/ai/README.md b/packages/ai/README.md index e343a941b92..b6b32643954 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -29,6 +29,47 @@ await Effect.runPromise(program.pipe(Effect.provide(llmLayer))) Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment. +## Experimental evaluation + +Evaluation models compare shared state with typed choice, score, and boolean questions. The API is +isolated under an experimental entrypoint and provider namespace while the contract evolves: + +```ts +import { Effect } from "effect" +import { Evaluation, EvaluationClient } from "@opencode/ai/experimental" +import { TypeSafeAI } from "@opencode/ai/providers" + +const model = TypeSafeAI.configure().experimental.evaluation("jev-latest") + +const program = Evaluation.evaluate({ + model, + state: "I was charged twice. Please refund the duplicate payment.", + questions: { + department: { + type: "choice", + instructions: "Which team should handle this?", + criteria: { billing: "Payments and refunds", technical: "Bugs and outages" }, + }, + urgency: { + type: "score", + instructions: "How urgent is this?", + criteria: ["Can wait", "Needs prompt attention", "Blocking revenue"], + }, + refund: { type: "boolean", instructions: "Is the customer asking for a refund?" }, + }, +}) + +const response = await Effect.runPromise(program.pipe(Effect.provide(EvaluationClient.fetchLayer))) + +console.log(response.answers.department.choice) +console.log(response.answers.refund.probability) +``` + +`TypeSafeAI` reads `TYPESAFE_API_KEY`. `OpenCodeZen` exposes the same selector and reads +`OPENCODE_API_KEY`. The common API uses `boolean`; System One routes lower it to native `noul`. +Choice and score confidence plus score legends remain available in provider metadata, and the +provider's rounded probabilities are returned unchanged. + ## Alibaba Cloud Model Studio `Alibaba` provides standard Model Studio inference. Configure a region explicitly, then select diff --git a/packages/ai/src/experimental.ts b/packages/ai/src/experimental.ts new file mode 100644 index 00000000000..50b54a41113 --- /dev/null +++ b/packages/ai/src/experimental.ts @@ -0,0 +1,29 @@ +export { EvaluationClient } from "./experimental/evaluation-client.js" +export { + BooleanAnswer, + BooleanQuestion, + ChoiceAnswer, + ChoiceQuestion, + Evaluation, + EvaluationAnswer, + EvaluationInput, + EvaluationModel, + EvaluationModelSchema, + EvaluationQuestion, + EvaluationRequest, + EvaluationResponse, + EvaluationRounding, + ScoreAnswer, + ScoreQuestion, +} from "./experimental/evaluation.js" +export type { + AnswerFor, + AnswersFor, + EvaluationModelOptions, + EvaluationOptions, + EvaluationQuestions, + EvaluationRequestFor, + EvaluationRequestInput, + EvaluationResponseFor, + EvaluationRoute, +} from "./experimental/evaluation.js" diff --git a/packages/ai/src/experimental/evaluation-client.ts b/packages/ai/src/experimental/evaluation-client.ts new file mode 100644 index 00000000000..b6be4722efc --- /dev/null +++ b/packages/ai/src/experimental/evaluation-client.ts @@ -0,0 +1,54 @@ +import { Context, Effect, Layer } from "effect" +import { RequestExecutor } from "../route/executor.js" +import { mergeHttpOptions, type AIError } from "../schema/index.js" +import { sanitizeSurrogates } from "../utils/sanitize.js" +import { + type EvaluationOptions, + type EvaluationQuestions, + type EvaluationRequestFor, + type EvaluationResponseFor, +} from "./evaluation.js" + +export type Execute = RequestExecutor.Interface["execute"] + +export interface Interface { + readonly evaluate: ( + request: EvaluationRequestFor, + ) => Effect.Effect, AIError> +} + +export class Service extends Context.Service()("@opencode/AI/Experimental/EvaluationClient") {} + +export const evaluate = ( + request: EvaluationRequestFor, +): Effect.Effect, AIError, Service> => + Effect.flatMap(Service, (client) => client.evaluate(request)) + +export const layer: Layer.Layer = Layer.effect( + Service, + Effect.gen(function* () { + const executor = yield* RequestExecutor.Service + return Service.of({ + evaluate: (request) => + request.model.route.evaluate( + { + ...sanitizeSurrogates({ + ...request, + model: undefined, + http: mergeHttpOptions(request.model.http, request.http), + }), + model: request.model, + }, + executor.execute, + ), + }) + }), +) +export const fetchLayer = layer.pipe(Layer.provide(RequestExecutor.fetchLayer)) + +export const EvaluationClient = { + Service, + layer, + fetchLayer, + evaluate, +} as const diff --git a/packages/ai/src/experimental/evaluation.ts b/packages/ai/src/experimental/evaluation.ts new file mode 100644 index 00000000000..b7f591fb95c --- /dev/null +++ b/packages/ai/src/experimental/evaluation.ts @@ -0,0 +1,245 @@ +import { Effect, Schema } from "effect" +import { + AIError, + HttpOptions, + InvalidRequestError, + ModelID, + ProviderID, + ProviderMetadata, + Usage, +} from "../schema/index.js" +import { EvaluationClient, Service, type Execute } from "./evaluation-client.js" + +export const EvaluationInput = Schema.Union([Schema.String, Schema.JsonObject, Schema.Array(Schema.Json)]) +export type EvaluationInput = Schema.Schema.Type + +const EvaluationCriterion = Schema.NullOr(EvaluationInput) +const ChoiceCriteria = Schema.Record(Schema.String, EvaluationCriterion).pipe( + Schema.refine((x): x is typeof x => Object.keys(x).length > 0, { + message: "Choice criteria must be a nonempty option map", + }), +) + +export const ChoiceQuestion = Schema.Struct({ + type: Schema.Literal("choice"), + instructions: EvaluationInput, + criteria: ChoiceCriteria, +}) +export type ChoiceQuestion = Schema.Schema.Type + +export const ScoreQuestion = Schema.Struct({ + type: Schema.Literal("score"), + instructions: EvaluationInput, + criteria: Schema.Array(EvaluationCriterion).check(Schema.isMinLength(2)), +}) +export type ScoreQuestion = Schema.Schema.Type + +export const BooleanQuestion = Schema.Struct({ + type: Schema.Literal("boolean"), + instructions: EvaluationInput, + criteria: Schema.optional( + Schema.Struct({ + true: Schema.optional(EvaluationCriterion), + false: Schema.optional(EvaluationCriterion), + }), + ), +}) +export type BooleanQuestion = Schema.Schema.Type + +export const EvaluationQuestion = Schema.Union([ChoiceQuestion, ScoreQuestion, BooleanQuestion]).pipe( + Schema.toTaggedUnion("type"), +) +export type EvaluationQuestion = Schema.Schema.Type +export type EvaluationQuestions = Readonly> +const EvaluationQuestions = Schema.Record(Schema.String, EvaluationQuestion).pipe( + Schema.refine((x): x is typeof x => Object.keys(x).length > 0, { + message: "Evaluation questions must be a nonempty map", + }), +) + +const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 })) + +export const ChoiceAnswer = Schema.Struct({ + type: Schema.Literal("choice"), + choice: Schema.String, + probabilities: Schema.optional(Schema.Record(Schema.String, Probability)), +}) +export type ChoiceAnswer = Schema.Schema.Type + +export const ScoreAnswer = Schema.Struct({ + type: Schema.Literal("score"), + score: Schema.Number, + probabilities: Schema.optional(Schema.Record(Schema.String, Probability)), +}) +export type ScoreAnswer = Schema.Schema.Type + +export const BooleanAnswer = Schema.Struct({ + type: Schema.Literal("boolean"), + probability: Probability, +}) +export type BooleanAnswer = Schema.Schema.Type + +export const EvaluationAnswer = Schema.Union([ChoiceAnswer, ScoreAnswer, BooleanAnswer]).pipe( + Schema.toTaggedUnion("type"), +) +export type EvaluationAnswer = Schema.Schema.Type + +export type AnswerFor = Question extends { + readonly type: "choice" + readonly criteria: infer Criteria +} + ? { + readonly type: "choice" + readonly choice: Extract + readonly probabilities?: Readonly, number>> + } + : Question extends { readonly type: "score" } + ? ScoreAnswer + : BooleanAnswer + +export type AnswersFor = { + readonly [ID in keyof Questions]: AnswerFor +} + +export type EvaluationOptions = Record + +export interface EvaluationRoute { + readonly id: string + readonly evaluate: ( + request: EvaluationRequestFor, + execute: Execute, + ) => Effect.Effect, AIError> +} + +export class EvaluationModel { + declare protected readonly _Options: (options: Options) => Options + readonly id: ModelID + readonly provider: ProviderID + readonly route: EvaluationRoute + readonly http?: HttpOptions + + constructor(input: EvaluationModel.Input) { + this.id = input.id + this.provider = input.provider + this.route = input.route + this.http = input.http + } + + static make(input: EvaluationModel.MakeInput) { + return new EvaluationModel({ + id: ModelID.make(input.id), + provider: ProviderID.make(input.provider), + route: input.route, + http: input.http, + }) + } +} + +export namespace EvaluationModel { + export interface Input { + readonly id: ModelID + readonly provider: ProviderID + readonly route: EvaluationRoute + readonly http?: HttpOptions + } + + export interface MakeInput + extends Omit, "id" | "provider"> { + readonly id: string | ModelID + readonly provider: string | ProviderID + } +} + +export const EvaluationModelSchema = Schema.declare( + (value): value is EvaluationModel => value instanceof EvaluationModel, + { + expected: "Evaluation.Model", + }, +) + +export class EvaluationRequest extends Schema.Class("Evaluation.Request")({ + model: EvaluationModelSchema, + state: EvaluationInput, + questions: EvaluationQuestions, + options: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + http: Schema.optional(HttpOptions), +}) { + declare protected readonly _EvaluationRequest: void +} + +export type EvaluationModelOptions = Model extends EvaluationModel ? Options : never + +export type EvaluationRequestFor< + Options extends EvaluationOptions = EvaluationOptions, + Questions extends EvaluationQuestions = EvaluationQuestions, +> = Omit & { + readonly model: EvaluationModel + readonly questions: Questions + readonly options?: Options +} + +export type EvaluationRequestInput< + Model extends object = EvaluationModel, + Questions extends EvaluationQuestions = EvaluationQuestions, +> = Omit[0], "model" | "questions" | "options" | "http"> & { + readonly model: Model + readonly questions: Questions + readonly options?: NoInfer> + readonly http?: HttpOptions.Input +} & (Model extends EvaluationModel> ? unknown : never) + +export class EvaluationRounding extends Schema.Class("Evaluation.Rounding")({ + probabilityDecimals: Schema.optional(Schema.Int), + scoreDecimals: Schema.optional(Schema.Int), +}) {} + +export class EvaluationResponse extends Schema.Class("Evaluation.Response")({ + model: ModelID, + answers: Schema.Record(Schema.String, EvaluationAnswer), + usage: Schema.optional(Usage), + rounding: Schema.optional(EvaluationRounding), + providerMetadata: Schema.optional(ProviderMetadata), +}) {} + +export type EvaluationResponseFor = Omit & { + readonly answers: AnswersFor +} + +export function request( + input: EvaluationRequestInput, +): EvaluationRequestFor, Questions> +export function request(input: EvaluationRequest): EvaluationRequest +export function request(input: EvaluationRequest | EvaluationRequestInput) { + if (input instanceof EvaluationRequest) return input + return new EvaluationRequest({ + ...input, + model: input.model as unknown as EvaluationModel, + http: input.http === undefined ? undefined : HttpOptions.make(input.http), + }) +} + +export function evaluate( + input: EvaluationRequestInput, +): Effect.Effect, AIError, Service> +export function evaluate(input: EvaluationRequest): Effect.Effect +export function evaluate(input: EvaluationRequest | EvaluationRequestInput) { + return Effect.try({ + try: () => (input instanceof EvaluationRequest ? input : request(input)), + catch: (cause) => + new AIError({ + reason: new InvalidRequestError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }), + }), + }).pipe( + Effect.flatMap((request) => + EvaluationClient.evaluate(request as EvaluationRequestFor), + ), + ) +} + +export const Evaluation = { + request, + evaluate, +} as const diff --git a/packages/ai/src/experimental/system-one.ts b/packages/ai/src/experimental/system-one.ts new file mode 100644 index 00000000000..0ea8085d5c8 --- /dev/null +++ b/packages/ai/src/experimental/system-one.ts @@ -0,0 +1,222 @@ +import { Effect, Schema } from "effect" +import { Headers, HttpClientRequest } from "effect/unstable/http" +import { + BooleanAnswer, + ChoiceAnswer, + ChoiceQuestion, + EvaluationInput, + EvaluationModel, + EvaluationResponse, + EvaluationRounding, + ScoreAnswer, + ScoreQuestion, + type EvaluationResponseFor, + type EvaluationRoute, +} from "./evaluation.js" +import { Auth, type Definition as AuthDefinition } from "../route/auth.js" +import { + AIError, + HttpContext, + HttpOptions, + InvalidProviderOutputError, + InvalidRequestError, + ModelID, + Usage, + mergeJsonRecords, +} from "../schema/index.js" + +const Noul = Schema.Struct({ + type: Schema.Literal("noul"), + instructions: EvaluationInput, + criteria: Schema.optional( + Schema.Struct({ + true: Schema.optional(Schema.NullOr(EvaluationInput)), + false: Schema.optional(Schema.NullOr(EvaluationInput)), + }), + ), +}) +const Question = Schema.Union([ + ChoiceQuestion.pipe( + Schema.refine((x): x is typeof x => Object.keys(x.criteria).length <= 255, { + message: "System One Choice questions support at most 255 options", + }), + ), + ScoreQuestion.pipe( + Schema.refine((x): x is typeof x => x.criteria.length <= 10, { + message: "System One Score questions support at most 10 levels", + }), + ), + Noul, +]) +const Request = Schema.StructWithRest( + Schema.Struct({ + model: Schema.String, + state: EvaluationInput, + questions: Schema.Record(Schema.String, Question), + }), + [Schema.Record(Schema.String, Schema.Any)], +) + +const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 })) +const NoulAnswer = Schema.Struct({ type: Schema.Literal("noul"), noul: Probability }) +const Choice = Schema.Struct({ + type: Schema.Literal("choice"), + choice: Schema.String, + probabilities: Schema.Record(Schema.String, Probability), + confidence: Schema.optional(Probability), +}) +const Score = Schema.Struct({ + type: Schema.Literal("score"), + score: Schema.Number, + probabilities: Schema.Record(Schema.String, Probability), + legend: Schema.optional(Schema.Record(Schema.String, Schema.Json)), + confidence: Schema.optional(Probability), +}) +const NativeUsage = Schema.Struct({ + input_tokens: Schema.optional(Schema.Number), + output_tokens: Schema.optional(Schema.Number), +}) + +const encode = Schema.encodeUnknownEffect(Schema.fromJsonString(Request)) +const exact = (x: Readonly>, keys: ReadonlyArray) => + Object.keys(x).length === keys.length && keys.every((key) => Object.hasOwn(x, key)) + +export interface ModelInput { + readonly id: string | ModelID + readonly provider: string + readonly providerMetadataKey: string + readonly auth: AuthDefinition + readonly baseURL: string + readonly headers?: Record + readonly http?: HttpOptions +} + +export const model = (cfg: ModelInput) => { + const route: EvaluationRoute = { + id: "system-one", + evaluate: (req, send) => + Effect.gen(function* () { + const url = new URL(`${cfg.baseURL.replace(/\/$/, "")}/systemone`) + Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value)) + const body = yield* encode({ + ...mergeJsonRecords(req.options, req.http?.body), + model: req.model.id, + state: req.state, + questions: Object.fromEntries( + Object.entries(req.questions).map(([id, x]) => [id, x.type === "boolean" ? { ...x, type: "noul" } : x]), + ), + }).pipe( + Effect.mapError( + (cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }), + ), + ) + const headers = yield* Auth.toEffect(cfg.auth)({ + request: req, + method: "POST", + url: url.toString(), + body, + headers: Headers.fromInput({ ...cfg.headers, ...req.http?.headers }), + }) + const res = yield* send( + HttpClientRequest.post(url).pipe( + HttpClientRequest.setHeaders(headers), + HttpClientRequest.bodyText(body, "application/json"), + ), + ) + const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers }) + const fail = (message: string, cause: unknown, body?: string) => + new AIError({ reason: new InvalidProviderOutputError({ route: route.id, message, body, http, cause }) }) + const text = yield* res.text.pipe( + Effect.mapError((cause) => fail("Failed to read the System One response", cause)), + ) + const entries = Object.entries(req.questions) + const output = Schema.Struct({ + model: Schema.String, + answers: Schema.Struct( + Object.fromEntries( + entries.map(([id, question]) => { + if (question.type === "boolean") return [id, NoulAnswer] + if (question.type === "choice") { + const keys = Object.keys(question.criteria) + return [ + id, + Choice.pipe( + Schema.refine( + (x): x is typeof x => + Object.hasOwn(question.criteria, x.choice) && exact(x.probabilities, keys), + { message: `Question "${id}" returned an invalid choice answer` }, + ), + ), + ] + } + const keys = question.criteria.map((_, index) => String(index)) + return [ + id, + Score.pipe( + Schema.refine( + (x): x is typeof x => + x.score >= 0 && x.score <= question.criteria.length - 1 && exact(x.probabilities, keys), + { message: `Question "${id}" returned an invalid score answer` }, + ), + ), + ] + }), + ) as Record>, + ), + usage: Schema.optional(NativeUsage), + }) + const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(output))(text).pipe( + Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)), + ) + + const confidence: Record = {} + const legend: Record> = {} + const answers = Object.fromEntries( + entries.map(([id, question]) => { + const answer = data.answers[id] + if (question.type === "boolean") + return [ + id, + { type: "boolean", probability: (answer as typeof NoulAnswer.Type).noul } satisfies BooleanAnswer, + ] + if (question.type === "choice") { + const value = answer as typeof Choice.Type + if (value.confidence !== undefined) confidence[id] = value.confidence + return [ + id, + { type: "choice", choice: value.choice, probabilities: value.probabilities } satisfies ChoiceAnswer, + ] + } + const value = answer as typeof Score.Type + if (value.confidence !== undefined) confidence[id] = value.confidence + if (value.legend !== undefined) legend[id] = value.legend + return [id, { type: "score", score: value.score, probabilities: value.probabilities } satisfies ScoreAnswer] + }), + ) + const meta = { + ...(Object.keys(confidence).length === 0 ? {} : { confidence }), + ...(Object.keys(legend).length === 0 ? {} : { legend }), + } + return new EvaluationResponse({ + model: ModelID.make(data.model), + answers, + usage: data.usage + ? new Usage({ + inputTokens: data.usage.input_tokens, + outputTokens: data.usage.output_tokens, + totalTokens: + data.usage.input_tokens === undefined && data.usage.output_tokens === undefined + ? undefined + : (data.usage.input_tokens ?? 0) + (data.usage.output_tokens ?? 0), + providerMetadata: { [cfg.providerMetadataKey]: data.usage }, + }) + : undefined, + rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }), + providerMetadata: Object.keys(meta).length === 0 ? undefined : { [cfg.providerMetadataKey]: meta }, + }) as EvaluationResponseFor + }), + } + return EvaluationModel.make({ id: cfg.id, provider: cfg.provider, route, http: cfg.http }) +} + +export const SystemOne = { model } as const diff --git a/packages/ai/src/providers/index.ts b/packages/ai/src/providers/index.ts index f17beaf11d9..28cb2c361e6 100644 --- a/packages/ai/src/providers/index.ts +++ b/packages/ai/src/providers/index.ts @@ -24,8 +24,10 @@ export * as Moonshot from "./moonshot.js" export * as OpenAI from "./openai.js" export * as OpenAICompatible from "./openai-compatible.js" export * as OpenAICompatibleResponses from "./openai-compatible-responses.js" +export * as OpenCodeZen from "./opencode-zen.js" export * as OpenRouter from "./openrouter.js" export * as TogetherAI from "./togetherai.js" +export * as TypeSafeAI from "./typesafe-ai.js" export * as XAI from "./xai.js" export * as ZAI from "./zai.js" export * as ZAICodingPlan from "./zai-coding-plan.js" diff --git a/packages/ai/src/providers/opencode-zen.ts b/packages/ai/src/providers/opencode-zen.ts new file mode 100644 index 00000000000..ea9f9ef854e --- /dev/null +++ b/packages/ai/src/providers/opencode-zen.ts @@ -0,0 +1,31 @@ +import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js" +import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js" +import { SystemOne } from "../experimental/system-one.js" + +export const id = ProviderID.make("opencode") +const baseURL = "https://opencode.ai/zen/v1" + +export type Options = ProviderAuthOption<"optional"> & { + readonly baseURL?: string + readonly headers?: Record + readonly http?: HttpOptions.Input +} + +export const configure = (input: Options = {}) => { + const evaluation = (modelID: string | ModelID) => + SystemOne.model({ + id: modelID, + provider: id, + providerMetadataKey: "opencode", + auth: AuthOptions.bearer(input, "OPENCODE_API_KEY"), + baseURL: input.baseURL ?? baseURL, + headers: input.headers, + http: input.http === undefined ? undefined : HttpOptions.make(input.http), + }) + return { id, experimental: { evaluation }, configure } +} + +export const provider = configure() +export const experimental = provider.experimental + +export * as OpenCodeZen from "./opencode-zen.js" diff --git a/packages/ai/src/providers/typesafe-ai.ts b/packages/ai/src/providers/typesafe-ai.ts new file mode 100644 index 00000000000..971549d429a --- /dev/null +++ b/packages/ai/src/providers/typesafe-ai.ts @@ -0,0 +1,31 @@ +import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js" +import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js" +import { SystemOne } from "../experimental/system-one.js" + +export const id = ProviderID.make("typesafe-ai") +const baseURL = "https://api.typesafe.ai/v1" + +export type Options = ProviderAuthOption<"optional"> & { + readonly baseURL?: string + readonly headers?: Record + readonly http?: HttpOptions.Input +} + +export const configure = (input: Options = {}) => { + const evaluation = (modelID: string | ModelID) => + SystemOne.model({ + id: modelID, + provider: id, + providerMetadataKey: "typesafe", + auth: AuthOptions.bearer(input, "TYPESAFE_API_KEY"), + baseURL: input.baseURL ?? baseURL, + headers: input.headers, + http: input.http === undefined ? undefined : HttpOptions.make(input.http), + }) + return { id, experimental: { evaluation }, configure } +} + +export const provider = configure() +export const experimental = provider.experimental + +export * as TypeSafeAI from "./typesafe-ai.js" diff --git a/packages/ai/test/evaluation.test.ts b/packages/ai/test/evaluation.test.ts new file mode 100644 index 00000000000..65d6d5ec63e --- /dev/null +++ b/packages/ai/test/evaluation.test.ts @@ -0,0 +1,173 @@ +import { describe, expect } from "bun:test" +import { Effect, Layer } from "effect" +import { HttpClientRequest } from "effect/unstable/http" +import { Evaluation, EvaluationClient } from "../src/experimental.js" +import { OpenCodeZen, TypeSafeAI } from "../src/providers.js" +import { it } from "./lib/effect.js" +import { dynamicResponse } from "./lib/http.js" + +describe("experimental Evaluation", () => { + it.effect("evaluates typed questions through System One", () => + Effect.gen(function* () { + const response = yield* Evaluation.evaluate({ + model: TypeSafeAI.configure({ + apiKey: "test", + baseURL: "https://typesafe.test/v1/", + headers: { "x-default": "yes" }, + http: { body: { deployment: "test" }, query: { api: "v1" } }, + }).experimental.evaluation("jev-latest"), + state: { ticket: "Please refund the duplicate charge." }, + questions: { + department: { + type: "choice", + instructions: "Which team should handle this?", + criteria: { billing: "Payments and refunds", technical: "Bugs and outages" }, + }, + urgency: { + type: "score", + instructions: "How urgent is this?", + criteria: ["Can wait", "Needs attention", "Blocking"], + }, + refund: { type: "boolean", instructions: "Is the customer asking for a refund?" }, + }, + options: { trace: { enabled: true } }, + http: { body: { request_metadata: "value" }, headers: { "x-request": "yes" }, query: { trace: "1" } }, + }) + + expect(response.model).toBe("jev-1.13.0") + expect(response.answers.department).toEqual({ + type: "choice", + choice: "billing", + probabilities: { billing: 0.9, technical: 0.1 }, + }) + expect(response.answers.urgency).toEqual({ + type: "score", + score: 1.2, + probabilities: { "0": 0, "1": 0.8, "2": 0.2 }, + }) + expect(response.answers.refund).toEqual({ type: "boolean", probability: 0.97 }) + expect(response.usage?.totalTokens).toBe(36) + expect(response.providerMetadata).toEqual({ + typesafe: { + confidence: { department: 0.8, urgency: 0.6 }, + legend: { urgency: { "0": "Can wait", "1": "Needs attention", "2": "Blocking" } }, + }, + }) + }).pipe( + Effect.provide( + EvaluationClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => + Effect.gen(function* () { + const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie) + expect(request.url).toBe("https://typesafe.test/v1/systemone?api=v1&trace=1") + expect(request.headers.get("authorization")).toBe("Bearer test") + expect(request.headers.get("x-default")).toBe("yes") + expect(request.headers.get("x-request")).toBe("yes") + expect(JSON.parse(input.text)).toEqual({ + deployment: "test", + request_metadata: "value", + trace: { enabled: true }, + model: "jev-latest", + state: { ticket: "Please refund the duplicate charge." }, + questions: { + department: { + type: "choice", + instructions: "Which team should handle this?", + criteria: { billing: "Payments and refunds", technical: "Bugs and outages" }, + }, + urgency: { + type: "score", + instructions: "How urgent is this?", + criteria: ["Can wait", "Needs attention", "Blocking"], + }, + refund: { type: "noul", instructions: "Is the customer asking for a refund?" }, + }, + }) + return input.respond( + JSON.stringify({ + model: "jev-1.13.0", + answers: { + department: { + type: "choice", + choice: "billing", + probabilities: { billing: 0.9, technical: 0.1 }, + confidence: 0.8, + }, + urgency: { + type: "score", + score: 1.2, + probabilities: { "0": 0, "1": 0.8, "2": 0.2 }, + legend: { "0": "Can wait", "1": "Needs attention", "2": "Blocking" }, + confidence: 0.6, + }, + refund: { type: "noul", noul: 0.97 }, + }, + usage: { input_tokens: 30, output_tokens: 6 }, + }), + { headers: { "content-type": "application/json" } }, + ) + }), + ), + ), + ), + ), + ), + ) + + it.effect("configures the OpenCode Zen System One endpoint", () => + Evaluation.evaluate({ + model: OpenCodeZen.configure({ apiKey: "zen-key", baseURL: "https://zen.test/v1" }).experimental.evaluation( + "jev-1.13", + ), + state: "hello", + questions: { greeting: { type: "boolean", instructions: "Is this a greeting?" } }, + }).pipe( + Effect.tap((response) => + Effect.sync(() => { + expect(response.answers.greeting.probability).toBe(0.99) + expect(response.usage?.providerMetadata).toEqual({ + opencode: { input_tokens: 10, output_tokens: 2 }, + }) + }), + ), + Effect.provide( + EvaluationClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => { + expect(input.request.url).toBe("https://zen.test/v1/systemone") + expect(input.request.headers.authorization).toBe("Bearer zen-key") + return Effect.succeed( + input.respond( + JSON.stringify({ + model: "jev-1.13.0", + answers: { greeting: { type: "noul", noul: 0.99 } }, + usage: { input_tokens: 10, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ), + ) + }), + ), + ), + ), + ), + ) + + it.effect("rejects malformed questions before network I/O", () => + Effect.gen(function* () { + const error = yield* Evaluation.evaluate({ + model: TypeSafeAI.experimental.evaluation("jev-latest"), + state: "hello", + questions: { score: { type: "score", instructions: "How much?", criteria: ["only"] } }, + }).pipe(Effect.flip) + expect(error.reason._tag).toBe("InvalidRequest") + }).pipe( + Effect.provide( + EvaluationClient.layer.pipe( + Layer.provide(dynamicResponse(() => Effect.die("invalid evaluation reached the network"))), + ), + ), + ), + ) +}) diff --git a/packages/ai/test/evaluation.types.ts b/packages/ai/test/evaluation.types.ts new file mode 100644 index 00000000000..64396f6b115 --- /dev/null +++ b/packages/ai/test/evaluation.types.ts @@ -0,0 +1,60 @@ +import { Effect } from "effect" +import { Evaluation, EvaluationClient, EvaluationModel, type EvaluationRoute } from "../src/experimental.js" +import type { Service } from "../src/experimental/evaluation-client.js" +import { OpenCodeZen, TypeSafeAI } from "../src/providers.js" + +type Requirements = T extends Effect.Effect ? R : never +type Success = T extends Effect.Effect ? A : never +type Equal = [A, B] extends [B, A] ? true : false +type Assert = T + +const model = TypeSafeAI.configure({ apiKey: "test" }).experimental.evaluation("jev-latest") +const request = Evaluation.request({ + model, + state: { ticket: "refund" }, + questions: { + topic: { + type: "choice", + instructions: "Which team?", + criteria: { billing: null, support: { includes: ["help"] } }, + }, + severity: { type: "score", instructions: "How severe?", criteria: ["Low", "High"] }, + refund: { type: "boolean", instructions: "Refund?" }, + }, +}) + +const result = EvaluationClient.evaluate(request) +type Result = Success +type Choice = Assert> +type ClientRequirements = Assert, Service>> +void (true satisfies Choice) +void (true satisfies ClientRequirements) + +Effect.gen(function* () { + const response = yield* Evaluation.evaluate({ + model: OpenCodeZen.experimental.evaluation("jev-1.13"), + state: ["hello"], + questions: { greeting: { type: "boolean", instructions: "Greeting?" } }, + }) + response.answers.greeting.probability satisfies number + // @ts-expect-error Boolean answers do not contain a selected choice. + response.answers.greeting.choice + // @ts-expect-error Unknown question IDs are not exposed. + response.answers.missing +}) + +declare const route: EvaluationRoute<{ readonly temperature?: number }> +const custom = EvaluationModel.make({ id: "custom", provider: "custom", route }) +Evaluation.evaluate({ + model: custom, + state: "hello", + questions: { ok: { type: "boolean", instructions: "OK?" } }, + options: { temperature: 0.5 }, +}) +// @ts-expect-error Selected evaluation models retain their request option types. +Evaluation.evaluate({ + model: custom, + state: "hello", + questions: { ok: { type: "boolean", instructions: "OK?" } }, + options: { temperature: "high" }, +}) diff --git a/packages/ai/test/exports.test.ts b/packages/ai/test/exports.test.ts index fad7163797a..5b3851187f7 100644 --- a/packages/ai/test/exports.test.ts +++ b/packages/ai/test/exports.test.ts @@ -8,9 +8,11 @@ import { CloudflareWorkersAI, DeepSeek, Fireworks, + OpenCodeZen, OpenAI, OpenAICompatible, OpenRouter, + TypeSafeAI, XAI, } from "@opencode/ai/providers" import { @@ -23,6 +25,7 @@ import { } from "@opencode/ai/protocols" import * as AnthropicMessages from "@opencode/ai/protocols/anthropic-messages" import { TestLLM } from "@opencode/ai/testing" +import { Evaluation, EvaluationClient } from "@opencode/ai/experimental" describe("public exports", () => { test("root exposes app-facing runtime APIs", () => { @@ -37,6 +40,9 @@ describe("public exports", () => { expect(TestLLM.layer).toBeFunction() expect(TestLLM.testLayer).toBeFunction() expect(TestLLM.Test.of).toBeFunction() + expect(Evaluation.evaluate).toBeFunction() + expect(EvaluationClient.layer).toBeDefined() + expect(EvaluationClient.fetchLayer).toBeDefined() }) test("route barrel exposes route-authoring APIs", () => { @@ -66,6 +72,8 @@ describe("public exports", () => { expect(CloudflareWorkersAI.configure).toBeFunction() expect(CloudflareWorkersAI.configure({ accountId: "fixture", apiKey: "fixture" }).model).toBeFunction() expect(OpenRouter.model).toBeFunction() + expect(TypeSafeAI.experimental.evaluation).toBeFunction() + expect(OpenCodeZen.experimental.evaluation).toBeFunction() expect(XAI.model).toBeFunction() expect(XAI.provider.responses).toBe(XAI.responses) expect(XAI.provider.chat).toBe(XAI.chat) diff --git a/packages/ai/test/fixtures/recordings/opencode-zen-evaluation/evaluates-choice-score-and-boolean-questions.json b/packages/ai/test/fixtures/recordings/opencode-zen-evaluation/evaluates-choice-score-and-boolean-questions.json new file mode 100644 index 00000000000..5451189f443 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/opencode-zen-evaluation/evaluates-choice-score-and-boolean-questions.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "metadata": { + "model": "jev-1.13-free", + "tags": [ + "prefix:opencode-zen-evaluation", + "provider:opencode", + "protocol:system-one" + ], + "name": "opencode-zen-evaluation/evaluates-choice-score-and-boolean-questions", + "recordedAt": "2026-09-22T03:44:57.511Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://opencode.ai/zen/v1/systemone", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"jev-1.13-free\",\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"jev-1.13-free\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":1,\"probabilities\":{\"technical\":0,\"sales\":0,\"billing\":1}},\"urgency\":{\"type\":\"score\",\"score\":1.02,\"confidence\":0.95,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0,\"1\":0.97,\"2\":0.03}},\"refund\":{\"type\":\"noul\",\"noul\":0.99}},\"usage\":{\"input_tokens\":422,\"output_tokens\":69}}" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/typesafe-evaluation/evaluates-choice-score-and-boolean-questions.json b/packages/ai/test/fixtures/recordings/typesafe-evaluation/evaluates-choice-score-and-boolean-questions.json new file mode 100644 index 00000000000..9b0e9a3049b --- /dev/null +++ b/packages/ai/test/fixtures/recordings/typesafe-evaluation/evaluates-choice-score-and-boolean-questions.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "metadata": { + "model": "jev-latest", + "tags": [ + "prefix:typesafe-evaluation", + "provider:typesafe-ai", + "protocol:system-one" + ], + "name": "typesafe-evaluation/evaluates-choice-score-and-boolean-questions", + "recordedAt": "2026-09-22T03:44:28.626Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.typesafe.ai/v1/systemone", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"jev-latest\",\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"jev-1.13.0\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":1.0,\"probabilities\":{\"billing\":1.0,\"sales\":0.0,\"technical\":0.0}},\"urgency\":{\"type\":\"score\",\"score\":1.01,\"confidence\":0.97,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0.0,\"1\":0.98,\"2\":0.02}},\"refund\":{\"type\":\"noul\",\"noul\":0.99}},\"usage\":{\"input_tokens\":422,\"output_tokens\":69}}" + } + } + ] +} diff --git a/packages/ai/test/provider/evaluation.recorded.test.ts b/packages/ai/test/provider/evaluation.recorded.test.ts new file mode 100644 index 00000000000..4d409852aca --- /dev/null +++ b/packages/ai/test/provider/evaluation.recorded.test.ts @@ -0,0 +1,79 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { Evaluation } from "../../src/experimental.js" +import { OpenCodeZen, TypeSafeAI } from "../../src/providers.js" +import { recordedTests } from "../recorded-test.js" + +const questions = { + department: { + type: "choice", + instructions: "Which team should handle this support request?", + criteria: { + billing: "Payments, invoices, refunds, or failed charges", + technical: "Bugs, outages, or integrations", + sales: "Pricing, upgrades, or new accounts", + }, + }, + urgency: { + type: "score", + instructions: "How urgent is this support request?", + criteria: ["Can wait for normal support", "Needs prompt attention", "Actively blocking revenue"], + }, + refund: { type: "boolean", instructions: "Is the customer asking for a refund?" }, +} as const + +const state = "I was charged twice for the same invoice. Please refund the duplicate payment today." + +const typesafe = recordedTests({ + prefix: "typesafe-evaluation", + provider: "typesafe-ai", + protocol: "system-one", + requires: ["TYPESAFE_API_KEY"], + metadata: { model: "jev-latest" }, +}) + +const zen = recordedTests({ + prefix: "opencode-zen-evaluation", + provider: "opencode", + protocol: "system-one", + requires: ["OPENCODE_API_KEY"], + metadata: { model: "jev-1.13-free" }, +}) + +describe("experimental Evaluation recorded", () => { + typesafe.effect("evaluates choice score and boolean questions", () => + assertEvaluation( + TypeSafeAI.configure({ apiKey: process.env.TYPESAFE_API_KEY ?? "fixture" }).experimental.evaluation("jev-latest"), + "typesafe", + ), + ) + + zen.effect("evaluates choice score and boolean questions", () => + assertEvaluation( + OpenCodeZen.configure({ apiKey: process.env.OPENCODE_API_KEY ?? "fixture" }).experimental.evaluation( + "jev-1.13-free", + ), + "opencode", + ), + ) +}) + +const assertEvaluation = ( + model: ReturnType, + metadataKey: "typesafe" | "opencode", +) => + Effect.gen(function* () { + const response = yield* Evaluation.evaluate({ model, state, questions }) + expect(response.model).toStartWith("jev-") + expect(response.answers.department.type).toBe("choice") + expect(response.answers.department.choice).toBe("billing") + expect(response.answers.department.probabilities?.billing).toBeGreaterThan(0.5) + expect(response.answers.urgency.type).toBe("score") + expect(response.answers.urgency.score).toBeGreaterThanOrEqual(0) + expect(response.answers.urgency.score).toBeLessThanOrEqual(2) + expect(response.answers.refund.type).toBe("boolean") + expect(response.answers.refund.probability).toBeGreaterThan(0.5) + expect(response.usage?.inputTokens).toBeGreaterThan(0) + expect(response.usage?.outputTokens).toBeGreaterThan(0) + expect(response.providerMetadata?.[metadataKey]?.confidence).toBeDefined() + }) diff --git a/packages/ai/test/recorded-test.ts b/packages/ai/test/recorded-test.ts index be82f53b726..9bcab722b81 100644 --- a/packages/ai/test/recorded-test.ts +++ b/packages/ai/test/recorded-test.ts @@ -6,6 +6,8 @@ import * as path from "node:path" import { fileURLToPath } from "node:url" import { LLMClient, RequestExecutor } from "../src/route.js" import { ImageClient } from "../src/image-client.js" +import { EvaluationClient } from "../src/experimental/evaluation-client.js" +import type { Service as EvaluationClientService } from "../src/experimental/evaluation-client.js" import type { Service as ImageClientService } from "../src/image-client.js" import type { Service as LLMClientService } from "../src/route/client.js" import type { Service as RequestExecutorService } from "../src/route/executor.js" @@ -18,7 +20,12 @@ import { const __dirname = path.dirname(fileURLToPath(import.meta.url)) const FIXTURES_DIR = path.resolve(__dirname, "fixtures", "recordings") -type RecordedEnv = RequestExecutorService | LLMClientService | ImageClientService | Socket.WebSocketConstructor +type RecordedEnv = + | RequestExecutorService + | LLMClientService + | ImageClientService + | EvaluationClientService + | Socket.WebSocketConstructor type RecordedTestsOptions = RecordedGroupOptions & { readonly options?: HttpRecorder.RecorderOptions @@ -92,6 +99,7 @@ export const recordedTests = (options: RecordedTestsOptions) => requestExecutor, LLMClient.layer.pipe(Layer.provide(requestExecutor)), ImageClient.layer.pipe(Layer.provide(requestExecutor)), + EvaluationClient.layer.pipe(Layer.provide(requestExecutor)), webSocket, ) },