feat(ai): add experimental evaluation API (#50506)

packages/ai/README.md

@@ -29,6 +29,47 @@ await Effect.runPromise(program.pipe(Effect.provide(llmLayer)))
 
 Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
 
+## Experimental evaluation
+
+Evaluation models compare shared state with typed choice, score, and boolean questions. The API is
+isolated under an experimental entrypoint and provider namespace while the contract evolves:
+
+```ts
+import { Effect } from "effect"
+import { Evaluation, EvaluationClient } from "@opencode/ai/experimental"
+import { TypeSafeAI } from "@opencode/ai/providers"
+
+const model = TypeSafeAI.configure().experimental.evaluation("jev-latest")
+
+const program = Evaluation.evaluate({
+  model,
+  state: "I was charged twice. Please refund the duplicate payment.",
+  questions: {
+    department: {
+      type: "choice",
+      instructions: "Which team should handle this?",
+      criteria: { billing: "Payments and refunds", technical: "Bugs and outages" },
+    },
+    urgency: {
+      type: "score",
+      instructions: "How urgent is this?",
+      criteria: ["Can wait", "Needs prompt attention", "Blocking revenue"],
+    },
+    refund: { type: "boolean", instructions: "Is the customer asking for a refund?" },
+  },
+})
+
+const response = await Effect.runPromise(program.pipe(Effect.provide(EvaluationClient.fetchLayer)))
+
+console.log(response.answers.department.choice)
+console.log(response.answers.refund.probability)
+```
+
+`TypeSafeAI` reads `TYPESAFE_API_KEY`. `OpenCodeZen` exposes the same selector and reads
+`OPENCODE_API_KEY`. The common API uses `boolean`; System One routes lower it to native `noul`.
+Choice and score confidence plus score legends remain available in provider metadata, and the
+provider's rounded probabilities are returned unchanged.
+
 ## Alibaba Cloud Model Studio
 
 `Alibaba` provides standard Model Studio inference. Configure a region explicitly, then select

packages/ai/src/experimental.ts

@@ -0,0 +1,29 @@
+export { EvaluationClient } from "./experimental/evaluation-client.js"
+export {
+  BooleanAnswer,
+  BooleanQuestion,
+  ChoiceAnswer,
+  ChoiceQuestion,
+  Evaluation,
+  EvaluationAnswer,
+  EvaluationInput,
+  EvaluationModel,
+  EvaluationModelSchema,
+  EvaluationQuestion,
+  EvaluationRequest,
+  EvaluationResponse,
+  EvaluationRounding,
+  ScoreAnswer,
+  ScoreQuestion,
+} from "./experimental/evaluation.js"
+export type {
+  AnswerFor,
+  AnswersFor,
+  EvaluationModelOptions,
+  EvaluationOptions,
+  EvaluationQuestions,
+  EvaluationRequestFor,
+  EvaluationRequestInput,
+  EvaluationResponseFor,
+  EvaluationRoute,
+} from "./experimental/evaluation.js"

packages/ai/src/experimental/evaluation-client.ts

@@ -0,0 +1,54 @@
+import { Context, Effect, Layer } from "effect"
+import { RequestExecutor } from "../route/executor.js"
+import { mergeHttpOptions, type AIError } from "../schema/index.js"
+import { sanitizeSurrogates } from "../utils/sanitize.js"
+import {
+  type EvaluationOptions,
+  type EvaluationQuestions,
+  type EvaluationRequestFor,
+  type EvaluationResponseFor,
+} from "./evaluation.js"
+
+export type Execute = RequestExecutor.Interface["execute"]
+
+export interface Interface {
+  readonly evaluate: <Options extends EvaluationOptions, const Questions extends EvaluationQuestions>(
+    request: EvaluationRequestFor<Options, Questions>,
+  ) => Effect.Effect<EvaluationResponseFor<Questions>, AIError>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@opencode/AI/Experimental/EvaluationClient") {}
+
+export const evaluate = <Options extends EvaluationOptions, const Questions extends EvaluationQuestions>(
+  request: EvaluationRequestFor<Options, Questions>,
+): Effect.Effect<EvaluationResponseFor<Questions>, AIError, Service> =>
+  Effect.flatMap(Service, (client) => client.evaluate(request))
+
+export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const executor = yield* RequestExecutor.Service
+    return Service.of({
+      evaluate: (request) =>
+        request.model.route.evaluate(
+          {
+            ...sanitizeSurrogates({
+              ...request,
+              model: undefined,
+              http: mergeHttpOptions(request.model.http, request.http),
+            }),
+            model: request.model,
+          },
+          executor.execute,
+        ),
+    })
+  }),
+)
+export const fetchLayer = layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
+
+export const EvaluationClient = {
+  Service,
+  layer,
+  fetchLayer,
+  evaluate,
+} as const

packages/ai/src/experimental/evaluation.ts

@@ -0,0 +1,245 @@
+import { Effect, Schema } from "effect"
+import {
+  AIError,
+  HttpOptions,
+  InvalidRequestError,
+  ModelID,
+  ProviderID,
+  ProviderMetadata,
+  Usage,
+} from "../schema/index.js"
+import { EvaluationClient, Service, type Execute } from "./evaluation-client.js"
+
+export const EvaluationInput = Schema.Union([Schema.String, Schema.JsonObject, Schema.Array(Schema.Json)])
+export type EvaluationInput = Schema.Schema.Type<typeof EvaluationInput>
+
+const EvaluationCriterion = Schema.NullOr(EvaluationInput)
+const ChoiceCriteria = Schema.Record(Schema.String, EvaluationCriterion).pipe(
+  Schema.refine((x): x is typeof x => Object.keys(x).length > 0, {
+    message: "Choice criteria must be a nonempty option map",
+  }),
+)
+
+export const ChoiceQuestion = Schema.Struct({
+  type: Schema.Literal("choice"),
+  instructions: EvaluationInput,
+  criteria: ChoiceCriteria,
+})
+export type ChoiceQuestion = Schema.Schema.Type<typeof ChoiceQuestion>
+
+export const ScoreQuestion = Schema.Struct({
+  type: Schema.Literal("score"),
+  instructions: EvaluationInput,
+  criteria: Schema.Array(EvaluationCriterion).check(Schema.isMinLength(2)),
+})
+export type ScoreQuestion = Schema.Schema.Type<typeof ScoreQuestion>
+
+export const BooleanQuestion = Schema.Struct({
+  type: Schema.Literal("boolean"),
+  instructions: EvaluationInput,
+  criteria: Schema.optional(
+    Schema.Struct({
+      true: Schema.optional(EvaluationCriterion),
+      false: Schema.optional(EvaluationCriterion),
+    }),
+  ),
+})
+export type BooleanQuestion = Schema.Schema.Type<typeof BooleanQuestion>
+
+export const EvaluationQuestion = Schema.Union([ChoiceQuestion, ScoreQuestion, BooleanQuestion]).pipe(
+  Schema.toTaggedUnion("type"),
+)
+export type EvaluationQuestion = Schema.Schema.Type<typeof EvaluationQuestion>
+export type EvaluationQuestions = Readonly<Record<string, EvaluationQuestion>>
+const EvaluationQuestions = Schema.Record(Schema.String, EvaluationQuestion).pipe(
+  Schema.refine((x): x is typeof x => Object.keys(x).length > 0, {
+    message: "Evaluation questions must be a nonempty map",
+  }),
+)
+
+const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 }))
+
+export const ChoiceAnswer = Schema.Struct({
+  type: Schema.Literal("choice"),
+  choice: Schema.String,
+  probabilities: Schema.optional(Schema.Record(Schema.String, Probability)),
+})
+export type ChoiceAnswer = Schema.Schema.Type<typeof ChoiceAnswer>
+
+export const ScoreAnswer = Schema.Struct({
+  type: Schema.Literal("score"),
+  score: Schema.Number,
+  probabilities: Schema.optional(Schema.Record(Schema.String, Probability)),
+})
+export type ScoreAnswer = Schema.Schema.Type<typeof ScoreAnswer>
+
+export const BooleanAnswer = Schema.Struct({
+  type: Schema.Literal("boolean"),
+  probability: Probability,
+})
+export type BooleanAnswer = Schema.Schema.Type<typeof BooleanAnswer>
+
+export const EvaluationAnswer = Schema.Union([ChoiceAnswer, ScoreAnswer, BooleanAnswer]).pipe(
+  Schema.toTaggedUnion("type"),
+)
+export type EvaluationAnswer = Schema.Schema.Type<typeof EvaluationAnswer>
+
+export type AnswerFor<Question extends EvaluationQuestion> = Question extends {
+  readonly type: "choice"
+  readonly criteria: infer Criteria
+}
+  ? {
+      readonly type: "choice"
+      readonly choice: Extract<keyof Criteria, string>
+      readonly probabilities?: Readonly<Record<Extract<keyof Criteria, string>, number>>
+    }
+  : Question extends { readonly type: "score" }
+    ? ScoreAnswer
+    : BooleanAnswer
+
+export type AnswersFor<Questions extends EvaluationQuestions> = {
+  readonly [ID in keyof Questions]: AnswerFor<Questions[ID]>
+}
+
+export type EvaluationOptions = Record<string, unknown>
+
+export interface EvaluationRoute<Options extends EvaluationOptions = EvaluationOptions> {
+  readonly id: string
+  readonly evaluate: <const Questions extends EvaluationQuestions>(
+    request: EvaluationRequestFor<Options, Questions>,
+    execute: Execute,
+  ) => Effect.Effect<EvaluationResponseFor<Questions>, AIError>
+}
+
+export class EvaluationModel<Options extends EvaluationOptions = EvaluationOptions> {
+  declare protected readonly _Options: (options: Options) => Options
+  readonly id: ModelID
+  readonly provider: ProviderID
+  readonly route: EvaluationRoute<Options>
+  readonly http?: HttpOptions
+
+  constructor(input: EvaluationModel.Input<Options>) {
+    this.id = input.id
+    this.provider = input.provider
+    this.route = input.route
+    this.http = input.http
+  }
+
+  static make<Options extends EvaluationOptions = EvaluationOptions>(input: EvaluationModel.MakeInput<Options>) {
+    return new EvaluationModel<Options>({
+      id: ModelID.make(input.id),
+      provider: ProviderID.make(input.provider),
+      route: input.route,
+      http: input.http,
+    })
+  }
+}
+
+export namespace EvaluationModel {
+  export interface Input<Options extends EvaluationOptions = EvaluationOptions> {
+    readonly id: ModelID
+    readonly provider: ProviderID
+    readonly route: EvaluationRoute<Options>
+    readonly http?: HttpOptions
+  }
+
+  export interface MakeInput<Options extends EvaluationOptions = EvaluationOptions>
+    extends Omit<Input<Options>, "id" | "provider"> {
+    readonly id: string | ModelID
+    readonly provider: string | ProviderID
+  }
+}
+
+export const EvaluationModelSchema = Schema.declare(
+  (value): value is EvaluationModel => value instanceof EvaluationModel,
+  {
+    expected: "Evaluation.Model",
+  },
+)
+
+export class EvaluationRequest extends Schema.Class<EvaluationRequest>("Evaluation.Request")({
+  model: EvaluationModelSchema,
+  state: EvaluationInput,
+  questions: EvaluationQuestions,
+  options: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
+  http: Schema.optional(HttpOptions),
+}) {
+  declare protected readonly _EvaluationRequest: void
+}
+
+export type EvaluationModelOptions<Model> = Model extends EvaluationModel<infer Options> ? Options : never
+
+export type EvaluationRequestFor<
+  Options extends EvaluationOptions = EvaluationOptions,
+  Questions extends EvaluationQuestions = EvaluationQuestions,
+> = Omit<EvaluationRequest, "model" | "questions" | "options"> & {
+  readonly model: EvaluationModel<Options>
+  readonly questions: Questions
+  readonly options?: Options
+}
+
+export type EvaluationRequestInput<
+  Model extends object = EvaluationModel,
+  Questions extends EvaluationQuestions = EvaluationQuestions,
+> = Omit<ConstructorParameters<typeof EvaluationRequest>[0], "model" | "questions" | "options" | "http"> & {
+  readonly model: Model
+  readonly questions: Questions
+  readonly options?: NoInfer<EvaluationModelOptions<Model>>
+  readonly http?: HttpOptions.Input
+} & (Model extends EvaluationModel<EvaluationModelOptions<Model>> ? unknown : never)
+
+export class EvaluationRounding extends Schema.Class<EvaluationRounding>("Evaluation.Rounding")({
+  probabilityDecimals: Schema.optional(Schema.Int),
+  scoreDecimals: Schema.optional(Schema.Int),
+}) {}
+
+export class EvaluationResponse extends Schema.Class<EvaluationResponse>("Evaluation.Response")({
+  model: ModelID,
+  answers: Schema.Record(Schema.String, EvaluationAnswer),
+  usage: Schema.optional(Usage),
+  rounding: Schema.optional(EvaluationRounding),
+  providerMetadata: Schema.optional(ProviderMetadata),
+}) {}
+
+export type EvaluationResponseFor<Questions extends EvaluationQuestions> = Omit<EvaluationResponse, "answers"> & {
+  readonly answers: AnswersFor<Questions>
+}
+
+export function request<const Model extends object, const Questions extends EvaluationQuestions>(
+  input: EvaluationRequestInput<Model, Questions>,
+): EvaluationRequestFor<EvaluationModelOptions<Model>, Questions>
+export function request(input: EvaluationRequest): EvaluationRequest
+export function request(input: EvaluationRequest | EvaluationRequestInput) {
+  if (input instanceof EvaluationRequest) return input
+  return new EvaluationRequest({
+    ...input,
+    model: input.model as unknown as EvaluationModel,
+    http: input.http === undefined ? undefined : HttpOptions.make(input.http),
+  })
+}
+
+export function evaluate<const Model extends object, const Questions extends EvaluationQuestions>(
+  input: EvaluationRequestInput<Model, Questions>,
+): Effect.Effect<EvaluationResponseFor<Questions>, AIError, Service>
+export function evaluate(input: EvaluationRequest): Effect.Effect<EvaluationResponse, AIError, Service>
+export function evaluate(input: EvaluationRequest | EvaluationRequestInput) {
+  return Effect.try({
+    try: () => (input instanceof EvaluationRequest ? input : request(input)),
+    catch: (cause) =>
+      new AIError({
+        reason: new InvalidRequestError({
+          message: cause instanceof Error ? cause.message : String(cause),
+          cause,
+        }),
+      }),
+  }).pipe(
+    Effect.flatMap((request) =>
+      EvaluationClient.evaluate(request as EvaluationRequestFor<EvaluationOptions, EvaluationQuestions>),
+    ),
+  )
+}
+
+export const Evaluation = {
+  request,
+  evaluate,
+} as const

packages/ai/src/experimental/system-one.ts

@@ -0,0 +1,222 @@
+import { Effect, Schema } from "effect"
+import { Headers, HttpClientRequest } from "effect/unstable/http"
+import {
+  BooleanAnswer,
+  ChoiceAnswer,
+  ChoiceQuestion,
+  EvaluationInput,
+  EvaluationModel,
+  EvaluationResponse,
+  EvaluationRounding,
+  ScoreAnswer,
+  ScoreQuestion,
+  type EvaluationResponseFor,
+  type EvaluationRoute,
+} from "./evaluation.js"
+import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
+import {
+  AIError,
+  HttpContext,
+  HttpOptions,
+  InvalidProviderOutputError,
+  InvalidRequestError,
+  ModelID,
+  Usage,
+  mergeJsonRecords,
+} from "../schema/index.js"
+
+const Noul = Schema.Struct({
+  type: Schema.Literal("noul"),
+  instructions: EvaluationInput,
+  criteria: Schema.optional(
+    Schema.Struct({
+      true: Schema.optional(Schema.NullOr(EvaluationInput)),
+      false: Schema.optional(Schema.NullOr(EvaluationInput)),
+    }),
+  ),
+})
+const Question = Schema.Union([
+  ChoiceQuestion.pipe(
+    Schema.refine((x): x is typeof x => Object.keys(x.criteria).length <= 255, {
+      message: "System One Choice questions support at most 255 options",
+    }),
+  ),
+  ScoreQuestion.pipe(
+    Schema.refine((x): x is typeof x => x.criteria.length <= 10, {
+      message: "System One Score questions support at most 10 levels",
+    }),
+  ),
+  Noul,
+])
+const Request = Schema.StructWithRest(
+  Schema.Struct({
+    model: Schema.String,
+    state: EvaluationInput,
+    questions: Schema.Record(Schema.String, Question),
+  }),
+  [Schema.Record(Schema.String, Schema.Any)],
+)
+
+const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 }))
+const NoulAnswer = Schema.Struct({ type: Schema.Literal("noul"), noul: Probability })
+const Choice = Schema.Struct({
+  type: Schema.Literal("choice"),
+  choice: Schema.String,
+  probabilities: Schema.Record(Schema.String, Probability),
+  confidence: Schema.optional(Probability),
+})
+const Score = Schema.Struct({
+  type: Schema.Literal("score"),
+  score: Schema.Number,
+  probabilities: Schema.Record(Schema.String, Probability),
+  legend: Schema.optional(Schema.Record(Schema.String, Schema.Json)),
+  confidence: Schema.optional(Probability),
+})
+const NativeUsage = Schema.Struct({
+  input_tokens: Schema.optional(Schema.Number),
+  output_tokens: Schema.optional(Schema.Number),
+})
+
+const encode = Schema.encodeUnknownEffect(Schema.fromJsonString(Request))
+const exact = (x: Readonly<Record<string, unknown>>, keys: ReadonlyArray<string>) =>
+  Object.keys(x).length === keys.length && keys.every((key) => Object.hasOwn(x, key))
+
+export interface ModelInput {
+  readonly id: string | ModelID
+  readonly provider: string
+  readonly providerMetadataKey: string
+  readonly auth: AuthDefinition
+  readonly baseURL: string
+  readonly headers?: Record<string, string>
+  readonly http?: HttpOptions
+}
+
+export const model = (cfg: ModelInput) => {
+  const route: EvaluationRoute = {
+    id: "system-one",
+    evaluate: (req, send) =>
+      Effect.gen(function* () {
+        const url = new URL(`${cfg.baseURL.replace(/\/$/, "")}/systemone`)
+        Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value))
+        const body = yield* encode({
+          ...mergeJsonRecords(req.options, req.http?.body),
+          model: req.model.id,
+          state: req.state,
+          questions: Object.fromEntries(
+            Object.entries(req.questions).map(([id, x]) => [id, x.type === "boolean" ? { ...x, type: "noul" } : x]),
+          ),
+        }).pipe(
+          Effect.mapError(
+            (cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }),
+          ),
+        )
+        const headers = yield* Auth.toEffect(cfg.auth)({
+          request: req,
+          method: "POST",
+          url: url.toString(),
+          body,
+          headers: Headers.fromInput({ ...cfg.headers, ...req.http?.headers }),
+        })
+        const res = yield* send(
+          HttpClientRequest.post(url).pipe(
+            HttpClientRequest.setHeaders(headers),
+            HttpClientRequest.bodyText(body, "application/json"),
+          ),
+        )
+        const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers })
+        const fail = (message: string, cause: unknown, body?: string) =>
+          new AIError({ reason: new InvalidProviderOutputError({ route: route.id, message, body, http, cause }) })
+        const text = yield* res.text.pipe(
+          Effect.mapError((cause) => fail("Failed to read the System One response", cause)),
+        )
+        const entries = Object.entries(req.questions)
+        const output = Schema.Struct({
+          model: Schema.String,
+          answers: Schema.Struct(
+            Object.fromEntries(
+              entries.map(([id, question]) => {
+                if (question.type === "boolean") return [id, NoulAnswer]
+                if (question.type === "choice") {
+                  const keys = Object.keys(question.criteria)
+                  return [
+                    id,
+                    Choice.pipe(
+                      Schema.refine(
+                        (x): x is typeof x =>
+                          Object.hasOwn(question.criteria, x.choice) && exact(x.probabilities, keys),
+                        { message: `Question "${id}" returned an invalid choice answer` },
+                      ),
+                    ),
+                  ]
+                }
+                const keys = question.criteria.map((_, index) => String(index))
+                return [
+                  id,
+                  Score.pipe(
+                    Schema.refine(
+                      (x): x is typeof x =>
+                        x.score >= 0 && x.score <= question.criteria.length - 1 && exact(x.probabilities, keys),
+                      { message: `Question "${id}" returned an invalid score answer` },
+                    ),
+                  ),
+                ]
+              }),
+            ) as Record<string, Schema.Codec<unknown>>,
+          ),
+          usage: Schema.optional(NativeUsage),
+        })
+        const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(output))(text).pipe(
+          Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)),
+        )
+
+        const confidence: Record<string, number> = {}
+        const legend: Record<string, Record<string, Schema.Json>> = {}
+        const answers = Object.fromEntries(
+          entries.map(([id, question]) => {
+            const answer = data.answers[id]
+            if (question.type === "boolean")
+              return [
+                id,
+                { type: "boolean", probability: (answer as typeof NoulAnswer.Type).noul } satisfies BooleanAnswer,
+              ]
+            if (question.type === "choice") {
+              const value = answer as typeof Choice.Type
+              if (value.confidence !== undefined) confidence[id] = value.confidence
+              return [
+                id,
+                { type: "choice", choice: value.choice, probabilities: value.probabilities } satisfies ChoiceAnswer,
+              ]
+            }
+            const value = answer as typeof Score.Type
+            if (value.confidence !== undefined) confidence[id] = value.confidence
+            if (value.legend !== undefined) legend[id] = value.legend
+            return [id, { type: "score", score: value.score, probabilities: value.probabilities } satisfies ScoreAnswer]
+          }),
+        )
+        const meta = {
+          ...(Object.keys(confidence).length === 0 ? {} : { confidence }),
+          ...(Object.keys(legend).length === 0 ? {} : { legend }),
+        }
+        return new EvaluationResponse({
+          model: ModelID.make(data.model),
+          answers,
+          usage: data.usage
+            ? new Usage({
+                inputTokens: data.usage.input_tokens,
+                outputTokens: data.usage.output_tokens,
+                totalTokens:
+                  data.usage.input_tokens === undefined && data.usage.output_tokens === undefined
+                    ? undefined
+                    : (data.usage.input_tokens ?? 0) + (data.usage.output_tokens ?? 0),
+                providerMetadata: { [cfg.providerMetadataKey]: data.usage },
+              })
+            : undefined,
+          rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }),
+          providerMetadata: Object.keys(meta).length === 0 ? undefined : { [cfg.providerMetadataKey]: meta },
+        }) as EvaluationResponseFor<typeof req.questions>
+      }),
+  }
+  return EvaluationModel.make({ id: cfg.id, provider: cfg.provider, route, http: cfg.http })
+}
+
+export const SystemOne = { model } as const

packages/ai/src/providers/index.ts

@@ -24,8 +24,10 @@ export * as Moonshot from "./moonshot.js"
 export * as OpenAI from "./openai.js"
 export * as OpenAICompatible from "./openai-compatible.js"
 export * as OpenAICompatibleResponses from "./openai-compatible-responses.js"
+export * as OpenCodeZen from "./opencode-zen.js"
 export * as OpenRouter from "./openrouter.js"
 export * as TogetherAI from "./togetherai.js"
+export * as TypeSafeAI from "./typesafe-ai.js"
 export * as XAI from "./xai.js"
 export * as ZAI from "./zai.js"
 export * as ZAICodingPlan from "./zai-coding-plan.js"

packages/ai/src/providers/opencode-zen.ts

@@ -0,0 +1,31 @@
+import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
+import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
+import { SystemOne } from "../experimental/system-one.js"
+
+export const id = ProviderID.make("opencode")
+const baseURL = "https://opencode.ai/zen/v1"
+
+export type Options = ProviderAuthOption<"optional"> & {
+  readonly baseURL?: string
+  readonly headers?: Record<string, string>
+  readonly http?: HttpOptions.Input
+}
+
+export const configure = (input: Options = {}) => {
+  const evaluation = (modelID: string | ModelID) =>
+    SystemOne.model({
+      id: modelID,
+      provider: id,
+      providerMetadataKey: "opencode",
+      auth: AuthOptions.bearer(input, "OPENCODE_API_KEY"),
+      baseURL: input.baseURL ?? baseURL,
+      headers: input.headers,
+      http: input.http === undefined ? undefined : HttpOptions.make(input.http),
+    })
+  return { id, experimental: { evaluation }, configure }
+}
+
+export const provider = configure()
+export const experimental = provider.experimental
+
+export * as OpenCodeZen from "./opencode-zen.js"

packages/ai/src/providers/typesafe-ai.ts

@@ -0,0 +1,31 @@
+import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
+import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
+import { SystemOne } from "../experimental/system-one.js"
+
+export const id = ProviderID.make("typesafe-ai")
+const baseURL = "https://api.typesafe.ai/v1"
+
+export type Options = ProviderAuthOption<"optional"> & {
+  readonly baseURL?: string
+  readonly headers?: Record<string, string>
+  readonly http?: HttpOptions.Input
+}
+
+export const configure = (input: Options = {}) => {
+  const evaluation = (modelID: string | ModelID) =>
+    SystemOne.model({
+      id: modelID,
+      provider: id,
+      providerMetadataKey: "typesafe",
+      auth: AuthOptions.bearer(input, "TYPESAFE_API_KEY"),
+      baseURL: input.baseURL ?? baseURL,
+      headers: input.headers,
+      http: input.http === undefined ? undefined : HttpOptions.make(input.http),
+    })
+  return { id, experimental: { evaluation }, configure }
+}
+
+export const provider = configure()
+export const experimental = provider.experimental
+
+export * as TypeSafeAI from "./typesafe-ai.js"

packages/ai/test/evaluation.test.ts

@@ -0,0 +1,173 @@
+import { describe, expect } from "bun:test"
+import { Effect, Layer } from "effect"
+import { HttpClientRequest } from "effect/unstable/http"
+import { Evaluation, EvaluationClient } from "../src/experimental.js"
+import { OpenCodeZen, TypeSafeAI } from "../src/providers.js"
+import { it } from "./lib/effect.js"
+import { dynamicResponse } from "./lib/http.js"
+
+describe("experimental Evaluation", () => {
+  it.effect("evaluates typed questions through System One", () =>
+    Effect.gen(function* () {
+      const response = yield* Evaluation.evaluate({
+        model: TypeSafeAI.configure({
+          apiKey: "test",
+          baseURL: "https://typesafe.test/v1/",
+          headers: { "x-default": "yes" },
+          http: { body: { deployment: "test" }, query: { api: "v1" } },
+        }).experimental.evaluation("jev-latest"),
+        state: { ticket: "Please refund the duplicate charge." },
+        questions: {
+          department: {
+            type: "choice",
+            instructions: "Which team should handle this?",
+            criteria: { billing: "Payments and refunds", technical: "Bugs and outages" },
+          },
+          urgency: {
+            type: "score",
+            instructions: "How urgent is this?",
+            criteria: ["Can wait", "Needs attention", "Blocking"],
+          },
+          refund: { type: "boolean", instructions: "Is the customer asking for a refund?" },
+        },
+        options: { trace: { enabled: true } },
+        http: { body: { request_metadata: "value" }, headers: { "x-request": "yes" }, query: { trace: "1" } },
+      })
+
+      expect(response.model).toBe("jev-1.13.0")
+      expect(response.answers.department).toEqual({
+        type: "choice",
+        choice: "billing",
+        probabilities: { billing: 0.9, technical: 0.1 },
+      })
+      expect(response.answers.urgency).toEqual({
+        type: "score",
+        score: 1.2,
+        probabilities: { "0": 0, "1": 0.8, "2": 0.2 },
+      })
+      expect(response.answers.refund).toEqual({ type: "boolean", probability: 0.97 })
+      expect(response.usage?.totalTokens).toBe(36)
+      expect(response.providerMetadata).toEqual({
+        typesafe: {
+          confidence: { department: 0.8, urgency: 0.6 },
+          legend: { urgency: { "0": "Can wait", "1": "Needs attention", "2": "Blocking" } },
+        },
+      })
+    }).pipe(
+      Effect.provide(
+        EvaluationClient.layer.pipe(
+          Layer.provide(
+            dynamicResponse((input) =>
+              Effect.gen(function* () {
+                const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
+                expect(request.url).toBe("https://typesafe.test/v1/systemone?api=v1&trace=1")
+                expect(request.headers.get("authorization")).toBe("Bearer test")
+                expect(request.headers.get("x-default")).toBe("yes")
+                expect(request.headers.get("x-request")).toBe("yes")
+                expect(JSON.parse(input.text)).toEqual({
+                  deployment: "test",
+                  request_metadata: "value",
+                  trace: { enabled: true },
+                  model: "jev-latest",
+                  state: { ticket: "Please refund the duplicate charge." },
+                  questions: {
+                    department: {
+                      type: "choice",
+                      instructions: "Which team should handle this?",
+                      criteria: { billing: "Payments and refunds", technical: "Bugs and outages" },
+                    },
+                    urgency: {
+                      type: "score",
+                      instructions: "How urgent is this?",
+                      criteria: ["Can wait", "Needs attention", "Blocking"],
+                    },
+                    refund: { type: "noul", instructions: "Is the customer asking for a refund?" },
+                  },
+                })
+                return input.respond(
+                  JSON.stringify({
+                    model: "jev-1.13.0",
+                    answers: {
+                      department: {
+                        type: "choice",
+                        choice: "billing",
+                        probabilities: { billing: 0.9, technical: 0.1 },
+                        confidence: 0.8,
+                      },
+                      urgency: {
+                        type: "score",
+                        score: 1.2,
+                        probabilities: { "0": 0, "1": 0.8, "2": 0.2 },
+                        legend: { "0": "Can wait", "1": "Needs attention", "2": "Blocking" },
+                        confidence: 0.6,
+                      },
+                      refund: { type: "noul", noul: 0.97 },
+                    },
+                    usage: { input_tokens: 30, output_tokens: 6 },
+                  }),
+                  { headers: { "content-type": "application/json" } },
+                )
+              }),
+            ),
+          ),
+        ),
+      ),
+    ),
+  )
+
+  it.effect("configures the OpenCode Zen System One endpoint", () =>
+    Evaluation.evaluate({
+      model: OpenCodeZen.configure({ apiKey: "zen-key", baseURL: "https://zen.test/v1" }).experimental.evaluation(
+        "jev-1.13",
+      ),
+      state: "hello",
+      questions: { greeting: { type: "boolean", instructions: "Is this a greeting?" } },
+    }).pipe(
+      Effect.tap((response) =>
+        Effect.sync(() => {
+          expect(response.answers.greeting.probability).toBe(0.99)
+          expect(response.usage?.providerMetadata).toEqual({
+            opencode: { input_tokens: 10, output_tokens: 2 },
+          })
+        }),
+      ),
+      Effect.provide(
+        EvaluationClient.layer.pipe(
+          Layer.provide(
+            dynamicResponse((input) => {
+              expect(input.request.url).toBe("https://zen.test/v1/systemone")
+              expect(input.request.headers.authorization).toBe("Bearer zen-key")
+              return Effect.succeed(
+                input.respond(
+                  JSON.stringify({
+                    model: "jev-1.13.0",
+                    answers: { greeting: { type: "noul", noul: 0.99 } },
+                    usage: { input_tokens: 10, output_tokens: 2 },
+                  }),
+                  { headers: { "content-type": "application/json" } },
+                ),
+              )
+            }),
+          ),
+        ),
+      ),
+    ),
+  )
+
+  it.effect("rejects malformed questions before network I/O", () =>
+    Effect.gen(function* () {
+      const error = yield* Evaluation.evaluate({
+        model: TypeSafeAI.experimental.evaluation("jev-latest"),
+        state: "hello",
+        questions: { score: { type: "score", instructions: "How much?", criteria: ["only"] } },
+      }).pipe(Effect.flip)
+      expect(error.reason._tag).toBe("InvalidRequest")
+    }).pipe(
+      Effect.provide(
+        EvaluationClient.layer.pipe(
+          Layer.provide(dynamicResponse(() => Effect.die("invalid evaluation reached the network"))),
+        ),
+      ),
+    ),
+  )
+})

packages/ai/test/evaluation.types.ts

@@ -0,0 +1,60 @@
+import { Effect } from "effect"
+import { Evaluation, EvaluationClient, EvaluationModel, type EvaluationRoute } from "../src/experimental.js"
+import type { Service } from "../src/experimental/evaluation-client.js"
+import { OpenCodeZen, TypeSafeAI } from "../src/providers.js"
+
+type Requirements<T> = T extends Effect.Effect<infer _A, infer _E, infer R> ? R : never
+type Success<T> = T extends Effect.Effect<infer A, infer _E, infer _R> ? A : never
+type Equal<A, B> = [A, B] extends [B, A] ? true : false
+type Assert<T extends true> = T
+
+const model = TypeSafeAI.configure({ apiKey: "test" }).experimental.evaluation("jev-latest")
+const request = Evaluation.request({
+  model,
+  state: { ticket: "refund" },
+  questions: {
+    topic: {
+      type: "choice",
+      instructions: "Which team?",
+      criteria: { billing: null, support: { includes: ["help"] } },
+    },
+    severity: { type: "score", instructions: "How severe?", criteria: ["Low", "High"] },
+    refund: { type: "boolean", instructions: "Refund?" },
+  },
+})
+
+const result = EvaluationClient.evaluate(request)
+type Result = Success<typeof result>
+type Choice = Assert<Equal<Result["answers"]["topic"]["choice"], "billing" | "support">>
+type ClientRequirements = Assert<Equal<Requirements<typeof result>, Service>>
+void (true satisfies Choice)
+void (true satisfies ClientRequirements)
+
+Effect.gen(function* () {
+  const response = yield* Evaluation.evaluate({
+    model: OpenCodeZen.experimental.evaluation("jev-1.13"),
+    state: ["hello"],
+    questions: { greeting: { type: "boolean", instructions: "Greeting?" } },
+  })
+  response.answers.greeting.probability satisfies number
+  // @ts-expect-error Boolean answers do not contain a selected choice.
+  response.answers.greeting.choice
+  // @ts-expect-error Unknown question IDs are not exposed.
+  response.answers.missing
+})
+
+declare const route: EvaluationRoute<{ readonly temperature?: number }>
+const custom = EvaluationModel.make({ id: "custom", provider: "custom", route })
+Evaluation.evaluate({
+  model: custom,
+  state: "hello",
+  questions: { ok: { type: "boolean", instructions: "OK?" } },
+  options: { temperature: 0.5 },
+})
+// @ts-expect-error Selected evaluation models retain their request option types.
+Evaluation.evaluate({
+  model: custom,
+  state: "hello",
+  questions: { ok: { type: "boolean", instructions: "OK?" } },
+  options: { temperature: "high" },
+})

packages/ai/test/exports.test.ts

@@ -8,9 +8,11 @@ import {
   CloudflareWorkersAI,
   DeepSeek,
   Fireworks,
+  OpenCodeZen,
   OpenAI,
   OpenAICompatible,
   OpenRouter,
+  TypeSafeAI,
   XAI,
 } from "@opencode/ai/providers"
 import {
@@ -23,6 +25,7 @@ import {
 } from "@opencode/ai/protocols"
 import * as AnthropicMessages from "@opencode/ai/protocols/anthropic-messages"
 import { TestLLM } from "@opencode/ai/testing"
+import { Evaluation, EvaluationClient } from "@opencode/ai/experimental"
 
 describe("public exports", () => {
   test("root exposes app-facing runtime APIs", () => {
@@ -37,6 +40,9 @@ describe("public exports", () => {
     expect(TestLLM.layer).toBeFunction()
     expect(TestLLM.testLayer).toBeFunction()
     expect(TestLLM.Test.of).toBeFunction()
+    expect(Evaluation.evaluate).toBeFunction()
+    expect(EvaluationClient.layer).toBeDefined()
+    expect(EvaluationClient.fetchLayer).toBeDefined()
   })
 
   test("route barrel exposes route-authoring APIs", () => {
@@ -66,6 +72,8 @@ describe("public exports", () => {
     expect(CloudflareWorkersAI.configure).toBeFunction()
     expect(CloudflareWorkersAI.configure({ accountId: "fixture", apiKey: "fixture" }).model).toBeFunction()
     expect(OpenRouter.model).toBeFunction()
+    expect(TypeSafeAI.experimental.evaluation).toBeFunction()
+    expect(OpenCodeZen.experimental.evaluation).toBeFunction()
     expect(XAI.model).toBeFunction()
     expect(XAI.provider.responses).toBe(XAI.responses)
     expect(XAI.provider.chat).toBe(XAI.chat)

packages/ai/test/fixtures/recordings/opencode-zen-evaluation/evaluates-choice-score-and-boolean-questions.json

@@ -0,0 +1,33 @@
+{
+  "version": 1,
+  "metadata": {
+    "model": "jev-1.13-free",
+    "tags": [
+      "prefix:opencode-zen-evaluation",
+      "provider:opencode",
+      "protocol:system-one"
+    ],
+    "name": "opencode-zen-evaluation/evaluates-choice-score-and-boolean-questions",
+    "recordedAt": "2026-09-22T03:44:57.511Z"
+  },
+  "interactions": [
+    {
+      "transport": "http",
+      "request": {
+        "method": "POST",
+        "url": "https://opencode.ai/zen/v1/systemone",
+        "headers": {
+          "content-type": "application/json"
+        },
+        "body": "{\"model\":\"jev-1.13-free\",\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}"
+      },
+      "response": {
+        "status": 200,
+        "headers": {
+          "content-type": "application/json"
+        },
+        "body": "{\"model\":\"jev-1.13-free\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":1,\"probabilities\":{\"technical\":0,\"sales\":0,\"billing\":1}},\"urgency\":{\"type\":\"score\",\"score\":1.02,\"confidence\":0.95,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0,\"1\":0.97,\"2\":0.03}},\"refund\":{\"type\":\"noul\",\"noul\":0.99}},\"usage\":{\"input_tokens\":422,\"output_tokens\":69}}"
+      }
+    }
+  ]
+}

packages/ai/test/fixtures/recordings/typesafe-evaluation/evaluates-choice-score-and-boolean-questions.json

@@ -0,0 +1,33 @@
+{
+  "version": 1,
+  "metadata": {
+    "model": "jev-latest",
+    "tags": [
+      "prefix:typesafe-evaluation",
+      "provider:typesafe-ai",
+      "protocol:system-one"
+    ],
+    "name": "typesafe-evaluation/evaluates-choice-score-and-boolean-questions",
+    "recordedAt": "2026-09-22T03:44:28.626Z"
+  },
+  "interactions": [
+    {
+      "transport": "http",
+      "request": {
+        "method": "POST",
+        "url": "https://api.typesafe.ai/v1/systemone",
+        "headers": {
+          "content-type": "application/json"
+        },
+        "body": "{\"model\":\"jev-latest\",\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}"
+      },
+      "response": {
+        "status": 200,
+        "headers": {
+          "content-type": "application/json"
+        },
+        "body": "{\"model\":\"jev-1.13.0\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":1.0,\"probabilities\":{\"billing\":1.0,\"sales\":0.0,\"technical\":0.0}},\"urgency\":{\"type\":\"score\",\"score\":1.01,\"confidence\":0.97,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0.0,\"1\":0.98,\"2\":0.02}},\"refund\":{\"type\":\"noul\",\"noul\":0.99}},\"usage\":{\"input_tokens\":422,\"output_tokens\":69}}"
+      }
+    }
+  ]
+}

packages/ai/test/provider/evaluation.recorded.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect } from "bun:test"
+import { Effect } from "effect"
+import { Evaluation } from "../../src/experimental.js"
+import { OpenCodeZen, TypeSafeAI } from "../../src/providers.js"
+import { recordedTests } from "../recorded-test.js"
+
+const questions = {
+  department: {
+    type: "choice",
+    instructions: "Which team should handle this support request?",
+    criteria: {
+      billing: "Payments, invoices, refunds, or failed charges",
+      technical: "Bugs, outages, or integrations",
+      sales: "Pricing, upgrades, or new accounts",
+    },
+  },
+  urgency: {
+    type: "score",
+    instructions: "How urgent is this support request?",
+    criteria: ["Can wait for normal support", "Needs prompt attention", "Actively blocking revenue"],
+  },
+  refund: { type: "boolean", instructions: "Is the customer asking for a refund?" },
+} as const
+
+const state = "I was charged twice for the same invoice. Please refund the duplicate payment today."
+
+const typesafe = recordedTests({
+  prefix: "typesafe-evaluation",
+  provider: "typesafe-ai",
+  protocol: "system-one",
+  requires: ["TYPESAFE_API_KEY"],
+  metadata: { model: "jev-latest" },
+})
+
+const zen = recordedTests({
+  prefix: "opencode-zen-evaluation",
+  provider: "opencode",
+  protocol: "system-one",
+  requires: ["OPENCODE_API_KEY"],
+  metadata: { model: "jev-1.13-free" },
+})
+
+describe("experimental Evaluation recorded", () => {
+  typesafe.effect("evaluates choice score and boolean questions", () =>
+    assertEvaluation(
+      TypeSafeAI.configure({ apiKey: process.env.TYPESAFE_API_KEY ?? "fixture" }).experimental.evaluation("jev-latest"),
+      "typesafe",
+    ),
+  )
+
+  zen.effect("evaluates choice score and boolean questions", () =>
+    assertEvaluation(
+      OpenCodeZen.configure({ apiKey: process.env.OPENCODE_API_KEY ?? "fixture" }).experimental.evaluation(
+        "jev-1.13-free",
+      ),
+      "opencode",
+    ),
+  )
+})
+
+const assertEvaluation = (
+  model: ReturnType<typeof TypeSafeAI.experimental.evaluation>,
+  metadataKey: "typesafe" | "opencode",
+) =>
+  Effect.gen(function* () {
+    const response = yield* Evaluation.evaluate({ model, state, questions })
+    expect(response.model).toStartWith("jev-")
+    expect(response.answers.department.type).toBe("choice")
+    expect(response.answers.department.choice).toBe("billing")
+    expect(response.answers.department.probabilities?.billing).toBeGreaterThan(0.5)
+    expect(response.answers.urgency.type).toBe("score")
+    expect(response.answers.urgency.score).toBeGreaterThanOrEqual(0)
+    expect(response.answers.urgency.score).toBeLessThanOrEqual(2)
+    expect(response.answers.refund.type).toBe("boolean")
+    expect(response.answers.refund.probability).toBeGreaterThan(0.5)
+    expect(response.usage?.inputTokens).toBeGreaterThan(0)
+    expect(response.usage?.outputTokens).toBeGreaterThan(0)
+    expect(response.providerMetadata?.[metadataKey]?.confidence).toBeDefined()
+  })

packages/ai/test/recorded-test.ts

@@ -6,6 +6,8 @@ import * as path from "node:path"
 import { fileURLToPath } from "node:url"
 import { LLMClient, RequestExecutor } from "../src/route.js"
 import { ImageClient } from "../src/image-client.js"
+import { EvaluationClient } from "../src/experimental/evaluation-client.js"
+import type { Service as EvaluationClientService } from "../src/experimental/evaluation-client.js"
 import type { Service as ImageClientService } from "../src/image-client.js"
 import type { Service as LLMClientService } from "../src/route/client.js"
 import type { Service as RequestExecutorService } from "../src/route/executor.js"
@@ -18,7 +20,12 @@ import {
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 const FIXTURES_DIR = path.resolve(__dirname, "fixtures", "recordings")
 
-type RecordedEnv = RequestExecutorService | LLMClientService | ImageClientService | Socket.WebSocketConstructor
+type RecordedEnv =
+  | RequestExecutorService
+  | LLMClientService
+  | ImageClientService
+  | EvaluationClientService
+  | Socket.WebSocketConstructor
 
 type RecordedTestsOptions = RecordedGroupOptions & {
   readonly options?: HttpRecorder.RecorderOptions
@@ -92,6 +99,7 @@ export const recordedTests = (options: RecordedTestsOptions) =>
         requestExecutor,
         LLMClient.layer.pipe(Layer.provide(requestExecutor)),
         ImageClient.layer.pipe(Layer.provide(requestExecutor)),
+        EvaluationClient.layer.pipe(Layer.provide(requestExecutor)),
         webSocket,
       )
     },