feat(ai): add OpenAI Decisions evaluation route (#58382)

This commit is contained in:
Aiden Cline
2026-10-23 09:42:08 -05:00
committed by GitHub
Unverified
parent 24b5ed8bcc
commit 71f3ca1860
7 changed files with 397 additions and 5 deletions
+7 -3
View File
@@ -183,17 +183,21 @@ console.log(response.answers.refund.probability)
```
`TypeSafeAI` reads `TYPESAFE_API_KEY`. `OpenCode` exposes the same selector and reads
`OPENCODE_API_KEY`. OpenRouter and Vercel AI Gateway use the same provider shape:
`OPENCODE_API_KEY`. OpenRouter, Vercel AI Gateway, and OpenAI use the same provider shape:
```ts
import { OpenRouter, VercelAIGateway } from "@opencode/ai/providers"
import { OpenAI, OpenRouter, VercelAIGateway } from "@opencode/ai/providers"
OpenRouter.configure().experimental.evaluation("typesafe/jev-1.13")
VercelAIGateway.configure().experimental.evaluation("typesafe-ai/jev")
OpenAI.configure().experimental.evaluation("gpt-6-luna")
```
OpenRouter reads `OPENROUTER_API_KEY`. Vercel reads `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN`.
The common API uses `boolean`; System One routes lower it to native `noul`.
OpenAI reads `OPENAI_API_KEY` and calls `/v1/decisions`.
The common API uses `boolean`; System One routes lower it to native `noul`, and OpenAI lowers it to
`predicate`. OpenAI predicates take no criteria, so boolean criteria fail before the request, and a
refused question fails the call with a `ContentPolicy` error.
Choice and score answers include `confidence` when the provider returns it, such as
`response.answers.department.confidence`. Score legends remain available in provider metadata, and
the provider's rounded probabilities are returned unchanged.
@@ -0,0 +1,148 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/http"
import {
EvaluationModel,
EvaluationResponse,
EvaluationRounding,
type EvaluationAnswer,
type EvaluationInput,
type EvaluationRequestFor,
} from "./evaluation.js"
import { OpenResponses } from "../protocols/open-responses.js"
import { Endpoint } from "../route/endpoint.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ModelID, mergeJsonRecords, type OpenString } from "../schema/index.js"
import { encodeJson } from "../utils/json.js"
const route = MediaProtocol.identity({ id: "openai-decisions", name: "OpenAI Decisions", provider: "openai" })
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/decisions"
/** Native body fields, sent as given. */
export type OpenAIDecisionOptions = {
readonly prompt_cache_key?: string
readonly prompt_cache_options?: Readonly<{ mode?: OpenString<"implicit" | "explicit">; ttl?: OpenString<"30m"> }>
readonly safety_identifier?: string
} & Record<string, unknown>
type Request = EvaluationRequestFor<OpenAIDecisionOptions>
const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 }))
const Answer = Schema.Union([
Schema.Struct({ type: Schema.Literal("predicate"), name: Schema.String, probability: Probability }),
Schema.Struct({
type: Schema.Literal("choice"),
name: Schema.String,
choice: Schema.String,
probabilities: Schema.Array(Schema.Struct({ value: Schema.String, probability: Probability })),
confidence: Schema.optional(Probability),
}),
Schema.Struct({
type: Schema.Literal("score"),
name: Schema.String,
score: Schema.Number,
probabilities: Schema.Array(Schema.Struct({ value: Schema.Int, probability: Probability })),
confidence: Schema.optional(Probability),
}),
Schema.Struct({ type: Schema.Literal("refusal"), name: Schema.String }),
])
const Response = Schema.Struct({
model: Schema.String,
answers: Schema.Array(Answer),
usage: Schema.optional(OpenResponses.OpenResponsesUsage),
})
const fromRequest = Effect.fn("OpenAIDecisions.fromRequest")(function* (request: Request) {
const criteria = Object.entries(request.questions).flatMap(([name, question]) =>
question.type === "boolean" && question.criteria !== undefined ? [name] : [],
)
if (criteria.length > 0)
return yield* route.unsupported(
"evaluation.criteria",
`${route.name} boolean questions take instructions only; remove criteria from ${criteria.join(", ")}`,
)
return MediaProtocol.json({
...mergeJsonRecords(request.options, request.http?.body),
model: request.model.id,
input: typeof request.state === "string" ? request.state : [{ role: "user", content: json(request.state) }],
questions: Object.entries(request.questions).map(([name, question]) => {
const instructions = typeof question.instructions === "string" ? question.instructions : json(question.instructions)
if (question.type === "boolean") return { type: "predicate", name, instructions }
if (question.type === "choice")
return {
type: "choice",
name,
instructions,
choices: Object.entries(question.criteria).map(([value, description]) => ({
value,
...(description === null ? {} : { description: text(description) }),
})),
}
return {
type: "score",
name,
instructions,
levels: question.criteria.map((level, index) =>
typeof level === "string"
? { label: level }
: { label: String(index), ...(level === null ? {} : { description: text(level) }) },
),
}
}),
})
})
const json = (data: EvaluationInput) => [{ type: "input_json", data }]
const text = (value: EvaluationInput) => (typeof value === "string" ? value : encodeJson(value))
const decodeDocument = route.decodeJson(Response)
const decodeResponse = Effect.fn("OpenAIDecisions.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* decodeDocument(response)
const refused = output.value.answers.flatMap((answer) => (answer.type === "refusal" ? [answer.name] : []))
if (refused.length > 0) return yield* output.contentPolicy(`${route.name} refused to answer ${refused.join(", ")}`)
return new EvaluationResponse({
model: ModelID.make(output.value.model),
answers: Object.fromEntries(
output.value.answers.flatMap((answer): Array<[string, EvaluationAnswer]> => {
if (answer.type === "refusal") return []
if (answer.type === "predicate") return [[answer.name, { type: "boolean", probability: answer.probability }]]
const probabilities = Object.fromEntries(answer.probabilities.map((item) => [item.value, item.probability]))
const confidence = answer.confidence === undefined ? {} : { confidence: answer.confidence }
if (answer.type === "choice")
return [[answer.name, { type: "choice", choice: answer.choice, probabilities, ...confidence }]]
return [[answer.name, { type: "score", score: answer.score, probabilities, ...confidence }]]
}),
),
usage: OpenResponses.mapUsage(output.value.usage, "openai"),
rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }),
})
})
export const protocol = MediaProtocol.inline<Request, EvaluationResponse>(route, {
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) => {
const composed = MediaRoute.inline({
protocol,
endpoint: Endpoint.path<MediaProtocol.Body, Request>(PATH, { baseURL: input.baseURL ?? DEFAULT_BASE_URL }),
auth: input.auth,
headers: input.headers,
})
return EvaluationModel.make<OpenAIDecisionOptions>({
id: input.id,
provider: protocol.provider,
http: input.http,
route: { id: composed.id, evaluate: composed.generate },
})
}
export const OpenAIDecisions = {
protocol,
model,
} as const
+5
View File
@@ -16,11 +16,13 @@ import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-opt
import { OpenAIImages } from "../protocols/openai-images.js"
import { OpenAISpeech } from "../protocols/openai-speech.js"
import { OpenAITranscription } from "../protocols/openai-transcription.js"
import { OpenAIDecisions } from "../experimental/openai-decisions.js"
export type { OpenAIOptionsInput, OpenAIResponseIncludable } from "./openai-options.js"
export type { OpenAIImageOptions } from "../protocols/openai-images.js"
export type { OpenAISpeechOptions } from "../protocols/openai-speech.js"
export type { OpenAITranscriptionOptions } from "../protocols/openai-transcription.js"
export type { OpenAIDecisionOptions } from "../experimental/openai-decisions.js"
export const id = ProviderID.make("openai")
@@ -118,6 +120,7 @@ export const configure = (input: Config = {}) => {
const image = (modelID: string | ModelID) => OpenAIImages.model({ ...media, id: modelID })
const speech = (modelID: string | ModelID) => OpenAISpeech.model({ ...media, id: modelID })
const transcription = (modelID: string | ModelID) => OpenAITranscription.model({ ...media, id: modelID })
const evaluation = (modelID: string | ModelID) => OpenAIDecisions.model({ ...media, id: modelID })
return {
id,
@@ -127,6 +130,7 @@ export const configure = (input: Config = {}) => {
image,
speech,
transcription,
experimental: { evaluation },
configure,
}
}
@@ -175,3 +179,4 @@ export const chat = provider.chat
export const image = provider.image
export const speech = provider.speech
export const transcription = provider.transcription
export const experimental = provider.experimental
+172 -1
View File
@@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/http"
import { Evaluation, EvaluationClient } from "../src/experimental.js"
import { OpenCode, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js"
import { OpenAI, OpenCode, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js"
import { it } from "./lib/effect.js"
import { dynamicResponse } from "./lib/http.js"
@@ -254,6 +254,177 @@ describe("experimental Evaluation", () => {
),
)
it.effect("evaluates through OpenAI Decisions", () =>
Evaluation.run({
model: OpenAI.configure({
apiKey: "openai-key",
baseURL: "https://openai.test/v1",
queryParams: { api: "v1" },
}).experimental.evaluation("gpt-6-luna"),
state: { ticket: "Please refund the duplicate charge." },
questions: {
department: {
type: "choice",
instructions: { question: "Which team should handle this?" },
criteria: { billing: "Payments and refunds", technical: { examples: ["outage"] }, sales: null },
},
urgency: { type: "score", instructions: "How urgent is this?", criteria: ["Can wait", { level: "now" }, null] },
refund: { type: "boolean", instructions: "Is the customer asking for a refund?" },
},
options: { safety_identifier: "user-1" },
}).pipe(
Effect.tap((response) =>
Effect.sync(() => {
expect(response.model).toBe("gpt-6-luna")
expect(response.answers.department).toEqual({
type: "choice",
choice: "billing",
probabilities: { billing: 0.9, technical: 0.1, sales: 0 },
confidence: 0.8,
})
expect(response.answers.urgency).toEqual({
type: "score",
score: 1.1,
probabilities: { "0": 0.1, "1": 0.7, "2": 0.2 },
confidence: 0.4,
})
expect(response.answers.refund).toEqual({ type: "boolean", probability: 0.97 })
}),
),
Effect.provide(
EvaluationClient.layer.pipe(
Layer.provide(
dynamicResponse((input) => {
expect(input.request.url).toBe("https://openai.test/v1/decisions?api=v1")
expect(input.request.headers.authorization).toBe("Bearer openai-key")
expect(JSON.parse(input.text)).toEqual({
safety_identifier: "user-1",
model: "gpt-6-luna",
input: [
{
role: "user",
content: [{ type: "input_json", data: { ticket: "Please refund the duplicate charge." } }],
},
],
questions: [
{
type: "choice",
name: "department",
instructions: [{ type: "input_json", data: { question: "Which team should handle this?" } }],
choices: [
{ value: "billing", description: "Payments and refunds" },
{ value: "technical", description: '{"examples":["outage"]}' },
{ value: "sales" },
],
},
{
type: "score",
name: "urgency",
instructions: "How urgent is this?",
levels: [{ label: "Can wait" }, { label: "1", description: '{"level":"now"}' }, { label: "2" }],
},
{ type: "predicate", name: "refund", instructions: "Is the customer asking for a refund?" },
],
})
return Effect.succeed(
input.respond(
JSON.stringify({
model: "gpt-6-luna",
answers: [
{
type: "choice",
name: "department",
choice: "billing",
probabilities: [
{ value: "billing", probability: 0.9 },
{ value: "technical", probability: 0.1 },
{ value: "sales", probability: 0 },
],
confidence: 0.8,
},
{
type: "score",
name: "urgency",
score: 1.1,
probabilities: [
{ value: 0, label: "Can wait", probability: 0.1 },
{ value: 1, label: "1", probability: 0.7 },
{ value: 2, label: "2", probability: 0.2 },
],
confidence: 0.4,
},
{ type: "predicate", name: "refund", probability: 0.97 },
],
}),
{ headers: { "content-type": "application/json" } },
),
)
}),
),
),
),
),
)
it.effect("fails OpenAI Decisions refusals as content policy errors", () =>
Evaluation.run({
model: OpenAI.configure({ apiKey: "openai-key" }).experimental.evaluation("gpt-6-luna"),
state: "hello",
questions: {
greeting: { type: "boolean", instructions: "Is this a greeting?" },
unsafe: { type: "boolean", instructions: "Something the model declines" },
},
}).pipe(
Effect.flip,
Effect.tap((error) =>
Effect.sync(() => {
expect(error.reason._tag).toBe("ContentPolicy")
expect(error.reason.message).toContain("unsafe")
}),
),
Effect.provide(
EvaluationClient.layer.pipe(
Layer.provide(
dynamicResponse((input) =>
Effect.succeed(
input.respond(
JSON.stringify({
model: "gpt-6-luna",
answers: [
{ type: "predicate", name: "greeting", probability: 0.99 },
{ type: "refusal", name: "unsafe" },
],
}),
{ headers: { "content-type": "application/json" } },
),
),
),
),
),
),
),
)
it.effect("rejects boolean criteria for OpenAI Decisions before network I/O", () =>
Effect.gen(function* () {
const error = yield* Evaluation.run({
model: OpenAI.configure({ apiKey: "openai-key" }).experimental.evaluation("gpt-6-luna"),
state: "hello",
questions: {
greeting: { type: "boolean", instructions: "Is this a greeting?", criteria: { true: "Says hello" } },
},
}).pipe(Effect.flip)
expect(error.reason._tag).toBe("UnsupportedOperation")
expect(error.reason.message).toContain("greeting")
}).pipe(
Effect.provide(
EvaluationClient.layer.pipe(
Layer.provide(dynamicResponse(() => Effect.die("invalid evaluation reached the network"))),
),
),
),
)
it.effect("rejects answers that do not match their questions", () =>
Evaluation.run({
model: VercelAIGateway.configure({
+1
View File
@@ -151,6 +151,7 @@ describe("public exports", () => {
expect(TypeSafeAI.experimental.evaluation).toBeFunction()
expect(OpenCode.experimental.evaluation).toBeFunction()
expect(VercelAIGateway.experimental.evaluation).toBeFunction()
expect(OpenAI.experimental.evaluation("gpt-6-luna").route.id).toBe("openai-decisions")
expect(XAI.model).toBeFunction()
expect(XAI.provider.responses).toBe(XAI.responses)
expect(XAI.provider.chat).toBe(XAI.chat)
@@ -0,0 +1,33 @@
{
"version": 1,
"metadata": {
"model": "gpt-6-luna",
"tags": [
"prefix:openai-evaluation",
"provider:openai",
"protocol:openai-decisions"
],
"name": "openai-evaluation/evaluates-choice-score-and-boolean-questions-about-json-state",
"recordedAt": "2026-10-23T07:45:33.569Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.openai.com/v1/decisions",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"gpt-6-luna\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_json\",\"data\":{\"ticket\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"plan\":\"pro\"}}]}],\"questions\":[{\"type\":\"choice\",\"name\":\"department\",\"instructions\":\"Which team should handle this support request?\",\"choices\":[{\"value\":\"billing\",\"description\":\"Payments, invoices, refunds, or failed charges\"},{\"value\":\"technical\",\"description\":\"Bugs, outages, or integrations\"},{\"value\":\"sales\",\"description\":\"Pricing, upgrades, or new accounts\"}]},{\"type\":\"score\",\"name\":\"urgency\",\"instructions\":\"How urgent is this support request?\",\"levels\":[{\"label\":\"Can wait for normal support\"},{\"label\":\"Needs prompt attention\"},{\"label\":\"Actively blocking revenue\"}]},{\"type\":\"predicate\",\"name\":\"refund\",\"instructions\":\"Is the customer asking for a refund?\"}]}"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"gpt-6-luna\",\"answers\":[{\"type\":\"choice\",\"name\":\"department\",\"choice\":\"billing\",\"probabilities\":[{\"value\":\"billing\",\"probability\":1.0},{\"value\":\"technical\",\"probability\":0.0},{\"value\":\"sales\",\"probability\":0.0}],\"confidence\":1.0},{\"type\":\"score\",\"name\":\"urgency\",\"score\":1.04,\"probabilities\":[{\"value\":0,\"label\":\"Can wait for normal support\",\"probability\":0.01},{\"value\":1,\"label\":\"Needs prompt attention\",\"probability\":0.94},{\"value\":2,\"label\":\"Actively blocking revenue\",\"probability\":0.05}],\"confidence\":0.91},{\"type\":\"predicate\",\"name\":\"refund\",\"probability\":1.0}],\"usage\":{\"input_tokens\":430,\"input_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":430}}"
}
}
]
}
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Evaluation, EvaluationModel, type EvaluationOptions } from "../../src/experimental.js"
import { OpenCode, OpenRouter, TypeSafeAI } from "../../src/providers.js"
import { OpenAI, OpenCode, OpenRouter, TypeSafeAI } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
const questions = {
@@ -48,6 +48,14 @@ const openrouter = recordedTests({
metadata: { model: "typesafe/jev-1.13" },
})
const openai = recordedTests({
prefix: "openai-evaluation",
provider: "openai",
protocol: "openai-decisions",
requires: ["OPENAI_API_KEY"],
metadata: { model: "gpt-6-luna" },
})
describe("experimental Evaluation recorded", () => {
typesafe.effect("evaluates choice score and boolean questions", () =>
assertEvaluation(
@@ -70,6 +78,28 @@ describe("experimental Evaluation recorded", () => {
),
),
)
openai.effect("evaluates choice score and boolean questions about JSON state", () =>
Effect.gen(function* () {
const response = yield* Evaluation.run({
model: OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY ?? "fixture" }).experimental.evaluation(
"gpt-6-luna",
),
state: { ticket: state, plan: "pro" },
questions,
})
expect(response.model).toBe("gpt-6-luna")
expect(response.answers.department.choice).toBe("billing")
expect(response.answers.department.probabilities?.billing).toBeGreaterThan(0.5)
expect(response.answers.department.confidence).toBeGreaterThan(0)
expect(Object.keys(response.answers.urgency.probabilities ?? {})).toEqual(["0", "1", "2"])
expect(response.answers.urgency.score).toBeGreaterThanOrEqual(0)
expect(response.answers.urgency.score).toBeLessThanOrEqual(2)
expect(response.answers.refund.probability).toBeGreaterThan(0.5)
expect(response.usage?.inputTokens).toBeGreaterThan(0)
expect(response.usage?.outputTokens).toBe(0)
}),
)
})
const assertEvaluation = <Options extends EvaluationOptions>(model: EvaluationModel<Options>) =>