From 71f3ca1860939591184ef223a968616268d8c7f5 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:42:08 -0500 Subject: [PATCH] feat(ai): add OpenAI Decisions evaluation route (#58382) --- packages/ai/README.md | 10 +- .../ai/src/experimental/openai-decisions.ts | 148 +++++++++++++++ packages/ai/src/providers/openai.ts | 5 + packages/ai/test/evaluation.test.ts | 173 +++++++++++++++++- packages/ai/test/exports.test.ts | 1 + ...nd-boolean-questions-about-json-state.json | 33 ++++ .../test/provider/evaluation.recorded.test.ts | 32 +++- 7 files changed, 397 insertions(+), 5 deletions(-) create mode 100644 packages/ai/src/experimental/openai-decisions.ts create mode 100644 packages/ai/test/fixtures/recordings/openai-evaluation/evaluates-choice-score-and-boolean-questions-about-json-state.json diff --git a/packages/ai/README.md b/packages/ai/README.md index 1e2ba9d0ce4..b4d07fcef28 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -183,17 +183,21 @@ console.log(response.answers.refund.probability) ``` `TypeSafeAI` reads `TYPESAFE_API_KEY`. `OpenCode` exposes the same selector and reads -`OPENCODE_API_KEY`. OpenRouter and Vercel AI Gateway use the same provider shape: +`OPENCODE_API_KEY`. OpenRouter, Vercel AI Gateway, and OpenAI use the same provider shape: ```ts -import { OpenRouter, VercelAIGateway } from "@opencode/ai/providers" +import { OpenAI, OpenRouter, VercelAIGateway } from "@opencode/ai/providers" OpenRouter.configure().experimental.evaluation("typesafe/jev-1.13") VercelAIGateway.configure().experimental.evaluation("typesafe-ai/jev") +OpenAI.configure().experimental.evaluation("gpt-6-luna") ``` OpenRouter reads `OPENROUTER_API_KEY`. Vercel reads `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN`. -The common API uses `boolean`; System One routes lower it to native `noul`. +OpenAI reads `OPENAI_API_KEY` and calls `/v1/decisions`. +The common API uses `boolean`; System One routes lower it to native `noul`, and OpenAI lowers it to +`predicate`. OpenAI predicates take no criteria, so boolean criteria fail before the request, and a +refused question fails the call with a `ContentPolicy` error. Choice and score answers include `confidence` when the provider returns it, such as `response.answers.department.confidence`. Score legends remain available in provider metadata, and the provider's rounded probabilities are returned unchanged. diff --git a/packages/ai/src/experimental/openai-decisions.ts b/packages/ai/src/experimental/openai-decisions.ts new file mode 100644 index 00000000000..049020e0564 --- /dev/null +++ b/packages/ai/src/experimental/openai-decisions.ts @@ -0,0 +1,148 @@ +import { Effect, Schema } from "effect" +import type { HttpClientResponse } from "effect/http" +import { + EvaluationModel, + EvaluationResponse, + EvaluationRounding, + type EvaluationAnswer, + type EvaluationInput, + type EvaluationRequestFor, +} from "./evaluation.js" +import { OpenResponses } from "../protocols/open-responses.js" +import { Endpoint } from "../route/endpoint.js" +import { MediaProtocol } from "../route/media-protocol.js" +import { MediaRoute } from "../route/media.js" +import { ModelID, mergeJsonRecords, type OpenString } from "../schema/index.js" +import { encodeJson } from "../utils/json.js" + +const route = MediaProtocol.identity({ id: "openai-decisions", name: "OpenAI Decisions", provider: "openai" }) +export const DEFAULT_BASE_URL = "https://api.openai.com/v1" +export const PATH = "/decisions" + +/** Native body fields, sent as given. */ +export type OpenAIDecisionOptions = { + readonly prompt_cache_key?: string + readonly prompt_cache_options?: Readonly<{ mode?: OpenString<"implicit" | "explicit">; ttl?: OpenString<"30m"> }> + readonly safety_identifier?: string +} & Record + +type Request = EvaluationRequestFor + +const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 })) +const Answer = Schema.Union([ + Schema.Struct({ type: Schema.Literal("predicate"), name: Schema.String, probability: Probability }), + Schema.Struct({ + type: Schema.Literal("choice"), + name: Schema.String, + choice: Schema.String, + probabilities: Schema.Array(Schema.Struct({ value: Schema.String, probability: Probability })), + confidence: Schema.optional(Probability), + }), + Schema.Struct({ + type: Schema.Literal("score"), + name: Schema.String, + score: Schema.Number, + probabilities: Schema.Array(Schema.Struct({ value: Schema.Int, probability: Probability })), + confidence: Schema.optional(Probability), + }), + Schema.Struct({ type: Schema.Literal("refusal"), name: Schema.String }), +]) +const Response = Schema.Struct({ + model: Schema.String, + answers: Schema.Array(Answer), + usage: Schema.optional(OpenResponses.OpenResponsesUsage), +}) + +const fromRequest = Effect.fn("OpenAIDecisions.fromRequest")(function* (request: Request) { + const criteria = Object.entries(request.questions).flatMap(([name, question]) => + question.type === "boolean" && question.criteria !== undefined ? [name] : [], + ) + if (criteria.length > 0) + return yield* route.unsupported( + "evaluation.criteria", + `${route.name} boolean questions take instructions only; remove criteria from ${criteria.join(", ")}`, + ) + return MediaProtocol.json({ + ...mergeJsonRecords(request.options, request.http?.body), + model: request.model.id, + input: typeof request.state === "string" ? request.state : [{ role: "user", content: json(request.state) }], + questions: Object.entries(request.questions).map(([name, question]) => { + const instructions = typeof question.instructions === "string" ? question.instructions : json(question.instructions) + if (question.type === "boolean") return { type: "predicate", name, instructions } + if (question.type === "choice") + return { + type: "choice", + name, + instructions, + choices: Object.entries(question.criteria).map(([value, description]) => ({ + value, + ...(description === null ? {} : { description: text(description) }), + })), + } + return { + type: "score", + name, + instructions, + levels: question.criteria.map((level, index) => + typeof level === "string" + ? { label: level } + : { label: String(index), ...(level === null ? {} : { description: text(level) }) }, + ), + } + }), + }) +}) + +const json = (data: EvaluationInput) => [{ type: "input_json", data }] +const text = (value: EvaluationInput) => (typeof value === "string" ? value : encodeJson(value)) + +const decodeDocument = route.decodeJson(Response) + +const decodeResponse = Effect.fn("OpenAIDecisions.decodeResponse")(function* ( + response: HttpClientResponse.HttpClientResponse, +) { + const output = yield* decodeDocument(response) + const refused = output.value.answers.flatMap((answer) => (answer.type === "refusal" ? [answer.name] : [])) + if (refused.length > 0) return yield* output.contentPolicy(`${route.name} refused to answer ${refused.join(", ")}`) + return new EvaluationResponse({ + model: ModelID.make(output.value.model), + answers: Object.fromEntries( + output.value.answers.flatMap((answer): Array<[string, EvaluationAnswer]> => { + if (answer.type === "refusal") return [] + if (answer.type === "predicate") return [[answer.name, { type: "boolean", probability: answer.probability }]] + const probabilities = Object.fromEntries(answer.probabilities.map((item) => [item.value, item.probability])) + const confidence = answer.confidence === undefined ? {} : { confidence: answer.confidence } + if (answer.type === "choice") + return [[answer.name, { type: "choice", choice: answer.choice, probabilities, ...confidence }]] + return [[answer.name, { type: "score", score: answer.score, probabilities, ...confidence }]] + }), + ), + usage: OpenResponses.mapUsage(output.value.usage, "openai"), + rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }), + }) +}) + +export const protocol = MediaProtocol.inline(route, { + body: { from: fromRequest }, + response: { decode: decodeResponse }, +}) + +export const model = (input: MediaRoute.ModelInput) => { + const composed = MediaRoute.inline({ + protocol, + endpoint: Endpoint.path(PATH, { baseURL: input.baseURL ?? DEFAULT_BASE_URL }), + auth: input.auth, + headers: input.headers, + }) + return EvaluationModel.make({ + id: input.id, + provider: protocol.provider, + http: input.http, + route: { id: composed.id, evaluate: composed.generate }, + }) +} + +export const OpenAIDecisions = { + protocol, + model, +} as const diff --git a/packages/ai/src/providers/openai.ts b/packages/ai/src/providers/openai.ts index 4542cf6d030..9b2e547ff6f 100644 --- a/packages/ai/src/providers/openai.ts +++ b/packages/ai/src/providers/openai.ts @@ -16,11 +16,13 @@ import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-opt import { OpenAIImages } from "../protocols/openai-images.js" import { OpenAISpeech } from "../protocols/openai-speech.js" import { OpenAITranscription } from "../protocols/openai-transcription.js" +import { OpenAIDecisions } from "../experimental/openai-decisions.js" export type { OpenAIOptionsInput, OpenAIResponseIncludable } from "./openai-options.js" export type { OpenAIImageOptions } from "../protocols/openai-images.js" export type { OpenAISpeechOptions } from "../protocols/openai-speech.js" export type { OpenAITranscriptionOptions } from "../protocols/openai-transcription.js" +export type { OpenAIDecisionOptions } from "../experimental/openai-decisions.js" export const id = ProviderID.make("openai") @@ -118,6 +120,7 @@ export const configure = (input: Config = {}) => { const image = (modelID: string | ModelID) => OpenAIImages.model({ ...media, id: modelID }) const speech = (modelID: string | ModelID) => OpenAISpeech.model({ ...media, id: modelID }) const transcription = (modelID: string | ModelID) => OpenAITranscription.model({ ...media, id: modelID }) + const evaluation = (modelID: string | ModelID) => OpenAIDecisions.model({ ...media, id: modelID }) return { id, @@ -127,6 +130,7 @@ export const configure = (input: Config = {}) => { image, speech, transcription, + experimental: { evaluation }, configure, } } @@ -175,3 +179,4 @@ export const chat = provider.chat export const image = provider.image export const speech = provider.speech export const transcription = provider.transcription +export const experimental = provider.experimental diff --git a/packages/ai/test/evaluation.test.ts b/packages/ai/test/evaluation.test.ts index 4769d5fb5c5..722de600b5a 100644 --- a/packages/ai/test/evaluation.test.ts +++ b/packages/ai/test/evaluation.test.ts @@ -2,7 +2,7 @@ import { describe, expect } from "bun:test" import { Effect, Layer } from "effect" import { HttpClientRequest } from "effect/http" import { Evaluation, EvaluationClient } from "../src/experimental.js" -import { OpenCode, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js" +import { OpenAI, OpenCode, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js" import { it } from "./lib/effect.js" import { dynamicResponse } from "./lib/http.js" @@ -254,6 +254,177 @@ describe("experimental Evaluation", () => { ), ) + it.effect("evaluates through OpenAI Decisions", () => + Evaluation.run({ + model: OpenAI.configure({ + apiKey: "openai-key", + baseURL: "https://openai.test/v1", + queryParams: { api: "v1" }, + }).experimental.evaluation("gpt-6-luna"), + state: { ticket: "Please refund the duplicate charge." }, + questions: { + department: { + type: "choice", + instructions: { question: "Which team should handle this?" }, + criteria: { billing: "Payments and refunds", technical: { examples: ["outage"] }, sales: null }, + }, + urgency: { type: "score", instructions: "How urgent is this?", criteria: ["Can wait", { level: "now" }, null] }, + refund: { type: "boolean", instructions: "Is the customer asking for a refund?" }, + }, + options: { safety_identifier: "user-1" }, + }).pipe( + Effect.tap((response) => + Effect.sync(() => { + expect(response.model).toBe("gpt-6-luna") + expect(response.answers.department).toEqual({ + type: "choice", + choice: "billing", + probabilities: { billing: 0.9, technical: 0.1, sales: 0 }, + confidence: 0.8, + }) + expect(response.answers.urgency).toEqual({ + type: "score", + score: 1.1, + probabilities: { "0": 0.1, "1": 0.7, "2": 0.2 }, + confidence: 0.4, + }) + expect(response.answers.refund).toEqual({ type: "boolean", probability: 0.97 }) + }), + ), + Effect.provide( + EvaluationClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => { + expect(input.request.url).toBe("https://openai.test/v1/decisions?api=v1") + expect(input.request.headers.authorization).toBe("Bearer openai-key") + expect(JSON.parse(input.text)).toEqual({ + safety_identifier: "user-1", + model: "gpt-6-luna", + input: [ + { + role: "user", + content: [{ type: "input_json", data: { ticket: "Please refund the duplicate charge." } }], + }, + ], + questions: [ + { + type: "choice", + name: "department", + instructions: [{ type: "input_json", data: { question: "Which team should handle this?" } }], + choices: [ + { value: "billing", description: "Payments and refunds" }, + { value: "technical", description: '{"examples":["outage"]}' }, + { value: "sales" }, + ], + }, + { + type: "score", + name: "urgency", + instructions: "How urgent is this?", + levels: [{ label: "Can wait" }, { label: "1", description: '{"level":"now"}' }, { label: "2" }], + }, + { type: "predicate", name: "refund", instructions: "Is the customer asking for a refund?" }, + ], + }) + return Effect.succeed( + input.respond( + JSON.stringify({ + model: "gpt-6-luna", + answers: [ + { + type: "choice", + name: "department", + choice: "billing", + probabilities: [ + { value: "billing", probability: 0.9 }, + { value: "technical", probability: 0.1 }, + { value: "sales", probability: 0 }, + ], + confidence: 0.8, + }, + { + type: "score", + name: "urgency", + score: 1.1, + probabilities: [ + { value: 0, label: "Can wait", probability: 0.1 }, + { value: 1, label: "1", probability: 0.7 }, + { value: 2, label: "2", probability: 0.2 }, + ], + confidence: 0.4, + }, + { type: "predicate", name: "refund", probability: 0.97 }, + ], + }), + { headers: { "content-type": "application/json" } }, + ), + ) + }), + ), + ), + ), + ), + ) + + it.effect("fails OpenAI Decisions refusals as content policy errors", () => + Evaluation.run({ + model: OpenAI.configure({ apiKey: "openai-key" }).experimental.evaluation("gpt-6-luna"), + state: "hello", + questions: { + greeting: { type: "boolean", instructions: "Is this a greeting?" }, + unsafe: { type: "boolean", instructions: "Something the model declines" }, + }, + }).pipe( + Effect.flip, + Effect.tap((error) => + Effect.sync(() => { + expect(error.reason._tag).toBe("ContentPolicy") + expect(error.reason.message).toContain("unsafe") + }), + ), + Effect.provide( + EvaluationClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => + Effect.succeed( + input.respond( + JSON.stringify({ + model: "gpt-6-luna", + answers: [ + { type: "predicate", name: "greeting", probability: 0.99 }, + { type: "refusal", name: "unsafe" }, + ], + }), + { headers: { "content-type": "application/json" } }, + ), + ), + ), + ), + ), + ), + ), + ) + + it.effect("rejects boolean criteria for OpenAI Decisions before network I/O", () => + Effect.gen(function* () { + const error = yield* Evaluation.run({ + model: OpenAI.configure({ apiKey: "openai-key" }).experimental.evaluation("gpt-6-luna"), + state: "hello", + questions: { + greeting: { type: "boolean", instructions: "Is this a greeting?", criteria: { true: "Says hello" } }, + }, + }).pipe(Effect.flip) + expect(error.reason._tag).toBe("UnsupportedOperation") + expect(error.reason.message).toContain("greeting") + }).pipe( + Effect.provide( + EvaluationClient.layer.pipe( + Layer.provide(dynamicResponse(() => Effect.die("invalid evaluation reached the network"))), + ), + ), + ), + ) + it.effect("rejects answers that do not match their questions", () => Evaluation.run({ model: VercelAIGateway.configure({ diff --git a/packages/ai/test/exports.test.ts b/packages/ai/test/exports.test.ts index 371c7713426..f34162a668c 100644 --- a/packages/ai/test/exports.test.ts +++ b/packages/ai/test/exports.test.ts @@ -151,6 +151,7 @@ describe("public exports", () => { expect(TypeSafeAI.experimental.evaluation).toBeFunction() expect(OpenCode.experimental.evaluation).toBeFunction() expect(VercelAIGateway.experimental.evaluation).toBeFunction() + expect(OpenAI.experimental.evaluation("gpt-6-luna").route.id).toBe("openai-decisions") expect(XAI.model).toBeFunction() expect(XAI.provider.responses).toBe(XAI.responses) expect(XAI.provider.chat).toBe(XAI.chat) diff --git a/packages/ai/test/fixtures/recordings/openai-evaluation/evaluates-choice-score-and-boolean-questions-about-json-state.json b/packages/ai/test/fixtures/recordings/openai-evaluation/evaluates-choice-score-and-boolean-questions-about-json-state.json new file mode 100644 index 00000000000..9c064d3e0d5 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/openai-evaluation/evaluates-choice-score-and-boolean-questions-about-json-state.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "metadata": { + "model": "gpt-6-luna", + "tags": [ + "prefix:openai-evaluation", + "provider:openai", + "protocol:openai-decisions" + ], + "name": "openai-evaluation/evaluates-choice-score-and-boolean-questions-about-json-state", + "recordedAt": "2026-10-23T07:45:33.569Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/decisions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-6-luna\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_json\",\"data\":{\"ticket\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"plan\":\"pro\"}}]}],\"questions\":[{\"type\":\"choice\",\"name\":\"department\",\"instructions\":\"Which team should handle this support request?\",\"choices\":[{\"value\":\"billing\",\"description\":\"Payments, invoices, refunds, or failed charges\"},{\"value\":\"technical\",\"description\":\"Bugs, outages, or integrations\"},{\"value\":\"sales\",\"description\":\"Pricing, upgrades, or new accounts\"}]},{\"type\":\"score\",\"name\":\"urgency\",\"instructions\":\"How urgent is this support request?\",\"levels\":[{\"label\":\"Can wait for normal support\"},{\"label\":\"Needs prompt attention\"},{\"label\":\"Actively blocking revenue\"}]},{\"type\":\"predicate\",\"name\":\"refund\",\"instructions\":\"Is the customer asking for a refund?\"}]}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-6-luna\",\"answers\":[{\"type\":\"choice\",\"name\":\"department\",\"choice\":\"billing\",\"probabilities\":[{\"value\":\"billing\",\"probability\":1.0},{\"value\":\"technical\",\"probability\":0.0},{\"value\":\"sales\",\"probability\":0.0}],\"confidence\":1.0},{\"type\":\"score\",\"name\":\"urgency\",\"score\":1.04,\"probabilities\":[{\"value\":0,\"label\":\"Can wait for normal support\",\"probability\":0.01},{\"value\":1,\"label\":\"Needs prompt attention\",\"probability\":0.94},{\"value\":2,\"label\":\"Actively blocking revenue\",\"probability\":0.05}],\"confidence\":0.91},{\"type\":\"predicate\",\"name\":\"refund\",\"probability\":1.0}],\"usage\":{\"input_tokens\":430,\"input_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":430}}" + } + } + ] +} diff --git a/packages/ai/test/provider/evaluation.recorded.test.ts b/packages/ai/test/provider/evaluation.recorded.test.ts index 24570c9759c..49b3098c45c 100644 --- a/packages/ai/test/provider/evaluation.recorded.test.ts +++ b/packages/ai/test/provider/evaluation.recorded.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" import { Evaluation, EvaluationModel, type EvaluationOptions } from "../../src/experimental.js" -import { OpenCode, OpenRouter, TypeSafeAI } from "../../src/providers.js" +import { OpenAI, OpenCode, OpenRouter, TypeSafeAI } from "../../src/providers.js" import { recordedTests } from "../recorded-test.js" const questions = { @@ -48,6 +48,14 @@ const openrouter = recordedTests({ metadata: { model: "typesafe/jev-1.13" }, }) +const openai = recordedTests({ + prefix: "openai-evaluation", + provider: "openai", + protocol: "openai-decisions", + requires: ["OPENAI_API_KEY"], + metadata: { model: "gpt-6-luna" }, +}) + describe("experimental Evaluation recorded", () => { typesafe.effect("evaluates choice score and boolean questions", () => assertEvaluation( @@ -70,6 +78,28 @@ describe("experimental Evaluation recorded", () => { ), ), ) + + openai.effect("evaluates choice score and boolean questions about JSON state", () => + Effect.gen(function* () { + const response = yield* Evaluation.run({ + model: OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY ?? "fixture" }).experimental.evaluation( + "gpt-6-luna", + ), + state: { ticket: state, plan: "pro" }, + questions, + }) + expect(response.model).toBe("gpt-6-luna") + expect(response.answers.department.choice).toBe("billing") + expect(response.answers.department.probabilities?.billing).toBeGreaterThan(0.5) + expect(response.answers.department.confidence).toBeGreaterThan(0) + expect(Object.keys(response.answers.urgency.probabilities ?? {})).toEqual(["0", "1", "2"]) + expect(response.answers.urgency.score).toBeGreaterThanOrEqual(0) + expect(response.answers.urgency.score).toBeLessThanOrEqual(2) + expect(response.answers.refund.probability).toBeGreaterThan(0.5) + expect(response.usage?.inputTokens).toBeGreaterThan(0) + expect(response.usage?.outputTokens).toBe(0) + }), + ) }) const assertEvaluation = (model: EvaluationModel) =>