mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-17 05:26:47 +03:00
feat(ai): add OpenAI Decisions evaluation route (#58382)
This commit is contained in:
committed by
GitHub
Unverified
parent
24b5ed8bcc
commit
71f3ca1860
@@ -183,17 +183,21 @@ console.log(response.answers.refund.probability)
|
||||
```
|
||||
|
||||
`TypeSafeAI` reads `TYPESAFE_API_KEY`. `OpenCode` exposes the same selector and reads
|
||||
`OPENCODE_API_KEY`. OpenRouter and Vercel AI Gateway use the same provider shape:
|
||||
`OPENCODE_API_KEY`. OpenRouter, Vercel AI Gateway, and OpenAI use the same provider shape:
|
||||
|
||||
```ts
|
||||
import { OpenRouter, VercelAIGateway } from "@opencode/ai/providers"
|
||||
import { OpenAI, OpenRouter, VercelAIGateway } from "@opencode/ai/providers"
|
||||
|
||||
OpenRouter.configure().experimental.evaluation("typesafe/jev-1.13")
|
||||
VercelAIGateway.configure().experimental.evaluation("typesafe-ai/jev")
|
||||
OpenAI.configure().experimental.evaluation("gpt-6-luna")
|
||||
```
|
||||
|
||||
OpenRouter reads `OPENROUTER_API_KEY`. Vercel reads `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN`.
|
||||
The common API uses `boolean`; System One routes lower it to native `noul`.
|
||||
OpenAI reads `OPENAI_API_KEY` and calls `/v1/decisions`.
|
||||
The common API uses `boolean`; System One routes lower it to native `noul`, and OpenAI lowers it to
|
||||
`predicate`. OpenAI predicates take no criteria, so boolean criteria fail before the request, and a
|
||||
refused question fails the call with a `ContentPolicy` error.
|
||||
Choice and score answers include `confidence` when the provider returns it, such as
|
||||
`response.answers.department.confidence`. Score legends remain available in provider metadata, and
|
||||
the provider's rounded probabilities are returned unchanged.
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/http"
|
||||
import {
|
||||
EvaluationModel,
|
||||
EvaluationResponse,
|
||||
EvaluationRounding,
|
||||
type EvaluationAnswer,
|
||||
type EvaluationInput,
|
||||
type EvaluationRequestFor,
|
||||
} from "./evaluation.js"
|
||||
import { OpenResponses } from "../protocols/open-responses.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ModelID, mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { encodeJson } from "../utils/json.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "openai-decisions", name: "OpenAI Decisions", provider: "openai" })
|
||||
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
||||
export const PATH = "/decisions"
|
||||
|
||||
/** Native body fields, sent as given. */
|
||||
export type OpenAIDecisionOptions = {
|
||||
readonly prompt_cache_key?: string
|
||||
readonly prompt_cache_options?: Readonly<{ mode?: OpenString<"implicit" | "explicit">; ttl?: OpenString<"30m"> }>
|
||||
readonly safety_identifier?: string
|
||||
} & Record<string, unknown>
|
||||
|
||||
type Request = EvaluationRequestFor<OpenAIDecisionOptions>
|
||||
|
||||
const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 }))
|
||||
const Answer = Schema.Union([
|
||||
Schema.Struct({ type: Schema.Literal("predicate"), name: Schema.String, probability: Probability }),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("choice"),
|
||||
name: Schema.String,
|
||||
choice: Schema.String,
|
||||
probabilities: Schema.Array(Schema.Struct({ value: Schema.String, probability: Probability })),
|
||||
confidence: Schema.optional(Probability),
|
||||
}),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("score"),
|
||||
name: Schema.String,
|
||||
score: Schema.Number,
|
||||
probabilities: Schema.Array(Schema.Struct({ value: Schema.Int, probability: Probability })),
|
||||
confidence: Schema.optional(Probability),
|
||||
}),
|
||||
Schema.Struct({ type: Schema.Literal("refusal"), name: Schema.String }),
|
||||
])
|
||||
const Response = Schema.Struct({
|
||||
model: Schema.String,
|
||||
answers: Schema.Array(Answer),
|
||||
usage: Schema.optional(OpenResponses.OpenResponsesUsage),
|
||||
})
|
||||
|
||||
const fromRequest = Effect.fn("OpenAIDecisions.fromRequest")(function* (request: Request) {
|
||||
const criteria = Object.entries(request.questions).flatMap(([name, question]) =>
|
||||
question.type === "boolean" && question.criteria !== undefined ? [name] : [],
|
||||
)
|
||||
if (criteria.length > 0)
|
||||
return yield* route.unsupported(
|
||||
"evaluation.criteria",
|
||||
`${route.name} boolean questions take instructions only; remove criteria from ${criteria.join(", ")}`,
|
||||
)
|
||||
return MediaProtocol.json({
|
||||
...mergeJsonRecords(request.options, request.http?.body),
|
||||
model: request.model.id,
|
||||
input: typeof request.state === "string" ? request.state : [{ role: "user", content: json(request.state) }],
|
||||
questions: Object.entries(request.questions).map(([name, question]) => {
|
||||
const instructions = typeof question.instructions === "string" ? question.instructions : json(question.instructions)
|
||||
if (question.type === "boolean") return { type: "predicate", name, instructions }
|
||||
if (question.type === "choice")
|
||||
return {
|
||||
type: "choice",
|
||||
name,
|
||||
instructions,
|
||||
choices: Object.entries(question.criteria).map(([value, description]) => ({
|
||||
value,
|
||||
...(description === null ? {} : { description: text(description) }),
|
||||
})),
|
||||
}
|
||||
return {
|
||||
type: "score",
|
||||
name,
|
||||
instructions,
|
||||
levels: question.criteria.map((level, index) =>
|
||||
typeof level === "string"
|
||||
? { label: level }
|
||||
: { label: String(index), ...(level === null ? {} : { description: text(level) }) },
|
||||
),
|
||||
}
|
||||
}),
|
||||
})
|
||||
})
|
||||
|
||||
const json = (data: EvaluationInput) => [{ type: "input_json", data }]
|
||||
const text = (value: EvaluationInput) => (typeof value === "string" ? value : encodeJson(value))
|
||||
|
||||
const decodeDocument = route.decodeJson(Response)
|
||||
|
||||
const decodeResponse = Effect.fn("OpenAIDecisions.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const output = yield* decodeDocument(response)
|
||||
const refused = output.value.answers.flatMap((answer) => (answer.type === "refusal" ? [answer.name] : []))
|
||||
if (refused.length > 0) return yield* output.contentPolicy(`${route.name} refused to answer ${refused.join(", ")}`)
|
||||
return new EvaluationResponse({
|
||||
model: ModelID.make(output.value.model),
|
||||
answers: Object.fromEntries(
|
||||
output.value.answers.flatMap((answer): Array<[string, EvaluationAnswer]> => {
|
||||
if (answer.type === "refusal") return []
|
||||
if (answer.type === "predicate") return [[answer.name, { type: "boolean", probability: answer.probability }]]
|
||||
const probabilities = Object.fromEntries(answer.probabilities.map((item) => [item.value, item.probability]))
|
||||
const confidence = answer.confidence === undefined ? {} : { confidence: answer.confidence }
|
||||
if (answer.type === "choice")
|
||||
return [[answer.name, { type: "choice", choice: answer.choice, probabilities, ...confidence }]]
|
||||
return [[answer.name, { type: "score", score: answer.score, probabilities, ...confidence }]]
|
||||
}),
|
||||
),
|
||||
usage: OpenResponses.mapUsage(output.value.usage, "openai"),
|
||||
rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }),
|
||||
})
|
||||
})
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, EvaluationResponse>(route, {
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) => {
|
||||
const composed = MediaRoute.inline({
|
||||
protocol,
|
||||
endpoint: Endpoint.path<MediaProtocol.Body, Request>(PATH, { baseURL: input.baseURL ?? DEFAULT_BASE_URL }),
|
||||
auth: input.auth,
|
||||
headers: input.headers,
|
||||
})
|
||||
return EvaluationModel.make<OpenAIDecisionOptions>({
|
||||
id: input.id,
|
||||
provider: protocol.provider,
|
||||
http: input.http,
|
||||
route: { id: composed.id, evaluate: composed.generate },
|
||||
})
|
||||
}
|
||||
|
||||
export const OpenAIDecisions = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -16,11 +16,13 @@ import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-opt
|
||||
import { OpenAIImages } from "../protocols/openai-images.js"
|
||||
import { OpenAISpeech } from "../protocols/openai-speech.js"
|
||||
import { OpenAITranscription } from "../protocols/openai-transcription.js"
|
||||
import { OpenAIDecisions } from "../experimental/openai-decisions.js"
|
||||
|
||||
export type { OpenAIOptionsInput, OpenAIResponseIncludable } from "./openai-options.js"
|
||||
export type { OpenAIImageOptions } from "../protocols/openai-images.js"
|
||||
export type { OpenAISpeechOptions } from "../protocols/openai-speech.js"
|
||||
export type { OpenAITranscriptionOptions } from "../protocols/openai-transcription.js"
|
||||
export type { OpenAIDecisionOptions } from "../experimental/openai-decisions.js"
|
||||
|
||||
export const id = ProviderID.make("openai")
|
||||
|
||||
@@ -118,6 +120,7 @@ export const configure = (input: Config = {}) => {
|
||||
const image = (modelID: string | ModelID) => OpenAIImages.model({ ...media, id: modelID })
|
||||
const speech = (modelID: string | ModelID) => OpenAISpeech.model({ ...media, id: modelID })
|
||||
const transcription = (modelID: string | ModelID) => OpenAITranscription.model({ ...media, id: modelID })
|
||||
const evaluation = (modelID: string | ModelID) => OpenAIDecisions.model({ ...media, id: modelID })
|
||||
|
||||
return {
|
||||
id,
|
||||
@@ -127,6 +130,7 @@ export const configure = (input: Config = {}) => {
|
||||
image,
|
||||
speech,
|
||||
transcription,
|
||||
experimental: { evaluation },
|
||||
configure,
|
||||
}
|
||||
}
|
||||
@@ -175,3 +179,4 @@ export const chat = provider.chat
|
||||
export const image = provider.image
|
||||
export const speech = provider.speech
|
||||
export const transcription = provider.transcription
|
||||
export const experimental = provider.experimental
|
||||
|
||||
@@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClientRequest } from "effect/http"
|
||||
import { Evaluation, EvaluationClient } from "../src/experimental.js"
|
||||
import { OpenCode, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js"
|
||||
import { OpenAI, OpenCode, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
import { dynamicResponse } from "./lib/http.js"
|
||||
|
||||
@@ -254,6 +254,177 @@ describe("experimental Evaluation", () => {
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("evaluates through OpenAI Decisions", () =>
|
||||
Evaluation.run({
|
||||
model: OpenAI.configure({
|
||||
apiKey: "openai-key",
|
||||
baseURL: "https://openai.test/v1",
|
||||
queryParams: { api: "v1" },
|
||||
}).experimental.evaluation("gpt-6-luna"),
|
||||
state: { ticket: "Please refund the duplicate charge." },
|
||||
questions: {
|
||||
department: {
|
||||
type: "choice",
|
||||
instructions: { question: "Which team should handle this?" },
|
||||
criteria: { billing: "Payments and refunds", technical: { examples: ["outage"] }, sales: null },
|
||||
},
|
||||
urgency: { type: "score", instructions: "How urgent is this?", criteria: ["Can wait", { level: "now" }, null] },
|
||||
refund: { type: "boolean", instructions: "Is the customer asking for a refund?" },
|
||||
},
|
||||
options: { safety_identifier: "user-1" },
|
||||
}).pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => {
|
||||
expect(response.model).toBe("gpt-6-luna")
|
||||
expect(response.answers.department).toEqual({
|
||||
type: "choice",
|
||||
choice: "billing",
|
||||
probabilities: { billing: 0.9, technical: 0.1, sales: 0 },
|
||||
confidence: 0.8,
|
||||
})
|
||||
expect(response.answers.urgency).toEqual({
|
||||
type: "score",
|
||||
score: 1.1,
|
||||
probabilities: { "0": 0.1, "1": 0.7, "2": 0.2 },
|
||||
confidence: 0.4,
|
||||
})
|
||||
expect(response.answers.refund).toEqual({ type: "boolean", probability: 0.97 })
|
||||
}),
|
||||
),
|
||||
Effect.provide(
|
||||
EvaluationClient.layer.pipe(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) => {
|
||||
expect(input.request.url).toBe("https://openai.test/v1/decisions?api=v1")
|
||||
expect(input.request.headers.authorization).toBe("Bearer openai-key")
|
||||
expect(JSON.parse(input.text)).toEqual({
|
||||
safety_identifier: "user-1",
|
||||
model: "gpt-6-luna",
|
||||
input: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "input_json", data: { ticket: "Please refund the duplicate charge." } }],
|
||||
},
|
||||
],
|
||||
questions: [
|
||||
{
|
||||
type: "choice",
|
||||
name: "department",
|
||||
instructions: [{ type: "input_json", data: { question: "Which team should handle this?" } }],
|
||||
choices: [
|
||||
{ value: "billing", description: "Payments and refunds" },
|
||||
{ value: "technical", description: '{"examples":["outage"]}' },
|
||||
{ value: "sales" },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "score",
|
||||
name: "urgency",
|
||||
instructions: "How urgent is this?",
|
||||
levels: [{ label: "Can wait" }, { label: "1", description: '{"level":"now"}' }, { label: "2" }],
|
||||
},
|
||||
{ type: "predicate", name: "refund", instructions: "Is the customer asking for a refund?" },
|
||||
],
|
||||
})
|
||||
return Effect.succeed(
|
||||
input.respond(
|
||||
JSON.stringify({
|
||||
model: "gpt-6-luna",
|
||||
answers: [
|
||||
{
|
||||
type: "choice",
|
||||
name: "department",
|
||||
choice: "billing",
|
||||
probabilities: [
|
||||
{ value: "billing", probability: 0.9 },
|
||||
{ value: "technical", probability: 0.1 },
|
||||
{ value: "sales", probability: 0 },
|
||||
],
|
||||
confidence: 0.8,
|
||||
},
|
||||
{
|
||||
type: "score",
|
||||
name: "urgency",
|
||||
score: 1.1,
|
||||
probabilities: [
|
||||
{ value: 0, label: "Can wait", probability: 0.1 },
|
||||
{ value: 1, label: "1", probability: 0.7 },
|
||||
{ value: 2, label: "2", probability: 0.2 },
|
||||
],
|
||||
confidence: 0.4,
|
||||
},
|
||||
{ type: "predicate", name: "refund", probability: 0.97 },
|
||||
],
|
||||
}),
|
||||
{ headers: { "content-type": "application/json" } },
|
||||
),
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("fails OpenAI Decisions refusals as content policy errors", () =>
|
||||
Evaluation.run({
|
||||
model: OpenAI.configure({ apiKey: "openai-key" }).experimental.evaluation("gpt-6-luna"),
|
||||
state: "hello",
|
||||
questions: {
|
||||
greeting: { type: "boolean", instructions: "Is this a greeting?" },
|
||||
unsafe: { type: "boolean", instructions: "Something the model declines" },
|
||||
},
|
||||
}).pipe(
|
||||
Effect.flip,
|
||||
Effect.tap((error) =>
|
||||
Effect.sync(() => {
|
||||
expect(error.reason._tag).toBe("ContentPolicy")
|
||||
expect(error.reason.message).toContain("unsafe")
|
||||
}),
|
||||
),
|
||||
Effect.provide(
|
||||
EvaluationClient.layer.pipe(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.succeed(
|
||||
input.respond(
|
||||
JSON.stringify({
|
||||
model: "gpt-6-luna",
|
||||
answers: [
|
||||
{ type: "predicate", name: "greeting", probability: 0.99 },
|
||||
{ type: "refusal", name: "unsafe" },
|
||||
],
|
||||
}),
|
||||
{ headers: { "content-type": "application/json" } },
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("rejects boolean criteria for OpenAI Decisions before network I/O", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* Evaluation.run({
|
||||
model: OpenAI.configure({ apiKey: "openai-key" }).experimental.evaluation("gpt-6-luna"),
|
||||
state: "hello",
|
||||
questions: {
|
||||
greeting: { type: "boolean", instructions: "Is this a greeting?", criteria: { true: "Says hello" } },
|
||||
},
|
||||
}).pipe(Effect.flip)
|
||||
expect(error.reason._tag).toBe("UnsupportedOperation")
|
||||
expect(error.reason.message).toContain("greeting")
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
EvaluationClient.layer.pipe(
|
||||
Layer.provide(dynamicResponse(() => Effect.die("invalid evaluation reached the network"))),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("rejects answers that do not match their questions", () =>
|
||||
Evaluation.run({
|
||||
model: VercelAIGateway.configure({
|
||||
|
||||
@@ -151,6 +151,7 @@ describe("public exports", () => {
|
||||
expect(TypeSafeAI.experimental.evaluation).toBeFunction()
|
||||
expect(OpenCode.experimental.evaluation).toBeFunction()
|
||||
expect(VercelAIGateway.experimental.evaluation).toBeFunction()
|
||||
expect(OpenAI.experimental.evaluation("gpt-6-luna").route.id).toBe("openai-decisions")
|
||||
expect(XAI.model).toBeFunction()
|
||||
expect(XAI.provider.responses).toBe(XAI.responses)
|
||||
expect(XAI.provider.chat).toBe(XAI.chat)
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"model": "gpt-6-luna",
|
||||
"tags": [
|
||||
"prefix:openai-evaluation",
|
||||
"provider:openai",
|
||||
"protocol:openai-decisions"
|
||||
],
|
||||
"name": "openai-evaluation/evaluates-choice-score-and-boolean-questions-about-json-state",
|
||||
"recordedAt": "2026-10-23T07:45:33.569Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.openai.com/v1/decisions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"gpt-6-luna\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_json\",\"data\":{\"ticket\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"plan\":\"pro\"}}]}],\"questions\":[{\"type\":\"choice\",\"name\":\"department\",\"instructions\":\"Which team should handle this support request?\",\"choices\":[{\"value\":\"billing\",\"description\":\"Payments, invoices, refunds, or failed charges\"},{\"value\":\"technical\",\"description\":\"Bugs, outages, or integrations\"},{\"value\":\"sales\",\"description\":\"Pricing, upgrades, or new accounts\"}]},{\"type\":\"score\",\"name\":\"urgency\",\"instructions\":\"How urgent is this support request?\",\"levels\":[{\"label\":\"Can wait for normal support\"},{\"label\":\"Needs prompt attention\"},{\"label\":\"Actively blocking revenue\"}]},{\"type\":\"predicate\",\"name\":\"refund\",\"instructions\":\"Is the customer asking for a refund?\"}]}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"gpt-6-luna\",\"answers\":[{\"type\":\"choice\",\"name\":\"department\",\"choice\":\"billing\",\"probabilities\":[{\"value\":\"billing\",\"probability\":1.0},{\"value\":\"technical\",\"probability\":0.0},{\"value\":\"sales\",\"probability\":0.0}],\"confidence\":1.0},{\"type\":\"score\",\"name\":\"urgency\",\"score\":1.04,\"probabilities\":[{\"value\":0,\"label\":\"Can wait for normal support\",\"probability\":0.01},{\"value\":1,\"label\":\"Needs prompt attention\",\"probability\":0.94},{\"value\":2,\"label\":\"Actively blocking revenue\",\"probability\":0.05}],\"confidence\":0.91},{\"type\":\"predicate\",\"name\":\"refund\",\"probability\":1.0}],\"usage\":{\"input_tokens\":430,\"input_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":430}}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { Evaluation, EvaluationModel, type EvaluationOptions } from "../../src/experimental.js"
|
||||
import { OpenCode, OpenRouter, TypeSafeAI } from "../../src/providers.js"
|
||||
import { OpenAI, OpenCode, OpenRouter, TypeSafeAI } from "../../src/providers.js"
|
||||
import { recordedTests } from "../recorded-test.js"
|
||||
|
||||
const questions = {
|
||||
@@ -48,6 +48,14 @@ const openrouter = recordedTests({
|
||||
metadata: { model: "typesafe/jev-1.13" },
|
||||
})
|
||||
|
||||
const openai = recordedTests({
|
||||
prefix: "openai-evaluation",
|
||||
provider: "openai",
|
||||
protocol: "openai-decisions",
|
||||
requires: ["OPENAI_API_KEY"],
|
||||
metadata: { model: "gpt-6-luna" },
|
||||
})
|
||||
|
||||
describe("experimental Evaluation recorded", () => {
|
||||
typesafe.effect("evaluates choice score and boolean questions", () =>
|
||||
assertEvaluation(
|
||||
@@ -70,6 +78,28 @@ describe("experimental Evaluation recorded", () => {
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
openai.effect("evaluates choice score and boolean questions about JSON state", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* Evaluation.run({
|
||||
model: OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY ?? "fixture" }).experimental.evaluation(
|
||||
"gpt-6-luna",
|
||||
),
|
||||
state: { ticket: state, plan: "pro" },
|
||||
questions,
|
||||
})
|
||||
expect(response.model).toBe("gpt-6-luna")
|
||||
expect(response.answers.department.choice).toBe("billing")
|
||||
expect(response.answers.department.probabilities?.billing).toBeGreaterThan(0.5)
|
||||
expect(response.answers.department.confidence).toBeGreaterThan(0)
|
||||
expect(Object.keys(response.answers.urgency.probabilities ?? {})).toEqual(["0", "1", "2"])
|
||||
expect(response.answers.urgency.score).toBeGreaterThanOrEqual(0)
|
||||
expect(response.answers.urgency.score).toBeLessThanOrEqual(2)
|
||||
expect(response.answers.refund.probability).toBeGreaterThan(0.5)
|
||||
expect(response.usage?.inputTokens).toBeGreaterThan(0)
|
||||
expect(response.usage?.outputTokens).toBe(0)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
const assertEvaluation = <Options extends EvaluationOptions>(model: EvaluationModel<Options>) =>
|
||||
|
||||
Reference in New Issue
Block a user