mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-18 15:28:48 +03:00
refactor(ai): decision questions and answers keyed by name
This commit is contained in:
+11
-11
@@ -266,30 +266,30 @@ const program = ai.decision.generate({
|
||||
{ type: "text", text: "I was charged twice. Please refund the duplicate payment." },
|
||||
{ type: "media", media: Media.url("https://example.com/receipt.png", { mediaType: "image/png" }) },
|
||||
],
|
||||
questions: [
|
||||
{ name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" },
|
||||
{
|
||||
name: "department",
|
||||
questions: {
|
||||
refund: { type: "predicate", instructions: "Is the customer asking for a refund?" },
|
||||
department: {
|
||||
type: "choice",
|
||||
instructions: "Which team should handle this?",
|
||||
choices: [{ value: "billing", description: "Payments and refunds" }, { value: "technical" }],
|
||||
},
|
||||
{
|
||||
name: "urgency",
|
||||
urgency: {
|
||||
type: "score",
|
||||
instructions: "How urgent is this?",
|
||||
levels: [{ label: "Can wait" }, { label: "Needs prompt attention" }, { label: "Blocking revenue" }],
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
|
||||
const response = await Effect.runPromise(program.pipe(Effect.provide(DecisionClient.fetchLayer)))
|
||||
const [refund, department] = response.answers
|
||||
const department = response.answers.department
|
||||
if (department.type === "choice") department.choice // "billing" | "technical"
|
||||
```
|
||||
|
||||
`input` is a JSON object or an ordered list of `text`, `json` and `media` parts. Answers come back in question order.
|
||||
A question the host declines is `{ name, type: "error", reason, message? }`, with `reason` such as `"refusal"`; the
|
||||
other answers still come back.
|
||||
`input` is a JSON object or an ordered list of `text`, `json` and `media` parts. Questions are keyed by name, and
|
||||
answers come back under the same names; key order is the order sent to the host.
|
||||
A question the host declines is `{ type: "error", reason, message? }`, with `reason` such as `"refusal"`; the other
|
||||
answers still come back.
|
||||
`response.usage` has `inputTokens`, `outputTokens`, `totalTokens` and the host's reported `cost` when it sends one;
|
||||
the host's raw usage stays in `providerMetadata`.
|
||||
Media goes wherever the host reads it, and a kind the host has no place for fails the call. Host-specific fields go in
|
||||
|
||||
@@ -2,7 +2,6 @@ import { Context, Effect, Layer } from "effect"
|
||||
import type {
|
||||
DecisionAnswer,
|
||||
DecisionOptions,
|
||||
DecisionQuestion,
|
||||
DecisionQuestions,
|
||||
DecisionRequestFor,
|
||||
DecisionResponseFor,
|
||||
@@ -49,11 +48,11 @@ export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer
|
||||
export const fetchLayer = layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
|
||||
|
||||
/** Host bugs (a missing or extra answer, an option that wasn't offered, a score off the scale) fail the whole call. */
|
||||
const answersMatch = (questions: ReadonlyArray<DecisionQuestion>, answers: ReadonlyArray<DecisionAnswer>) =>
|
||||
questions.length === answers.length &&
|
||||
questions.every((question, index) => {
|
||||
const answer = answers[index]
|
||||
if (answer.name !== question.name) return false
|
||||
const answersMatch = (questions: DecisionQuestions, answers: Readonly<Record<string, DecisionAnswer>>) =>
|
||||
Object.keys(questions).length === Object.keys(answers).length &&
|
||||
Object.entries(questions).every(([name, question]) => {
|
||||
if (!Object.hasOwn(answers, name)) return false
|
||||
const answer = answers[name]
|
||||
if (answer.type === "error") return true
|
||||
if (question.type === "predicate") return answer.type === "predicate"
|
||||
if (question.type === "choice") {
|
||||
|
||||
+11
-21
@@ -40,14 +40,12 @@ export const ChoiceValue = Schema.Union([Schema.String, Schema.Boolean])
|
||||
export type ChoiceValue = Schema.Schema.Type<typeof ChoiceValue>
|
||||
|
||||
export const PredicateQuestion = Schema.Struct({
|
||||
name: Schema.String,
|
||||
type: Schema.Literal("predicate"),
|
||||
instructions: Schema.String,
|
||||
})
|
||||
export type PredicateQuestion = Schema.Schema.Type<typeof PredicateQuestion>
|
||||
|
||||
export const ChoiceQuestion = Schema.Struct({
|
||||
name: Schema.String,
|
||||
type: Schema.Literal("choice"),
|
||||
instructions: Schema.String,
|
||||
choices: Schema.Array(Schema.Struct({ value: ChoiceValue, description: Schema.optional(Schema.String) })),
|
||||
@@ -55,7 +53,6 @@ export const ChoiceQuestion = Schema.Struct({
|
||||
export type ChoiceQuestion = Schema.Schema.Type<typeof ChoiceQuestion>
|
||||
|
||||
export const ScoreQuestion = Schema.Struct({
|
||||
name: Schema.String,
|
||||
type: Schema.Literal("score"),
|
||||
instructions: Schema.String,
|
||||
levels: Schema.Array(Schema.Struct({ label: Schema.String, description: Schema.optional(Schema.String) })),
|
||||
@@ -66,14 +63,12 @@ export const DecisionQuestion = Schema.Union([PredicateQuestion, ChoiceQuestion,
|
||||
Schema.toTaggedUnion("type"),
|
||||
)
|
||||
export type DecisionQuestion = Schema.Schema.Type<typeof DecisionQuestion>
|
||||
export type DecisionQuestions = ReadonlyArray<DecisionQuestion>
|
||||
/** Questions keyed by name; key order is the order sent to the host. */
|
||||
export type DecisionQuestions = Readonly<Record<string, DecisionQuestion>>
|
||||
|
||||
// Only what makes a request meaningless is checked here; counts and sizes are the host's to enforce.
|
||||
const DecisionQuestionList = Schema.Array(DecisionQuestion).pipe(
|
||||
Schema.refine((x): x is typeof x => x.length > 0, { message: "Decision questions must not be empty" }),
|
||||
Schema.refine((x): x is typeof x => new Set(x.map((question) => question.name)).size === x.length, {
|
||||
message: "Decision question names must be unique",
|
||||
}),
|
||||
const DecisionQuestionRecord = Schema.Record(Schema.String, DecisionQuestion).pipe(
|
||||
Schema.refine((x): x is typeof x => Object.keys(x).length > 0, { message: "Decision questions must not be empty" }),
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -83,14 +78,12 @@ const DecisionQuestionList = Schema.Array(DecisionQuestion).pipe(
|
||||
const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 }))
|
||||
|
||||
export const PredicateAnswer = Schema.Struct({
|
||||
name: Schema.String,
|
||||
type: Schema.Literal("predicate"),
|
||||
probability: Probability,
|
||||
})
|
||||
export type PredicateAnswer = Schema.Schema.Type<typeof PredicateAnswer>
|
||||
|
||||
export const ChoiceAnswer = Schema.Struct({
|
||||
name: Schema.String,
|
||||
type: Schema.Literal("choice"),
|
||||
choice: ChoiceValue,
|
||||
probabilities: Schema.Array(Schema.Struct({ value: ChoiceValue, probability: Probability })),
|
||||
@@ -100,7 +93,6 @@ export type ChoiceAnswer = Schema.Schema.Type<typeof ChoiceAnswer>
|
||||
|
||||
/** `score` is the probability-weighted level index; `probabilities` has one entry per level. */
|
||||
export const ScoreAnswer = Schema.Struct({
|
||||
name: Schema.String,
|
||||
type: Schema.Literal("score"),
|
||||
score: Schema.Number,
|
||||
probabilities: Schema.Array(Schema.Struct({ value: Schema.Int, label: Schema.String, probability: Probability })),
|
||||
@@ -114,7 +106,6 @@ export const ErrorReason: Schema.Codec<ErrorReason> = Schema.Union([Schema.Liter
|
||||
|
||||
/** This one question has no answer; the other answers still came back. */
|
||||
export const ErrorAnswer = Schema.Struct({
|
||||
name: Schema.String,
|
||||
type: Schema.Literal("error"),
|
||||
reason: ErrorReason,
|
||||
message: Schema.optional(Schema.String),
|
||||
@@ -132,10 +123,9 @@ type ChoiceValueOf<Question> = Question extends { readonly choices: ReadonlyArra
|
||||
|
||||
export type AnswerFor<Question extends DecisionQuestion> =
|
||||
| (Question extends { readonly type: "predicate" }
|
||||
? PredicateAnswer & { readonly name: Question["name"] }
|
||||
? PredicateAnswer
|
||||
: Question extends { readonly type: "choice" }
|
||||
? {
|
||||
readonly name: Question["name"]
|
||||
readonly type: "choice"
|
||||
readonly choice: ChoiceValueOf<Question>
|
||||
readonly probabilities: ReadonlyArray<{
|
||||
@@ -144,12 +134,12 @@ export type AnswerFor<Question extends DecisionQuestion> =
|
||||
}>
|
||||
readonly confidence: number
|
||||
}
|
||||
: ScoreAnswer & { readonly name: Question["name"] })
|
||||
| (ErrorAnswer & { readonly name: Question["name"] })
|
||||
: ScoreAnswer)
|
||||
| ErrorAnswer
|
||||
|
||||
/** One answer per question, in question order. */
|
||||
/** One answer per question, keyed by question name. */
|
||||
export type AnswersFor<Questions extends DecisionQuestions> = {
|
||||
readonly [Index in keyof Questions]: AnswerFor<Questions[Index]>
|
||||
readonly [Name in keyof Questions]: AnswerFor<Questions[Name]>
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -241,7 +231,7 @@ export type DecisionModelOptions<Model> = Model extends DecisionModel<infer Opti
|
||||
export class DecisionRequest extends Schema.Class<DecisionRequest>("Decision.Request")({
|
||||
model: DecisionModelSchema,
|
||||
input: Schema.Array(DecisionPart),
|
||||
questions: DecisionQuestionList,
|
||||
questions: DecisionQuestionRecord,
|
||||
providerOptions: Schema.optional(ProviderOptions),
|
||||
http: Schema.optional(HttpOptions),
|
||||
}) {
|
||||
@@ -278,7 +268,7 @@ export class DecisionUsage extends Schema.Class<DecisionUsage>("Decision.Usage")
|
||||
}) {}
|
||||
|
||||
export class DecisionResponse extends Schema.Class<DecisionResponse>("Decision.Response")({
|
||||
answers: Schema.Array(DecisionAnswer),
|
||||
answers: Schema.Record(Schema.String, DecisionAnswer),
|
||||
usage: Schema.optional(DecisionUsage),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}) {}
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { DecisionResponse, DecisionUsage, type DecisionAnswer, type DecisionPart, type DecisionRequestFor } from "../decision.js"
|
||||
import {
|
||||
DecisionResponse,
|
||||
DecisionUsage,
|
||||
type DecisionAnswer,
|
||||
type DecisionPart,
|
||||
type DecisionRequestFor,
|
||||
} from "../decision.js"
|
||||
import { DecisionRoute } from "../route/decision.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
@@ -66,7 +72,7 @@ const fromRequest = Effect.fn("OpenAIDecisions.fromRequest")(function* (request:
|
||||
),
|
||||
model: request.model.id,
|
||||
input: [{ role: "user", content: yield* Effect.forEach(request.input, (part) => inputPart(request, part)) }],
|
||||
questions: request.questions,
|
||||
questions: Object.entries(request.questions).map(([name, question]) => ({ name, ...question })),
|
||||
}
|
||||
})
|
||||
|
||||
@@ -100,6 +106,13 @@ const inputPart = (
|
||||
|
||||
const isObject = (value: unknown) => typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
|
||||
const toAnswer = (answer: typeof Answer.Type): DecisionAnswer => {
|
||||
if (answer.type === "refusal")
|
||||
return { type: "error", reason: "refusal", ...(answer.message === undefined ? {} : { message: answer.message }) }
|
||||
const { name: _, ...rest } = answer
|
||||
return rest
|
||||
}
|
||||
|
||||
const decodeDocument = MediaProtocol.identity({ id: ID, name: NAME, provider: "openai" }).decodeJson(Response)
|
||||
|
||||
export const protocol = DecisionRoute.protocol({
|
||||
@@ -109,22 +122,20 @@ export const protocol = DecisionRoute.protocol({
|
||||
response: {
|
||||
decode: Effect.fn("OpenAIDecisions.decodeResponse")(function* (response, context) {
|
||||
const output = yield* decodeDocument(response)
|
||||
const answers = output.value.answers.map(
|
||||
(answer): DecisionAnswer =>
|
||||
answer.type === "refusal"
|
||||
? {
|
||||
name: answer.name,
|
||||
type: "error",
|
||||
reason: "refusal",
|
||||
...(answer.message === undefined ? {} : { message: answer.message }),
|
||||
}
|
||||
: answer,
|
||||
// Answers come back as a list in any order; key them by name in question order.
|
||||
const byName = new Map(output.value.answers.map((answer) => [answer.name, toAnswer(answer)]))
|
||||
const names = Object.keys(context.request.questions)
|
||||
if (byName.size !== output.value.answers.length || byName.size !== names.length)
|
||||
return yield* output.invalid(`${NAME} answers do not match the requested questions`, output.value.answers)
|
||||
const answers = Object.fromEntries(
|
||||
names.flatMap((name) => {
|
||||
const answer = byName.get(name)
|
||||
return answer === undefined ? [] : [[name, answer] as const]
|
||||
}),
|
||||
)
|
||||
const byName = new Map(answers.map((answer) => [answer.name, answer]))
|
||||
const ordered = context.request.questions.flatMap((question) => byName.get(question.name) ?? [])
|
||||
const usage = OpenResponses.mapUsage(output.value.usage, context.providerMetadataKey)
|
||||
return new DecisionResponse({
|
||||
answers: ordered.length === answers.length ? ordered : answers,
|
||||
answers,
|
||||
usage:
|
||||
usage === undefined
|
||||
? undefined
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
type DecisionModelCompatibility,
|
||||
type DecisionPart,
|
||||
type DecisionQuestion,
|
||||
type DecisionQuestions,
|
||||
type DecisionRequestFor,
|
||||
} from "../decision.js"
|
||||
import type { Media } from "../media.js"
|
||||
@@ -70,13 +71,13 @@ export const encodeBody = Effect.fn("SystemOne.encodeBody")(function* (
|
||||
return { ...mergeJsonRecords(request.providerOptions, request.http?.body), ...placed, questions }
|
||||
})
|
||||
|
||||
const encodeQuestions = (questions: ReadonlyArray<DecisionQuestion>) =>
|
||||
Effect.forEach(questions, (question): Effect.Effect<readonly [string, unknown], AIError> => {
|
||||
const encodeQuestions = (questions: DecisionQuestions) =>
|
||||
Effect.forEach(Object.entries(questions), ([name, question]): Effect.Effect<readonly [string, unknown], AIError> => {
|
||||
if (question.type === "predicate")
|
||||
return Effect.succeed([question.name, { type: "noul", instructions: question.instructions }] as const)
|
||||
return Effect.succeed([name, { type: "noul", instructions: question.instructions }] as const)
|
||||
if (question.type === "score")
|
||||
return Effect.succeed([
|
||||
question.name,
|
||||
name,
|
||||
{
|
||||
type: "score",
|
||||
instructions: question.instructions,
|
||||
@@ -90,11 +91,11 @@ const encodeQuestions = (questions: ReadonlyArray<DecisionQuestion>) =>
|
||||
if (new Set(keys).size !== keys.length)
|
||||
return Effect.fail(
|
||||
ProviderShared.invalidRequest(
|
||||
`System One choice options are strings; "${question.name}" has a boolean and a string with the same text`,
|
||||
`System One choice options are strings; "${name}" has a boolean and a string with the same text`,
|
||||
),
|
||||
)
|
||||
return Effect.succeed([
|
||||
question.name,
|
||||
name,
|
||||
{
|
||||
type: "choice",
|
||||
instructions: question.instructions,
|
||||
@@ -197,19 +198,21 @@ const stateItem = (
|
||||
)
|
||||
}
|
||||
|
||||
/** Map System One's answers, keyed by question name, back to the request's question order. */
|
||||
/** Map System One's answers, keyed by question name, to the request's question order. */
|
||||
export const toResponse = (
|
||||
data: Response,
|
||||
request: DecisionRequestFor,
|
||||
providerMetadataKey: string,
|
||||
invalid: (message: string) => AIError,
|
||||
) => {
|
||||
const names = request.questions.map((question) => question.name)
|
||||
const names = Object.keys(request.questions)
|
||||
if (Object.keys(data.answers).length !== names.length || !names.every((name) => Object.hasOwn(data.answers, name)))
|
||||
return Effect.fail(invalid("System One answers do not match the requested questions"))
|
||||
return Effect.succeed(
|
||||
new DecisionResponse({
|
||||
answers: request.questions.map((question) => toAnswer(question, data.answers[question.name])),
|
||||
answers: Object.fromEntries(
|
||||
Object.entries(request.questions).map(([name, question]) => [name, toAnswer(question, data.answers[name])]),
|
||||
),
|
||||
usage:
|
||||
data.usage === undefined
|
||||
? undefined
|
||||
@@ -233,7 +236,7 @@ export const toResponse = (
|
||||
}
|
||||
|
||||
const toAnswer = (question: DecisionQuestion, answer: typeof Answer.Type): DecisionAnswer => {
|
||||
if (answer.type === "noul") return { name: question.name, type: "predicate", probability: answer.noul }
|
||||
if (answer.type === "noul") return { type: "predicate", probability: answer.noul }
|
||||
if (answer.type === "choice") {
|
||||
const choices = question.type === "choice" ? question.choices : []
|
||||
const value = (key: string) => choices.find((choice) => String(choice.value) === key)?.value ?? key
|
||||
@@ -242,7 +245,6 @@ const toAnswer = (question: DecisionQuestion, answer: typeof Answer.Type): Decis
|
||||
return index === -1 ? choices.length : index
|
||||
}
|
||||
return {
|
||||
name: question.name,
|
||||
type: "choice",
|
||||
choice: value(answer.choice),
|
||||
probabilities: Object.entries(answer.probabilities)
|
||||
@@ -253,7 +255,6 @@ const toAnswer = (question: DecisionQuestion, answer: typeof Answer.Type): Decis
|
||||
}
|
||||
const levels = question.type === "score" ? question.levels : []
|
||||
return {
|
||||
name: question.name,
|
||||
type: "score",
|
||||
score: answer.score,
|
||||
probabilities: Object.entries(answer.probabilities)
|
||||
|
||||
@@ -19,21 +19,19 @@ const offline = client(() => Effect.die("an invalid decision request reached the
|
||||
|
||||
const png = Media.bytes(new Uint8Array([1, 2, 3]), "image/png")
|
||||
|
||||
const questions = [
|
||||
{
|
||||
name: "department",
|
||||
const questions = {
|
||||
department: {
|
||||
type: "choice",
|
||||
instructions: "Which team should handle this?",
|
||||
choices: [{ value: "billing", description: "Payments and refunds" }, { value: "technical" }],
|
||||
},
|
||||
{
|
||||
name: "urgency",
|
||||
urgency: {
|
||||
type: "score",
|
||||
instructions: "How urgent is this?",
|
||||
levels: [{ label: "Can wait" }, { label: "Needs attention", description: "Same day" }, { label: "Blocking" }],
|
||||
},
|
||||
{ name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" },
|
||||
] as const
|
||||
refund: { type: "predicate", instructions: "Is the customer asking for a refund?" },
|
||||
} as const
|
||||
|
||||
const systemOneAnswers = {
|
||||
department: { type: "choice", choice: "billing", probabilities: { technical: 0.1, billing: 0.9 }, confidence: 0.8 },
|
||||
@@ -47,9 +45,8 @@ const systemOneAnswers = {
|
||||
refund: { type: "noul", noul: 0.97 },
|
||||
}
|
||||
|
||||
const expectedAnswers = [
|
||||
{
|
||||
name: "department",
|
||||
const expectedAnswers = {
|
||||
department: {
|
||||
type: "choice",
|
||||
choice: "billing",
|
||||
probabilities: [
|
||||
@@ -58,8 +55,7 @@ const expectedAnswers = [
|
||||
],
|
||||
confidence: 0.8,
|
||||
},
|
||||
{
|
||||
name: "urgency",
|
||||
urgency: {
|
||||
type: "score",
|
||||
score: 1.2,
|
||||
probabilities: [
|
||||
@@ -69,8 +65,8 @@ const expectedAnswers = [
|
||||
],
|
||||
confidence: 0.6,
|
||||
},
|
||||
{ name: "refund", type: "predicate", probability: 0.97 },
|
||||
]
|
||||
refund: { type: "predicate", probability: 0.97 },
|
||||
}
|
||||
|
||||
describe("Decision", () => {
|
||||
it.effect("decides through System One with TypeSafe image assets", () =>
|
||||
@@ -144,19 +140,18 @@ describe("Decision", () => {
|
||||
.generate({
|
||||
model: OpenCode.configure({ apiKey: "opencode-key", baseURL: "https://opencode.test/v1" }).decision("jev-1.13"),
|
||||
input: { ticket: "hello" },
|
||||
questions: [
|
||||
{
|
||||
name: "greeting",
|
||||
questions: {
|
||||
greeting: {
|
||||
type: "choice",
|
||||
instructions: "Is this a greeting?",
|
||||
choices: [{ value: true }, { value: false }],
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
.pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => {
|
||||
const answer = response.answers[0]
|
||||
const answer = response.answers.greeting
|
||||
if (answer.type !== "choice") throw new Error("expected a choice answer")
|
||||
expect(answer.choice).toBe(true)
|
||||
expect(answer.probabilities).toEqual([
|
||||
@@ -212,9 +207,7 @@ describe("Decision", () => {
|
||||
.generate({
|
||||
model: TypeSafeAI.decision("jev-latest"),
|
||||
input: { ticket: "hello" },
|
||||
questions: [
|
||||
{ name: "ok", type: "choice", instructions: "OK?", choices: [{ value: true }, { value: "true" }] },
|
||||
],
|
||||
questions: { ok: { type: "choice", instructions: "OK?", choices: [{ value: true }, { value: "true" }] } },
|
||||
})
|
||||
.pipe(Effect.flip)
|
||||
expect(error.reason._tag).toBe("InvalidRequest")
|
||||
@@ -233,13 +226,13 @@ describe("Decision", () => {
|
||||
{ type: "media", media: Media.url("https://assets.test/before.png", { mediaType: "image/png" }) },
|
||||
{ type: "media", media: Media.base64("AAAA", "audio/wav") },
|
||||
],
|
||||
questions: [{ name: "red", type: "predicate", instructions: "Is the image mostly red?" }],
|
||||
questions: { red: { type: "predicate", instructions: "Is the image mostly red?" } },
|
||||
providerOptions: { user: "user-1", session_id: "session-1" },
|
||||
})
|
||||
.pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => {
|
||||
expect(response.answers).toEqual([{ name: "red", type: "predicate", probability: 0.98 }])
|
||||
expect(response.answers).toEqual({ red: { type: "predicate", probability: 0.98 } })
|
||||
expect(response.providerMetadata?.openrouter).toEqual({
|
||||
model: "cloudflare/clef-omni-20261001",
|
||||
responseId: "gen-1",
|
||||
@@ -289,11 +282,11 @@ describe("Decision", () => {
|
||||
{ type: "media", media: png },
|
||||
{ type: "json", value: { source: "upload" } },
|
||||
],
|
||||
questions: [{ name: "red", type: "predicate", instructions: "Is the photo mostly red?" }],
|
||||
questions: { red: { type: "predicate", instructions: "Is the photo mostly red?" } },
|
||||
})
|
||||
.pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => expect(response.answers).toEqual([{ name: "red", type: "predicate", probability: 0.99 }])),
|
||||
Effect.sync(() => expect(response.answers).toEqual({ red: { type: "predicate", probability: 0.99 } })),
|
||||
),
|
||||
Effect.provide(
|
||||
client((input) => {
|
||||
@@ -342,6 +335,7 @@ describe("Decision", () => {
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => {
|
||||
expect(response.answers).toEqual(expectedAnswers)
|
||||
expect(Object.keys(response.answers)).toEqual(["department", "urgency", "refund"])
|
||||
expect(response.usage?.inputTokens).toBe(40)
|
||||
expect(response.providerMetadata).toEqual({
|
||||
openai: { model: "gpt-6-luna", usage: { input_tokens: 40, output_tokens: 0, total_tokens: 40 } },
|
||||
@@ -388,7 +382,7 @@ describe("Decision", () => {
|
||||
{ name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" },
|
||||
],
|
||||
})
|
||||
// Answers come back out of order; the protocol restores question order by name.
|
||||
// Answers come back out of order; the protocol keys them in question order.
|
||||
return Effect.succeed(
|
||||
json(input, {
|
||||
model: "gpt-6-luna",
|
||||
@@ -429,20 +423,20 @@ describe("Decision", () => {
|
||||
.generate({
|
||||
model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"),
|
||||
input: [{ type: "text", text: "hello" }],
|
||||
questions: [
|
||||
{ name: "greeting", type: "predicate", instructions: "Is this a greeting?" },
|
||||
{ name: "unsafe", type: "predicate", instructions: "Something the model declines" },
|
||||
{ name: "explained", type: "predicate", instructions: "Something the model declines with a reason" },
|
||||
],
|
||||
questions: {
|
||||
greeting: { type: "predicate", instructions: "Is this a greeting?" },
|
||||
unsafe: { type: "predicate", instructions: "Something the model declines" },
|
||||
explained: { type: "predicate", instructions: "Something the model declines with a reason" },
|
||||
},
|
||||
})
|
||||
.pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() =>
|
||||
expect(response.answers).toEqual([
|
||||
{ name: "greeting", type: "predicate", probability: 0.99 },
|
||||
{ name: "unsafe", type: "error", reason: "refusal" },
|
||||
{ name: "explained", type: "error", reason: "refusal", message: "I can't help with that." },
|
||||
]),
|
||||
expect(response.answers).toEqual({
|
||||
greeting: { type: "predicate", probability: 0.99 },
|
||||
unsafe: { type: "error", reason: "refusal" },
|
||||
explained: { type: "error", reason: "refusal", message: "I can't help with that." },
|
||||
}),
|
||||
),
|
||||
),
|
||||
Effect.provide(
|
||||
@@ -468,7 +462,7 @@ describe("Decision", () => {
|
||||
.generate({
|
||||
model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"),
|
||||
input: [{ type: "media", media: Media.base64("AAAA", "audio/wav") }],
|
||||
questions: [{ name: "speech", type: "predicate", instructions: "Is someone speaking?" }],
|
||||
questions: { speech: { type: "predicate", instructions: "Is someone speaking?" } },
|
||||
})
|
||||
.pipe(Effect.flip)
|
||||
expect(error.reason._tag).toBe("UnsupportedOperation")
|
||||
@@ -483,12 +477,12 @@ describe("Decision", () => {
|
||||
"typesafe-ai/jev",
|
||||
),
|
||||
input: [{ type: "text", text: "refund" }],
|
||||
questions: [{ name: "refund", type: "predicate", instructions: "Is a refund requested?" }],
|
||||
questions: { refund: { type: "predicate", instructions: "Is a refund requested?" } },
|
||||
})
|
||||
.pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => {
|
||||
expect(response.answers).toEqual([{ name: "refund", type: "predicate", probability: 0.98 }])
|
||||
expect(response.answers).toEqual({ refund: { type: "predicate", probability: 0.98 } })
|
||||
expect(response.providerMetadata).toEqual({
|
||||
"vercel-ai-gateway": { model: "typesafe-ai/jev", usage: { input_tokens: 10, output_tokens: 0 } },
|
||||
})
|
||||
@@ -526,12 +520,12 @@ describe("Decision", () => {
|
||||
{ type: "media", media: png },
|
||||
{ type: "media", media: Media.base64("AAAA", "audio/wav") },
|
||||
],
|
||||
questions: [{ name: "red", type: "predicate", instructions: "Is the image mostly red?" }],
|
||||
questions: { red: { type: "predicate", instructions: "Is the image mostly red?" } },
|
||||
})
|
||||
.pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => {
|
||||
expect(response.answers).toEqual([{ name: "red", type: "predicate", probability: 0.9 }])
|
||||
expect(response.answers).toEqual({ red: { type: "predicate", probability: 0.9 } })
|
||||
expect(response.providerMetadata).toEqual({ "cloudflare-workers-ai": { model: "clef-omni" } })
|
||||
}),
|
||||
),
|
||||
@@ -562,7 +556,7 @@ describe("Decision", () => {
|
||||
"@cf/cloudflare/clef",
|
||||
),
|
||||
input: [{ type: "text", text: "refund" }],
|
||||
questions: [{ name: "refund", type: "predicate", instructions: "Is a refund requested?" }],
|
||||
questions: { refund: { type: "predicate", instructions: "Is a refund requested?" } },
|
||||
})
|
||||
.pipe(
|
||||
Effect.flip,
|
||||
@@ -602,7 +596,7 @@ describe("Decision", () => {
|
||||
.generate({
|
||||
model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"),
|
||||
input: [{ type: "text", text: "refund" }],
|
||||
questions: [{ name: "tone", type: "choice", instructions: "Tone?", choices: [{ value: "calm" }] }],
|
||||
questions: { tone: { type: "choice", instructions: "Tone?", choices: [{ value: "calm" }] } },
|
||||
})
|
||||
.pipe(
|
||||
Effect.flip,
|
||||
@@ -633,10 +627,10 @@ describe("Decision", () => {
|
||||
.generate({
|
||||
model: TypeSafeAI.configure({ apiKey: "test" }).decision("jev-latest"),
|
||||
input: [{ type: "text", text: "refund" }],
|
||||
questions: [
|
||||
{ name: "refund", type: "predicate", instructions: "Refund?" },
|
||||
{ name: "angry", type: "predicate", instructions: "Angry?" },
|
||||
],
|
||||
questions: {
|
||||
refund: { type: "predicate", instructions: "Refund?" },
|
||||
angry: { type: "predicate", instructions: "Angry?" },
|
||||
},
|
||||
})
|
||||
.pipe(
|
||||
Effect.flip,
|
||||
@@ -649,19 +643,38 @@ describe("Decision", () => {
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("rejects duplicate question names before network I/O", () =>
|
||||
it.effect("rejects an empty question record before network I/O", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* ai.decision
|
||||
.generate({
|
||||
model: TypeSafeAI.decision("jev-latest"),
|
||||
input: [{ type: "text", text: "hello" }],
|
||||
questions: [
|
||||
{ name: "same", type: "predicate", instructions: "One?" },
|
||||
{ name: "same", type: "predicate", instructions: "Two?" },
|
||||
],
|
||||
})
|
||||
.generate({ model: TypeSafeAI.decision("jev-latest"), input: [{ type: "text", text: "hello" }], questions: {} })
|
||||
.pipe(Effect.flip)
|
||||
expect(error.reason._tag).toBe("InvalidRequest")
|
||||
}).pipe(Effect.provide(offline)),
|
||||
)
|
||||
|
||||
it.effect("fails the call when OpenAI Decisions answers a question twice", () =>
|
||||
ai.decision
|
||||
.generate({
|
||||
model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"),
|
||||
input: [{ type: "text", text: "refund" }],
|
||||
questions: { refund: { type: "predicate", instructions: "Refund?" } },
|
||||
})
|
||||
.pipe(
|
||||
Effect.flip,
|
||||
Effect.tap((error) => Effect.sync(() => expect(error.reason._tag).toBe("InvalidProviderOutput"))),
|
||||
Effect.provide(
|
||||
client((input) =>
|
||||
Effect.succeed(
|
||||
json(input, {
|
||||
model: "gpt-6-luna",
|
||||
answers: [
|
||||
{ type: "predicate", name: "refund", probability: 0.9 },
|
||||
{ type: "predicate", name: "refund", probability: 0.1 },
|
||||
],
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
@@ -11,18 +11,18 @@ type Assert<T extends true> = T
|
||||
const request = ai.decision.request({
|
||||
model: TypeSafeAI.configure({ apiKey: "test" }).decision("jev-latest"),
|
||||
input: { ticket: "refund" },
|
||||
questions: [
|
||||
{ name: "topic", type: "choice", instructions: "Which team?", choices: [{ value: "billing" }, { value: true }] },
|
||||
{ name: "severity", type: "score", instructions: "How severe?", levels: [{ label: "Low" }, { label: "High" }] },
|
||||
{ name: "refund", type: "predicate", instructions: "Refund?" },
|
||||
],
|
||||
questions: {
|
||||
topic: { type: "choice", instructions: "Which team?", choices: [{ value: "billing" }, { value: true }] },
|
||||
severity: { type: "score", instructions: "How severe?", levels: [{ label: "Low" }, { label: "High" }] },
|
||||
refund: { type: "predicate", instructions: "Refund?" },
|
||||
},
|
||||
})
|
||||
|
||||
const result = DecisionClient.generate(request)
|
||||
type Answers = Success<typeof result>["answers"]
|
||||
type Topic = Exclude<Answers[0], { readonly type: "error" }>
|
||||
type Topic = Exclude<Answers["topic"], { readonly type: "error" }>
|
||||
type Choice = Assert<Equal<Topic["choice"], "billing" | true>>
|
||||
type Name = Assert<Equal<Answers[2]["name"], "refund">>
|
||||
type Name = Assert<Equal<keyof Answers, "topic" | "severity" | "refund">>
|
||||
type ClientRequirements = Assert<Equal<Requirements<typeof result>, Service>>
|
||||
void (true satisfies Choice)
|
||||
void (true satisfies Name)
|
||||
@@ -32,21 +32,21 @@ Effect.gen(function* () {
|
||||
const response = yield* ai.decision.generate({
|
||||
model: OpenCode.decision("jev-1.13"),
|
||||
input: [{ type: "text", text: "hello" }],
|
||||
questions: [{ name: "greeting", type: "predicate", instructions: "Greeting?" }],
|
||||
questions: { greeting: { type: "predicate", instructions: "Greeting?" } },
|
||||
})
|
||||
const greeting = response.answers[0]
|
||||
const greeting = response.answers.greeting
|
||||
if (greeting.type === "error") return
|
||||
greeting.probability satisfies number
|
||||
// @ts-expect-error Predicate answers do not contain a selected choice.
|
||||
greeting.choice
|
||||
// @ts-expect-error There is one answer per question.
|
||||
response.answers[1]
|
||||
response.answers.farewell
|
||||
})
|
||||
;({
|
||||
model: OpenCode.decision("jev-1.13"),
|
||||
// @ts-expect-error Input is a JSON object or a list of parts, not a plain string.
|
||||
input: "hello",
|
||||
questions: [{ name: "ok", type: "predicate", instructions: "OK?" }],
|
||||
questions: { ok: { type: "predicate", instructions: "OK?" } },
|
||||
}) satisfies DecisionRequestInput
|
||||
|
||||
declare const route: DecisionModelRoute<{ readonly temperature?: number }>
|
||||
@@ -54,13 +54,13 @@ const custom = DecisionModel.make({ id: "custom", provider: "custom", route })
|
||||
ai.decision.generate({
|
||||
model: custom,
|
||||
input: { text: "hello" },
|
||||
questions: [{ name: "ok", type: "predicate", instructions: "OK?" }],
|
||||
questions: { ok: { type: "predicate", instructions: "OK?" } },
|
||||
providerOptions: { temperature: 0.5 },
|
||||
})
|
||||
;({
|
||||
model: custom,
|
||||
input: { text: "hello" },
|
||||
questions: [{ name: "ok", type: "predicate", instructions: "OK?" }],
|
||||
questions: { ok: { type: "predicate", instructions: "OK?" } },
|
||||
// @ts-expect-error Selected decision models retain their provider option types.
|
||||
providerOptions: { temperature: "high" },
|
||||
}) satisfies DecisionRequestInput<typeof custom>
|
||||
@@ -68,14 +68,14 @@ ai.decision.generate({
|
||||
ai.decision.generate({
|
||||
model: OpenAI.decision("gpt-6-luna"),
|
||||
input: { text: "hello" },
|
||||
questions: [{ name: "ok", type: "predicate", instructions: "OK?" }],
|
||||
questions: { ok: { type: "predicate", instructions: "OK?" } },
|
||||
providerOptions: { promptCacheKey: "support-v1", safetyIdentifier: "user-1" },
|
||||
})
|
||||
const openai = OpenAI.decision("gpt-6-luna")
|
||||
;({
|
||||
model: openai,
|
||||
input: { text: "hello" },
|
||||
questions: [{ name: "ok", type: "predicate", instructions: "OK?" }],
|
||||
questions: { ok: { type: "predicate", instructions: "OK?" } },
|
||||
// @ts-expect-error OpenAI safety identifiers are strings.
|
||||
providerOptions: { safetyIdentifier: 1 },
|
||||
}) satisfies DecisionRequestInput<typeof openai>
|
||||
@@ -83,14 +83,14 @@ const openai = OpenAI.decision("gpt-6-luna")
|
||||
ai.decision.generate({
|
||||
model: OpenRouter.decision("typesafe/jev-1.13"),
|
||||
input: { text: "hello" },
|
||||
questions: [{ name: "ok", type: "predicate", instructions: "OK?" }],
|
||||
questions: { ok: { type: "predicate", instructions: "OK?" } },
|
||||
providerOptions: { provider: { zdr: true }, session_id: "session-1", user: "user-1" },
|
||||
})
|
||||
const openrouter = OpenRouter.decision("typesafe/jev-1.13")
|
||||
;({
|
||||
model: openrouter,
|
||||
input: { text: "hello" },
|
||||
questions: [{ name: "ok", type: "predicate", instructions: "OK?" }],
|
||||
questions: { ok: { type: "predicate", instructions: "OK?" } },
|
||||
// @ts-expect-error OpenRouter session IDs are strings.
|
||||
providerOptions: { session_id: 1 },
|
||||
}) satisfies DecisionRequestInput<typeof openrouter>
|
||||
@@ -98,14 +98,14 @@ const openrouter = OpenRouter.decision("typesafe/jev-1.13")
|
||||
ai.decision.generate({
|
||||
model: Baseten.decision("decision-models/baseten-elephant-9b-latest"),
|
||||
input: { text: "hello" },
|
||||
questions: [{ name: "ok", type: "predicate", instructions: "OK?" }],
|
||||
questions: { ok: { type: "predicate", instructions: "OK?" } },
|
||||
providerOptions: { max_len: 512 },
|
||||
})
|
||||
const baseten = Baseten.decision("decision-models/baseten-elephant-9b-latest")
|
||||
;({
|
||||
model: baseten,
|
||||
input: { text: "hello" },
|
||||
questions: [{ name: "ok", type: "predicate", instructions: "OK?" }],
|
||||
questions: { ok: { type: "predicate", instructions: "OK?" } },
|
||||
// @ts-expect-error Baseten max_len is a number.
|
||||
providerOptions: { max_len: "512" },
|
||||
}) satisfies DecisionRequestInput<typeof baseten>
|
||||
|
||||
@@ -13,9 +13,8 @@ import {
|
||||
} from "../../src/providers.js"
|
||||
import { recordedTests } from "../recorded-test.js"
|
||||
|
||||
const questions = [
|
||||
{
|
||||
name: "department",
|
||||
const questions = {
|
||||
department: {
|
||||
type: "choice",
|
||||
instructions: "Which team should handle this support request?",
|
||||
choices: [
|
||||
@@ -24,8 +23,7 @@ const questions = [
|
||||
{ value: "sales", description: "Pricing, upgrades, or new accounts" },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "urgency",
|
||||
urgency: {
|
||||
type: "score",
|
||||
instructions: "How urgent is this support request?",
|
||||
levels: [
|
||||
@@ -34,8 +32,8 @@ const questions = [
|
||||
{ label: "Actively blocking revenue" },
|
||||
],
|
||||
},
|
||||
{ name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" },
|
||||
] as const
|
||||
refund: { type: "predicate", instructions: "Is the customer asking for a refund?" },
|
||||
} as const
|
||||
|
||||
const ticket = "I was charged twice for the same invoice. Please refund the duplicate payment today."
|
||||
|
||||
@@ -175,13 +173,17 @@ const assertDecision = <Options extends DecisionOptions>(
|
||||
) =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* ai.decision.generate({ model, input, questions })
|
||||
const [department, urgency, refund] = response.answers
|
||||
const department = response.answers.department
|
||||
const urgency = response.answers.urgency
|
||||
const refund = response.answers.refund
|
||||
if (department.type === "error" || urgency.type === "error" || refund.type === "error")
|
||||
throw new Error("expected every question to be answered")
|
||||
expect(department.choice).toBe("billing")
|
||||
expect(department.probabilities.find((item) => item.value === "billing")?.probability).toBeGreaterThan(0.5)
|
||||
expect(department.confidence).toBeGreaterThan(0)
|
||||
expect(urgency.probabilities.map((item) => item.label)).toEqual(questions[1].levels.map((level) => level.label))
|
||||
expect(urgency.probabilities.map((item) => item.label)).toEqual(
|
||||
questions.urgency.levels.map((level) => level.label),
|
||||
)
|
||||
expect(urgency.score).toBeGreaterThanOrEqual(0)
|
||||
expect(urgency.score).toBeLessThanOrEqual(2)
|
||||
expect(urgency.confidence).toBeGreaterThan(0)
|
||||
|
||||
@@ -26,7 +26,7 @@ const sentDecision = (model: DecisionModel) =>
|
||||
.generate({
|
||||
model,
|
||||
input: [{ type: "text", text: "The sky is red." }],
|
||||
questions: [{ name: "red", type: "predicate", instructions: "Is the sky red?" }],
|
||||
questions: { red: { type: "predicate", instructions: "Is the sky red?" } },
|
||||
})
|
||||
.pipe(Effect.exit, Effect.provide(DecisionClient.layer.pipe(Layer.provide(record(calls)))))
|
||||
return calls
|
||||
|
||||
Reference in New Issue
Block a user