From 06f2715267b76d96583e8686833caa6a8c8c789e Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Sat, 31 Oct 2026 17:22:43 -0500 Subject: [PATCH] refactor(ai): decision questions and answers keyed by name --- packages/ai/README.md | 22 +-- packages/ai/src/decision-client.ts | 11 +- packages/ai/src/decision.ts | 32 ++--- packages/ai/src/protocols/openai-decisions.ts | 41 +++--- packages/ai/src/protocols/system-one.ts | 25 ++-- packages/ai/test/decision.test.ts | 127 ++++++++++-------- packages/ai/test/decision.types.ts | 38 +++--- .../test/provider/decision.recorded.test.ts | 20 +-- .../test/provider/media-entrypoints.test.ts | 2 +- 9 files changed, 167 insertions(+), 151 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index f45426dfe5d..eba5dfa8725 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -266,30 +266,30 @@ const program = ai.decision.generate({ { type: "text", text: "I was charged twice. Please refund the duplicate payment." }, { type: "media", media: Media.url("https://example.com/receipt.png", { mediaType: "image/png" }) }, ], - questions: [ - { name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" }, - { - name: "department", + questions: { + refund: { type: "predicate", instructions: "Is the customer asking for a refund?" }, + department: { type: "choice", instructions: "Which team should handle this?", choices: [{ value: "billing", description: "Payments and refunds" }, { value: "technical" }], }, - { - name: "urgency", + urgency: { type: "score", instructions: "How urgent is this?", levels: [{ label: "Can wait" }, { label: "Needs prompt attention" }, { label: "Blocking revenue" }], }, - ], + }, }) const response = await Effect.runPromise(program.pipe(Effect.provide(DecisionClient.fetchLayer))) -const [refund, department] = response.answers +const department = response.answers.department +if (department.type === "choice") department.choice // "billing" | "technical" ``` -`input` is a JSON object or an ordered list of `text`, `json` and `media` parts. Answers come back in question order. -A question the host declines is `{ name, type: "error", reason, message? }`, with `reason` such as `"refusal"`; the -other answers still come back. +`input` is a JSON object or an ordered list of `text`, `json` and `media` parts. Questions are keyed by name, and +answers come back under the same names; key order is the order sent to the host. +A question the host declines is `{ type: "error", reason, message? }`, with `reason` such as `"refusal"`; the other +answers still come back. `response.usage` has `inputTokens`, `outputTokens`, `totalTokens` and the host's reported `cost` when it sends one; the host's raw usage stays in `providerMetadata`. Media goes wherever the host reads it, and a kind the host has no place for fails the call. Host-specific fields go in diff --git a/packages/ai/src/decision-client.ts b/packages/ai/src/decision-client.ts index f3d32e2a412..d1ac607f91c 100644 --- a/packages/ai/src/decision-client.ts +++ b/packages/ai/src/decision-client.ts @@ -2,7 +2,6 @@ import { Context, Effect, Layer } from "effect" import type { DecisionAnswer, DecisionOptions, - DecisionQuestion, DecisionQuestions, DecisionRequestFor, DecisionResponseFor, @@ -49,11 +48,11 @@ export const layer: Layer.Layer = Layer export const fetchLayer = layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer)) /** Host bugs (a missing or extra answer, an option that wasn't offered, a score off the scale) fail the whole call. */ -const answersMatch = (questions: ReadonlyArray, answers: ReadonlyArray) => - questions.length === answers.length && - questions.every((question, index) => { - const answer = answers[index] - if (answer.name !== question.name) return false +const answersMatch = (questions: DecisionQuestions, answers: Readonly>) => + Object.keys(questions).length === Object.keys(answers).length && + Object.entries(questions).every(([name, question]) => { + if (!Object.hasOwn(answers, name)) return false + const answer = answers[name] if (answer.type === "error") return true if (question.type === "predicate") return answer.type === "predicate" if (question.type === "choice") { diff --git a/packages/ai/src/decision.ts b/packages/ai/src/decision.ts index 6a0daa9ff4b..6651b6551b6 100644 --- a/packages/ai/src/decision.ts +++ b/packages/ai/src/decision.ts @@ -40,14 +40,12 @@ export const ChoiceValue = Schema.Union([Schema.String, Schema.Boolean]) export type ChoiceValue = Schema.Schema.Type export const PredicateQuestion = Schema.Struct({ - name: Schema.String, type: Schema.Literal("predicate"), instructions: Schema.String, }) export type PredicateQuestion = Schema.Schema.Type export const ChoiceQuestion = Schema.Struct({ - name: Schema.String, type: Schema.Literal("choice"), instructions: Schema.String, choices: Schema.Array(Schema.Struct({ value: ChoiceValue, description: Schema.optional(Schema.String) })), @@ -55,7 +53,6 @@ export const ChoiceQuestion = Schema.Struct({ export type ChoiceQuestion = Schema.Schema.Type export const ScoreQuestion = Schema.Struct({ - name: Schema.String, type: Schema.Literal("score"), instructions: Schema.String, levels: Schema.Array(Schema.Struct({ label: Schema.String, description: Schema.optional(Schema.String) })), @@ -66,14 +63,12 @@ export const DecisionQuestion = Schema.Union([PredicateQuestion, ChoiceQuestion, Schema.toTaggedUnion("type"), ) export type DecisionQuestion = Schema.Schema.Type -export type DecisionQuestions = ReadonlyArray +/** Questions keyed by name; key order is the order sent to the host. */ +export type DecisionQuestions = Readonly> // Only what makes a request meaningless is checked here; counts and sizes are the host's to enforce. -const DecisionQuestionList = Schema.Array(DecisionQuestion).pipe( - Schema.refine((x): x is typeof x => x.length > 0, { message: "Decision questions must not be empty" }), - Schema.refine((x): x is typeof x => new Set(x.map((question) => question.name)).size === x.length, { - message: "Decision question names must be unique", - }), +const DecisionQuestionRecord = Schema.Record(Schema.String, DecisionQuestion).pipe( + Schema.refine((x): x is typeof x => Object.keys(x).length > 0, { message: "Decision questions must not be empty" }), ) // --------------------------------------------------------------------------- @@ -83,14 +78,12 @@ const DecisionQuestionList = Schema.Array(DecisionQuestion).pipe( const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 })) export const PredicateAnswer = Schema.Struct({ - name: Schema.String, type: Schema.Literal("predicate"), probability: Probability, }) export type PredicateAnswer = Schema.Schema.Type export const ChoiceAnswer = Schema.Struct({ - name: Schema.String, type: Schema.Literal("choice"), choice: ChoiceValue, probabilities: Schema.Array(Schema.Struct({ value: ChoiceValue, probability: Probability })), @@ -100,7 +93,6 @@ export type ChoiceAnswer = Schema.Schema.Type /** `score` is the probability-weighted level index; `probabilities` has one entry per level. */ export const ScoreAnswer = Schema.Struct({ - name: Schema.String, type: Schema.Literal("score"), score: Schema.Number, probabilities: Schema.Array(Schema.Struct({ value: Schema.Int, label: Schema.String, probability: Probability })), @@ -114,7 +106,6 @@ export const ErrorReason: Schema.Codec = Schema.Union([Schema.Liter /** This one question has no answer; the other answers still came back. */ export const ErrorAnswer = Schema.Struct({ - name: Schema.String, type: Schema.Literal("error"), reason: ErrorReason, message: Schema.optional(Schema.String), @@ -132,10 +123,9 @@ type ChoiceValueOf = Question extends { readonly choices: ReadonlyArra export type AnswerFor = | (Question extends { readonly type: "predicate" } - ? PredicateAnswer & { readonly name: Question["name"] } + ? PredicateAnswer : Question extends { readonly type: "choice" } ? { - readonly name: Question["name"] readonly type: "choice" readonly choice: ChoiceValueOf readonly probabilities: ReadonlyArray<{ @@ -144,12 +134,12 @@ export type AnswerFor = }> readonly confidence: number } - : ScoreAnswer & { readonly name: Question["name"] }) - | (ErrorAnswer & { readonly name: Question["name"] }) + : ScoreAnswer) + | ErrorAnswer -/** One answer per question, in question order. */ +/** One answer per question, keyed by question name. */ export type AnswersFor = { - readonly [Index in keyof Questions]: AnswerFor + readonly [Name in keyof Questions]: AnswerFor } // --------------------------------------------------------------------------- @@ -241,7 +231,7 @@ export type DecisionModelOptions = Model extends DecisionModel("Decision.Request")({ model: DecisionModelSchema, input: Schema.Array(DecisionPart), - questions: DecisionQuestionList, + questions: DecisionQuestionRecord, providerOptions: Schema.optional(ProviderOptions), http: Schema.optional(HttpOptions), }) { @@ -278,7 +268,7 @@ export class DecisionUsage extends Schema.Class("Decision.Usage") }) {} export class DecisionResponse extends Schema.Class("Decision.Response")({ - answers: Schema.Array(DecisionAnswer), + answers: Schema.Record(Schema.String, DecisionAnswer), usage: Schema.optional(DecisionUsage), providerMetadata: Schema.optional(ProviderMetadata), }) {} diff --git a/packages/ai/src/protocols/openai-decisions.ts b/packages/ai/src/protocols/openai-decisions.ts index 5be7c5dc9ca..acad28c18ab 100644 --- a/packages/ai/src/protocols/openai-decisions.ts +++ b/packages/ai/src/protocols/openai-decisions.ts @@ -1,5 +1,11 @@ import { Effect, Schema } from "effect" -import { DecisionResponse, DecisionUsage, type DecisionAnswer, type DecisionPart, type DecisionRequestFor } from "../decision.js" +import { + DecisionResponse, + DecisionUsage, + type DecisionAnswer, + type DecisionPart, + type DecisionRequestFor, +} from "../decision.js" import { DecisionRoute } from "../route/decision.js" import { Endpoint } from "../route/endpoint.js" import { MediaProtocol } from "../route/media-protocol.js" @@ -66,7 +72,7 @@ const fromRequest = Effect.fn("OpenAIDecisions.fromRequest")(function* (request: ), model: request.model.id, input: [{ role: "user", content: yield* Effect.forEach(request.input, (part) => inputPart(request, part)) }], - questions: request.questions, + questions: Object.entries(request.questions).map(([name, question]) => ({ name, ...question })), } }) @@ -100,6 +106,13 @@ const inputPart = ( const isObject = (value: unknown) => typeof value === "object" && value !== null && !Array.isArray(value) +const toAnswer = (answer: typeof Answer.Type): DecisionAnswer => { + if (answer.type === "refusal") + return { type: "error", reason: "refusal", ...(answer.message === undefined ? {} : { message: answer.message }) } + const { name: _, ...rest } = answer + return rest +} + const decodeDocument = MediaProtocol.identity({ id: ID, name: NAME, provider: "openai" }).decodeJson(Response) export const protocol = DecisionRoute.protocol({ @@ -109,22 +122,20 @@ export const protocol = DecisionRoute.protocol({ response: { decode: Effect.fn("OpenAIDecisions.decodeResponse")(function* (response, context) { const output = yield* decodeDocument(response) - const answers = output.value.answers.map( - (answer): DecisionAnswer => - answer.type === "refusal" - ? { - name: answer.name, - type: "error", - reason: "refusal", - ...(answer.message === undefined ? {} : { message: answer.message }), - } - : answer, + // Answers come back as a list in any order; key them by name in question order. + const byName = new Map(output.value.answers.map((answer) => [answer.name, toAnswer(answer)])) + const names = Object.keys(context.request.questions) + if (byName.size !== output.value.answers.length || byName.size !== names.length) + return yield* output.invalid(`${NAME} answers do not match the requested questions`, output.value.answers) + const answers = Object.fromEntries( + names.flatMap((name) => { + const answer = byName.get(name) + return answer === undefined ? [] : [[name, answer] as const] + }), ) - const byName = new Map(answers.map((answer) => [answer.name, answer])) - const ordered = context.request.questions.flatMap((question) => byName.get(question.name) ?? []) const usage = OpenResponses.mapUsage(output.value.usage, context.providerMetadataKey) return new DecisionResponse({ - answers: ordered.length === answers.length ? ordered : answers, + answers, usage: usage === undefined ? undefined diff --git a/packages/ai/src/protocols/system-one.ts b/packages/ai/src/protocols/system-one.ts index 9c81e587d81..d34f5e7fd5c 100644 --- a/packages/ai/src/protocols/system-one.ts +++ b/packages/ai/src/protocols/system-one.ts @@ -6,6 +6,7 @@ import { type DecisionModelCompatibility, type DecisionPart, type DecisionQuestion, + type DecisionQuestions, type DecisionRequestFor, } from "../decision.js" import type { Media } from "../media.js" @@ -70,13 +71,13 @@ export const encodeBody = Effect.fn("SystemOne.encodeBody")(function* ( return { ...mergeJsonRecords(request.providerOptions, request.http?.body), ...placed, questions } }) -const encodeQuestions = (questions: ReadonlyArray) => - Effect.forEach(questions, (question): Effect.Effect => { +const encodeQuestions = (questions: DecisionQuestions) => + Effect.forEach(Object.entries(questions), ([name, question]): Effect.Effect => { if (question.type === "predicate") - return Effect.succeed([question.name, { type: "noul", instructions: question.instructions }] as const) + return Effect.succeed([name, { type: "noul", instructions: question.instructions }] as const) if (question.type === "score") return Effect.succeed([ - question.name, + name, { type: "score", instructions: question.instructions, @@ -90,11 +91,11 @@ const encodeQuestions = (questions: ReadonlyArray) => if (new Set(keys).size !== keys.length) return Effect.fail( ProviderShared.invalidRequest( - `System One choice options are strings; "${question.name}" has a boolean and a string with the same text`, + `System One choice options are strings; "${name}" has a boolean and a string with the same text`, ), ) return Effect.succeed([ - question.name, + name, { type: "choice", instructions: question.instructions, @@ -197,19 +198,21 @@ const stateItem = ( ) } -/** Map System One's answers, keyed by question name, back to the request's question order. */ +/** Map System One's answers, keyed by question name, to the request's question order. */ export const toResponse = ( data: Response, request: DecisionRequestFor, providerMetadataKey: string, invalid: (message: string) => AIError, ) => { - const names = request.questions.map((question) => question.name) + const names = Object.keys(request.questions) if (Object.keys(data.answers).length !== names.length || !names.every((name) => Object.hasOwn(data.answers, name))) return Effect.fail(invalid("System One answers do not match the requested questions")) return Effect.succeed( new DecisionResponse({ - answers: request.questions.map((question) => toAnswer(question, data.answers[question.name])), + answers: Object.fromEntries( + Object.entries(request.questions).map(([name, question]) => [name, toAnswer(question, data.answers[name])]), + ), usage: data.usage === undefined ? undefined @@ -233,7 +236,7 @@ export const toResponse = ( } const toAnswer = (question: DecisionQuestion, answer: typeof Answer.Type): DecisionAnswer => { - if (answer.type === "noul") return { name: question.name, type: "predicate", probability: answer.noul } + if (answer.type === "noul") return { type: "predicate", probability: answer.noul } if (answer.type === "choice") { const choices = question.type === "choice" ? question.choices : [] const value = (key: string) => choices.find((choice) => String(choice.value) === key)?.value ?? key @@ -242,7 +245,6 @@ const toAnswer = (question: DecisionQuestion, answer: typeof Answer.Type): Decis return index === -1 ? choices.length : index } return { - name: question.name, type: "choice", choice: value(answer.choice), probabilities: Object.entries(answer.probabilities) @@ -253,7 +255,6 @@ const toAnswer = (question: DecisionQuestion, answer: typeof Answer.Type): Decis } const levels = question.type === "score" ? question.levels : [] return { - name: question.name, type: "score", score: answer.score, probabilities: Object.entries(answer.probabilities) diff --git a/packages/ai/test/decision.test.ts b/packages/ai/test/decision.test.ts index ab82af5b12e..93fec07409c 100644 --- a/packages/ai/test/decision.test.ts +++ b/packages/ai/test/decision.test.ts @@ -19,21 +19,19 @@ const offline = client(() => Effect.die("an invalid decision request reached the const png = Media.bytes(new Uint8Array([1, 2, 3]), "image/png") -const questions = [ - { - name: "department", +const questions = { + department: { type: "choice", instructions: "Which team should handle this?", choices: [{ value: "billing", description: "Payments and refunds" }, { value: "technical" }], }, - { - name: "urgency", + urgency: { type: "score", instructions: "How urgent is this?", levels: [{ label: "Can wait" }, { label: "Needs attention", description: "Same day" }, { label: "Blocking" }], }, - { name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" }, -] as const + refund: { type: "predicate", instructions: "Is the customer asking for a refund?" }, +} as const const systemOneAnswers = { department: { type: "choice", choice: "billing", probabilities: { technical: 0.1, billing: 0.9 }, confidence: 0.8 }, @@ -47,9 +45,8 @@ const systemOneAnswers = { refund: { type: "noul", noul: 0.97 }, } -const expectedAnswers = [ - { - name: "department", +const expectedAnswers = { + department: { type: "choice", choice: "billing", probabilities: [ @@ -58,8 +55,7 @@ const expectedAnswers = [ ], confidence: 0.8, }, - { - name: "urgency", + urgency: { type: "score", score: 1.2, probabilities: [ @@ -69,8 +65,8 @@ const expectedAnswers = [ ], confidence: 0.6, }, - { name: "refund", type: "predicate", probability: 0.97 }, -] + refund: { type: "predicate", probability: 0.97 }, +} describe("Decision", () => { it.effect("decides through System One with TypeSafe image assets", () => @@ -144,19 +140,18 @@ describe("Decision", () => { .generate({ model: OpenCode.configure({ apiKey: "opencode-key", baseURL: "https://opencode.test/v1" }).decision("jev-1.13"), input: { ticket: "hello" }, - questions: [ - { - name: "greeting", + questions: { + greeting: { type: "choice", instructions: "Is this a greeting?", choices: [{ value: true }, { value: false }], }, - ], + }, }) .pipe( Effect.tap((response) => Effect.sync(() => { - const answer = response.answers[0] + const answer = response.answers.greeting if (answer.type !== "choice") throw new Error("expected a choice answer") expect(answer.choice).toBe(true) expect(answer.probabilities).toEqual([ @@ -212,9 +207,7 @@ describe("Decision", () => { .generate({ model: TypeSafeAI.decision("jev-latest"), input: { ticket: "hello" }, - questions: [ - { name: "ok", type: "choice", instructions: "OK?", choices: [{ value: true }, { value: "true" }] }, - ], + questions: { ok: { type: "choice", instructions: "OK?", choices: [{ value: true }, { value: "true" }] } }, }) .pipe(Effect.flip) expect(error.reason._tag).toBe("InvalidRequest") @@ -233,13 +226,13 @@ describe("Decision", () => { { type: "media", media: Media.url("https://assets.test/before.png", { mediaType: "image/png" }) }, { type: "media", media: Media.base64("AAAA", "audio/wav") }, ], - questions: [{ name: "red", type: "predicate", instructions: "Is the image mostly red?" }], + questions: { red: { type: "predicate", instructions: "Is the image mostly red?" } }, providerOptions: { user: "user-1", session_id: "session-1" }, }) .pipe( Effect.tap((response) => Effect.sync(() => { - expect(response.answers).toEqual([{ name: "red", type: "predicate", probability: 0.98 }]) + expect(response.answers).toEqual({ red: { type: "predicate", probability: 0.98 } }) expect(response.providerMetadata?.openrouter).toEqual({ model: "cloudflare/clef-omni-20261001", responseId: "gen-1", @@ -289,11 +282,11 @@ describe("Decision", () => { { type: "media", media: png }, { type: "json", value: { source: "upload" } }, ], - questions: [{ name: "red", type: "predicate", instructions: "Is the photo mostly red?" }], + questions: { red: { type: "predicate", instructions: "Is the photo mostly red?" } }, }) .pipe( Effect.tap((response) => - Effect.sync(() => expect(response.answers).toEqual([{ name: "red", type: "predicate", probability: 0.99 }])), + Effect.sync(() => expect(response.answers).toEqual({ red: { type: "predicate", probability: 0.99 } })), ), Effect.provide( client((input) => { @@ -342,6 +335,7 @@ describe("Decision", () => { Effect.tap((response) => Effect.sync(() => { expect(response.answers).toEqual(expectedAnswers) + expect(Object.keys(response.answers)).toEqual(["department", "urgency", "refund"]) expect(response.usage?.inputTokens).toBe(40) expect(response.providerMetadata).toEqual({ openai: { model: "gpt-6-luna", usage: { input_tokens: 40, output_tokens: 0, total_tokens: 40 } }, @@ -388,7 +382,7 @@ describe("Decision", () => { { name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" }, ], }) - // Answers come back out of order; the protocol restores question order by name. + // Answers come back out of order; the protocol keys them in question order. return Effect.succeed( json(input, { model: "gpt-6-luna", @@ -429,20 +423,20 @@ describe("Decision", () => { .generate({ model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"), input: [{ type: "text", text: "hello" }], - questions: [ - { name: "greeting", type: "predicate", instructions: "Is this a greeting?" }, - { name: "unsafe", type: "predicate", instructions: "Something the model declines" }, - { name: "explained", type: "predicate", instructions: "Something the model declines with a reason" }, - ], + questions: { + greeting: { type: "predicate", instructions: "Is this a greeting?" }, + unsafe: { type: "predicate", instructions: "Something the model declines" }, + explained: { type: "predicate", instructions: "Something the model declines with a reason" }, + }, }) .pipe( Effect.tap((response) => Effect.sync(() => - expect(response.answers).toEqual([ - { name: "greeting", type: "predicate", probability: 0.99 }, - { name: "unsafe", type: "error", reason: "refusal" }, - { name: "explained", type: "error", reason: "refusal", message: "I can't help with that." }, - ]), + expect(response.answers).toEqual({ + greeting: { type: "predicate", probability: 0.99 }, + unsafe: { type: "error", reason: "refusal" }, + explained: { type: "error", reason: "refusal", message: "I can't help with that." }, + }), ), ), Effect.provide( @@ -468,7 +462,7 @@ describe("Decision", () => { .generate({ model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"), input: [{ type: "media", media: Media.base64("AAAA", "audio/wav") }], - questions: [{ name: "speech", type: "predicate", instructions: "Is someone speaking?" }], + questions: { speech: { type: "predicate", instructions: "Is someone speaking?" } }, }) .pipe(Effect.flip) expect(error.reason._tag).toBe("UnsupportedOperation") @@ -483,12 +477,12 @@ describe("Decision", () => { "typesafe-ai/jev", ), input: [{ type: "text", text: "refund" }], - questions: [{ name: "refund", type: "predicate", instructions: "Is a refund requested?" }], + questions: { refund: { type: "predicate", instructions: "Is a refund requested?" } }, }) .pipe( Effect.tap((response) => Effect.sync(() => { - expect(response.answers).toEqual([{ name: "refund", type: "predicate", probability: 0.98 }]) + expect(response.answers).toEqual({ refund: { type: "predicate", probability: 0.98 } }) expect(response.providerMetadata).toEqual({ "vercel-ai-gateway": { model: "typesafe-ai/jev", usage: { input_tokens: 10, output_tokens: 0 } }, }) @@ -526,12 +520,12 @@ describe("Decision", () => { { type: "media", media: png }, { type: "media", media: Media.base64("AAAA", "audio/wav") }, ], - questions: [{ name: "red", type: "predicate", instructions: "Is the image mostly red?" }], + questions: { red: { type: "predicate", instructions: "Is the image mostly red?" } }, }) .pipe( Effect.tap((response) => Effect.sync(() => { - expect(response.answers).toEqual([{ name: "red", type: "predicate", probability: 0.9 }]) + expect(response.answers).toEqual({ red: { type: "predicate", probability: 0.9 } }) expect(response.providerMetadata).toEqual({ "cloudflare-workers-ai": { model: "clef-omni" } }) }), ), @@ -562,7 +556,7 @@ describe("Decision", () => { "@cf/cloudflare/clef", ), input: [{ type: "text", text: "refund" }], - questions: [{ name: "refund", type: "predicate", instructions: "Is a refund requested?" }], + questions: { refund: { type: "predicate", instructions: "Is a refund requested?" } }, }) .pipe( Effect.flip, @@ -602,7 +596,7 @@ describe("Decision", () => { .generate({ model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"), input: [{ type: "text", text: "refund" }], - questions: [{ name: "tone", type: "choice", instructions: "Tone?", choices: [{ value: "calm" }] }], + questions: { tone: { type: "choice", instructions: "Tone?", choices: [{ value: "calm" }] } }, }) .pipe( Effect.flip, @@ -633,10 +627,10 @@ describe("Decision", () => { .generate({ model: TypeSafeAI.configure({ apiKey: "test" }).decision("jev-latest"), input: [{ type: "text", text: "refund" }], - questions: [ - { name: "refund", type: "predicate", instructions: "Refund?" }, - { name: "angry", type: "predicate", instructions: "Angry?" }, - ], + questions: { + refund: { type: "predicate", instructions: "Refund?" }, + angry: { type: "predicate", instructions: "Angry?" }, + }, }) .pipe( Effect.flip, @@ -649,19 +643,38 @@ describe("Decision", () => { ), ) - it.effect("rejects duplicate question names before network I/O", () => + it.effect("rejects an empty question record before network I/O", () => Effect.gen(function* () { const error = yield* ai.decision - .generate({ - model: TypeSafeAI.decision("jev-latest"), - input: [{ type: "text", text: "hello" }], - questions: [ - { name: "same", type: "predicate", instructions: "One?" }, - { name: "same", type: "predicate", instructions: "Two?" }, - ], - }) + .generate({ model: TypeSafeAI.decision("jev-latest"), input: [{ type: "text", text: "hello" }], questions: {} }) .pipe(Effect.flip) expect(error.reason._tag).toBe("InvalidRequest") }).pipe(Effect.provide(offline)), ) + + it.effect("fails the call when OpenAI Decisions answers a question twice", () => + ai.decision + .generate({ + model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"), + input: [{ type: "text", text: "refund" }], + questions: { refund: { type: "predicate", instructions: "Refund?" } }, + }) + .pipe( + Effect.flip, + Effect.tap((error) => Effect.sync(() => expect(error.reason._tag).toBe("InvalidProviderOutput"))), + Effect.provide( + client((input) => + Effect.succeed( + json(input, { + model: "gpt-6-luna", + answers: [ + { type: "predicate", name: "refund", probability: 0.9 }, + { type: "predicate", name: "refund", probability: 0.1 }, + ], + }), + ), + ), + ), + ), + ) }) diff --git a/packages/ai/test/decision.types.ts b/packages/ai/test/decision.types.ts index e522a21d85c..e36decead69 100644 --- a/packages/ai/test/decision.types.ts +++ b/packages/ai/test/decision.types.ts @@ -11,18 +11,18 @@ type Assert = T const request = ai.decision.request({ model: TypeSafeAI.configure({ apiKey: "test" }).decision("jev-latest"), input: { ticket: "refund" }, - questions: [ - { name: "topic", type: "choice", instructions: "Which team?", choices: [{ value: "billing" }, { value: true }] }, - { name: "severity", type: "score", instructions: "How severe?", levels: [{ label: "Low" }, { label: "High" }] }, - { name: "refund", type: "predicate", instructions: "Refund?" }, - ], + questions: { + topic: { type: "choice", instructions: "Which team?", choices: [{ value: "billing" }, { value: true }] }, + severity: { type: "score", instructions: "How severe?", levels: [{ label: "Low" }, { label: "High" }] }, + refund: { type: "predicate", instructions: "Refund?" }, + }, }) const result = DecisionClient.generate(request) type Answers = Success["answers"] -type Topic = Exclude +type Topic = Exclude type Choice = Assert> -type Name = Assert> +type Name = Assert> type ClientRequirements = Assert, Service>> void (true satisfies Choice) void (true satisfies Name) @@ -32,21 +32,21 @@ Effect.gen(function* () { const response = yield* ai.decision.generate({ model: OpenCode.decision("jev-1.13"), input: [{ type: "text", text: "hello" }], - questions: [{ name: "greeting", type: "predicate", instructions: "Greeting?" }], + questions: { greeting: { type: "predicate", instructions: "Greeting?" } }, }) - const greeting = response.answers[0] + const greeting = response.answers.greeting if (greeting.type === "error") return greeting.probability satisfies number // @ts-expect-error Predicate answers do not contain a selected choice. greeting.choice // @ts-expect-error There is one answer per question. - response.answers[1] + response.answers.farewell }) ;({ model: OpenCode.decision("jev-1.13"), // @ts-expect-error Input is a JSON object or a list of parts, not a plain string. input: "hello", - questions: [{ name: "ok", type: "predicate", instructions: "OK?" }], + questions: { ok: { type: "predicate", instructions: "OK?" } }, }) satisfies DecisionRequestInput declare const route: DecisionModelRoute<{ readonly temperature?: number }> @@ -54,13 +54,13 @@ const custom = DecisionModel.make({ id: "custom", provider: "custom", route }) ai.decision.generate({ model: custom, input: { text: "hello" }, - questions: [{ name: "ok", type: "predicate", instructions: "OK?" }], + questions: { ok: { type: "predicate", instructions: "OK?" } }, providerOptions: { temperature: 0.5 }, }) ;({ model: custom, input: { text: "hello" }, - questions: [{ name: "ok", type: "predicate", instructions: "OK?" }], + questions: { ok: { type: "predicate", instructions: "OK?" } }, // @ts-expect-error Selected decision models retain their provider option types. providerOptions: { temperature: "high" }, }) satisfies DecisionRequestInput @@ -68,14 +68,14 @@ ai.decision.generate({ ai.decision.generate({ model: OpenAI.decision("gpt-6-luna"), input: { text: "hello" }, - questions: [{ name: "ok", type: "predicate", instructions: "OK?" }], + questions: { ok: { type: "predicate", instructions: "OK?" } }, providerOptions: { promptCacheKey: "support-v1", safetyIdentifier: "user-1" }, }) const openai = OpenAI.decision("gpt-6-luna") ;({ model: openai, input: { text: "hello" }, - questions: [{ name: "ok", type: "predicate", instructions: "OK?" }], + questions: { ok: { type: "predicate", instructions: "OK?" } }, // @ts-expect-error OpenAI safety identifiers are strings. providerOptions: { safetyIdentifier: 1 }, }) satisfies DecisionRequestInput @@ -83,14 +83,14 @@ const openai = OpenAI.decision("gpt-6-luna") ai.decision.generate({ model: OpenRouter.decision("typesafe/jev-1.13"), input: { text: "hello" }, - questions: [{ name: "ok", type: "predicate", instructions: "OK?" }], + questions: { ok: { type: "predicate", instructions: "OK?" } }, providerOptions: { provider: { zdr: true }, session_id: "session-1", user: "user-1" }, }) const openrouter = OpenRouter.decision("typesafe/jev-1.13") ;({ model: openrouter, input: { text: "hello" }, - questions: [{ name: "ok", type: "predicate", instructions: "OK?" }], + questions: { ok: { type: "predicate", instructions: "OK?" } }, // @ts-expect-error OpenRouter session IDs are strings. providerOptions: { session_id: 1 }, }) satisfies DecisionRequestInput @@ -98,14 +98,14 @@ const openrouter = OpenRouter.decision("typesafe/jev-1.13") ai.decision.generate({ model: Baseten.decision("decision-models/baseten-elephant-9b-latest"), input: { text: "hello" }, - questions: [{ name: "ok", type: "predicate", instructions: "OK?" }], + questions: { ok: { type: "predicate", instructions: "OK?" } }, providerOptions: { max_len: 512 }, }) const baseten = Baseten.decision("decision-models/baseten-elephant-9b-latest") ;({ model: baseten, input: { text: "hello" }, - questions: [{ name: "ok", type: "predicate", instructions: "OK?" }], + questions: { ok: { type: "predicate", instructions: "OK?" } }, // @ts-expect-error Baseten max_len is a number. providerOptions: { max_len: "512" }, }) satisfies DecisionRequestInput diff --git a/packages/ai/test/provider/decision.recorded.test.ts b/packages/ai/test/provider/decision.recorded.test.ts index 396bbbb9eeb..39a44205806 100644 --- a/packages/ai/test/provider/decision.recorded.test.ts +++ b/packages/ai/test/provider/decision.recorded.test.ts @@ -13,9 +13,8 @@ import { } from "../../src/providers.js" import { recordedTests } from "../recorded-test.js" -const questions = [ - { - name: "department", +const questions = { + department: { type: "choice", instructions: "Which team should handle this support request?", choices: [ @@ -24,8 +23,7 @@ const questions = [ { value: "sales", description: "Pricing, upgrades, or new accounts" }, ], }, - { - name: "urgency", + urgency: { type: "score", instructions: "How urgent is this support request?", levels: [ @@ -34,8 +32,8 @@ const questions = [ { label: "Actively blocking revenue" }, ], }, - { name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" }, -] as const + refund: { type: "predicate", instructions: "Is the customer asking for a refund?" }, +} as const const ticket = "I was charged twice for the same invoice. Please refund the duplicate payment today." @@ -175,13 +173,17 @@ const assertDecision = ( ) => Effect.gen(function* () { const response = yield* ai.decision.generate({ model, input, questions }) - const [department, urgency, refund] = response.answers + const department = response.answers.department + const urgency = response.answers.urgency + const refund = response.answers.refund if (department.type === "error" || urgency.type === "error" || refund.type === "error") throw new Error("expected every question to be answered") expect(department.choice).toBe("billing") expect(department.probabilities.find((item) => item.value === "billing")?.probability).toBeGreaterThan(0.5) expect(department.confidence).toBeGreaterThan(0) - expect(urgency.probabilities.map((item) => item.label)).toEqual(questions[1].levels.map((level) => level.label)) + expect(urgency.probabilities.map((item) => item.label)).toEqual( + questions.urgency.levels.map((level) => level.label), + ) expect(urgency.score).toBeGreaterThanOrEqual(0) expect(urgency.score).toBeLessThanOrEqual(2) expect(urgency.confidence).toBeGreaterThan(0) diff --git a/packages/ai/test/provider/media-entrypoints.test.ts b/packages/ai/test/provider/media-entrypoints.test.ts index 79a1e33e24d..3dc5482b680 100644 --- a/packages/ai/test/provider/media-entrypoints.test.ts +++ b/packages/ai/test/provider/media-entrypoints.test.ts @@ -26,7 +26,7 @@ const sentDecision = (model: DecisionModel) => .generate({ model, input: [{ type: "text", text: "The sky is red." }], - questions: [{ name: "red", type: "predicate", instructions: "Is the sky red?" }], + questions: { red: { type: "predicate", instructions: "Is the sky red?" } }, }) .pipe(Effect.exit, Effect.provide(DecisionClient.layer.pipe(Layer.provide(record(calls))))) return calls