From fee1ca5aa9e1d62a7ca42ef7eeaa8c02fe6ed080 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:25:31 -0600 Subject: [PATCH] feat(ai): support predicate criteria in decisions (#62819) --- packages/ai/README.md | 2 + packages/ai/src/decision.ts | 4 + packages/ai/src/protocols/openai-decisions.ts | 15 +++- packages/ai/src/protocols/system-one.ts | 9 ++- packages/ai/test/decision.test.ts | 80 +++++++++++++++++++ .../client/src/promise/generated/types.ts | 7 +- packages/core/test/generate.test.ts | 16 +++- packages/protocol/openapi.json | 12 +++ packages/schema/src/generate.ts | 3 + 9 files changed, 144 insertions(+), 4 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index fd1b9783dd7..c05835007db 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -322,6 +322,8 @@ const [refund, department] = response.answers ``` `input` is a JSON object or an ordered list of `text`, `json` and `media` parts. Answers come back in question order. +A `predicate` can say what counts as true and as false with `criteria: { true, false }`; hosts without a criteria field +read them appended to the instructions. A question the host declines is `{ name, type: "error", reason, message? }`, with `reason` such as `"refusal"`; the other answers still come back. `response.usage` has `inputTokens`, `outputTokens`, `totalTokens` and the host's reported `cost` when it sends one; diff --git a/packages/ai/src/decision.ts b/packages/ai/src/decision.ts index b9f6181036f..216c8623574 100644 --- a/packages/ai/src/decision.ts +++ b/packages/ai/src/decision.ts @@ -51,6 +51,10 @@ export const PredicateQuestion = Schema.Struct({ name: Schema.String, type: Schema.Literal("predicate"), instructions: Schema.String, + /** What counts as true and as false, for statements the instructions alone leave ambiguous. */ + criteria: Schema.optional( + Schema.Struct({ true: Schema.optional(Schema.String), false: Schema.optional(Schema.String) }), + ), }) export type PredicateQuestion = Schema.Schema.Type diff --git a/packages/ai/src/protocols/openai-decisions.ts b/packages/ai/src/protocols/openai-decisions.ts index 50c27b7d314..0c300d98dfa 100644 --- a/packages/ai/src/protocols/openai-decisions.ts +++ b/packages/ai/src/protocols/openai-decisions.ts @@ -72,7 +72,20 @@ const fromRequest = Effect.fn("OpenAIDecisions.fromRequest")(function* (request: ), model: request.model.id, input: [{ role: "user", content: yield* Effect.forEach(request.input, (part) => inputPart(request, part)) }], - questions: request.questions, + questions: request.questions.map((question) => { + if (question.type !== "predicate") return question + // Predicates have no criteria field, so what true and false mean is appended to the instructions. + const meanings = [ + question.criteria?.true === undefined ? [] : [`True means: ${question.criteria.true}`], + question.criteria?.false === undefined ? [] : [`False means: ${question.criteria.false}`], + ].flat() + return { + name: question.name, + type: question.type, + instructions: + meanings.length === 0 ? question.instructions : `${question.instructions}\n\n${meanings.join("\n")}`, + } + }), } }) diff --git a/packages/ai/src/protocols/system-one.ts b/packages/ai/src/protocols/system-one.ts index 588cf57ca46..10bb6a60696 100644 --- a/packages/ai/src/protocols/system-one.ts +++ b/packages/ai/src/protocols/system-one.ts @@ -70,7 +70,14 @@ export const encodeBody = Effect.fn("SystemOne.encodeBody")(function* (request: const encodeQuestions = (questions: ReadonlyArray) => Effect.forEach(questions, (question): Effect.Effect => { if (question.type === "predicate") - return Effect.succeed([question.name, { type: "noul", instructions: question.instructions }] as const) + return Effect.succeed([ + question.name, + { + type: "noul", + instructions: question.instructions, + ...(question.criteria === undefined ? {} : { criteria: question.criteria }), + }, + ] as const) if (question.type === "score") return Effect.succeed([ question.name, diff --git a/packages/ai/test/decision.test.ts b/packages/ai/test/decision.test.ts index 03ed44c7585..8b84c828e89 100644 --- a/packages/ai/test/decision.test.ts +++ b/packages/ai/test/decision.test.ts @@ -35,6 +35,16 @@ const questions = [ { name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" }, ] as const +const defect = { + name: "defect", + type: "predicate", + instructions: "Is the customer reporting a product defect?", + criteria: { + true: "The product itself is broken or faulty.", + false: "Preference, sizing, shipping, or pricing complaint.", + }, +} as const + const systemOneAnswers = { department: { type: "choice", choice: "billing", probabilities: { technical: 0.1, billing: 0.9 }, confidence: 0.8 }, urgency: { @@ -516,6 +526,76 @@ describe("Decision", () => { ), ) + it.effect("sends predicate criteria as System One criteria", () => + ai.decision + .generate({ + model: TypeSafeAI.configure({ apiKey: "test" }).decision("jev-latest"), + input: [{ type: "text", text: "I returned the sweater because I prefer green." }], + questions: [defect], + }) + .pipe( + Effect.tap((response) => + Effect.sync(() => + expect(response.answers).toEqual([{ name: "defect", type: "predicate", probability: 0.03 }]), + ), + ), + Effect.provide( + client((input) => { + expect(JSON.parse(input.text).questions).toEqual({ + defect: { + type: "noul", + instructions: "Is the customer reporting a product defect?", + criteria: { + true: "The product itself is broken or faulty.", + false: "Preference, sizing, shipping, or pricing complaint.", + }, + }, + }) + return Effect.succeed( + json(input, { model: "jev-latest", answers: { defect: { type: "noul", noul: 0.03 } } }), + ) + }), + ), + ), + ) + + it.effect("appends predicate criteria to OpenAI Decisions instructions", () => + ai.decision + .generate({ + model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"), + input: [{ type: "text", text: "I returned the sweater because I prefer green." }], + questions: [defect, { ...defect, name: "only_true", criteria: { true: "The product is broken." } }], + }) + .pipe( + Effect.provide( + client((input) => { + expect(JSON.parse(input.text).questions).toEqual([ + { + name: "defect", + type: "predicate", + instructions: + "Is the customer reporting a product defect?\n\nTrue means: The product itself is broken or faulty.\nFalse means: Preference, sizing, shipping, or pricing complaint.", + }, + { + name: "only_true", + type: "predicate", + instructions: "Is the customer reporting a product defect?\n\nTrue means: The product is broken.", + }, + ]) + return Effect.succeed( + json(input, { + model: "gpt-6-luna", + answers: [ + { type: "predicate", name: "defect", probability: 0.03 }, + { type: "predicate", name: "only_true", probability: 0.04 }, + ], + }), + ) + }), + ), + ), + ) + it.effect("puts media in Clef's top-level lists on Cloudflare Run", () => ai.decision .generate({ diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index 61344721a1e..71b866111b2 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -3953,7 +3953,12 @@ export type ExperimentalGenerateDecisionsInput = { | { readonly type: "file"; readonly data: string; readonly mime: string; readonly name?: string } > readonly questions: ReadonlyArray< - | { readonly type: "predicate"; readonly name: string; readonly instructions: string } + | { + readonly type: "predicate" + readonly name: string + readonly instructions: string + readonly criteria?: { readonly true?: string; readonly false?: string } + } | { readonly type: "choice" readonly name: string diff --git a/packages/core/test/generate.test.ts b/packages/core/test/generate.test.ts index 77a3faa9013..005240b4c66 100644 --- a/packages/core/test/generate.test.ts +++ b/packages/core/test/generate.test.ts @@ -502,12 +502,26 @@ describe("image and decision generation", () => { { label: "Actively blocking revenue" }, ], }, - { type: "predicate", name: "refund", instructions: "Is the customer asking for a refund?" }, + { + type: "predicate", + name: "refund", + instructions: "Is the customer asking for a refund?", + criteria: { true: "They want money back.", false: "Anything else." }, + }, ], }) expect(requests[0]).toMatchObject({ url: "https://openrouter.ai/api/v1/systemone", headers: { authorization: "Bearer openrouter-key" }, + body: { + questions: { + refund: { + type: "noul", + instructions: "Is the customer asking for a refund?", + criteria: { true: "They want money back.", false: "Anything else." }, + }, + }, + }, }) expect(result.answers.map((answer) => [answer.name, answer.type])).toEqual([ ["department", "choice"], diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index aa5745c3e10..56c1a3c17df 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -19036,6 +19036,18 @@ }, "instructions": { "type": "string" + }, + "criteria": { + "type": "object", + "properties": { + "true": { + "type": "string" + }, + "false": { + "type": "string" + } + }, + "additionalProperties": false } }, "required": ["type", "name", "instructions"], diff --git a/packages/schema/src/generate.ts b/packages/schema/src/generate.ts index 35c0d9bf4fe..67f33a1dad8 100644 --- a/packages/schema/src/generate.ts +++ b/packages/schema/src/generate.ts @@ -115,6 +115,9 @@ export const PredicateQuestion = Schema.Struct({ type: Schema.Literal("predicate"), name: QuestionName, instructions: Schema.String, + criteria: Schema.Struct({ true: Schema.String.pipe(optional), false: Schema.String.pipe(optional) }) + .pipe(optional) + .annotate(description("What counts as true and as false, for statements the instructions alone leave ambiguous")), }).annotate({ identifier: "Generate.PredicateQuestion", description: "The probability that a statement is true" }) export interface ChoiceQuestion extends Schema.Schema.Type {}