diff --git a/packages/ai/src/protocols/system-one.ts b/packages/ai/src/protocols/system-one.ts index e2897dfb8c4..007d50cc459 100644 --- a/packages/ai/src/protocols/system-one.ts +++ b/packages/ai/src/protocols/system-one.ts @@ -38,6 +38,8 @@ const Answer = Schema.Union([ probabilities: Schema.Record(Schema.String, Probability), confidence: Probability, }), + // OpenRouter's and OpenCode Zen's answer for a question the upstream model declined. + Schema.Struct({ type: Schema.Literal("refusal"), message: Schema.optional(Schema.String) }), ]).pipe(Schema.toTaggedUnion("type")) const NativeUsage = Schema.StructWithRest( Schema.Struct({ @@ -122,7 +124,8 @@ const placeMedia = Effect.fnUntraced(function* (request: DecisionRequest, mediaI }) const media = parts.flatMap((part) => (part.type === "media" ? [part.media] : [])) if (media.length === 0) return { state: state(parts.map(plain)) } - if (mediaInput === "state") return { state: yield* Effect.forEach(parts, (part) => stateItem(part, unsupported)) } + if (mediaInput === "state") + return { state: yield* Effect.forEach(parts, (part) => stateItem(part, unsupported, true)) } // Chat-style content in one user message: text and JSON as text parts, media as in `state`. if (mediaInput === "state.messages") return { @@ -132,7 +135,7 @@ const placeMedia = Effect.fnUntraced(function* (request: DecisionRequest, mediaI role: "user", content: yield* Effect.forEach(parts, (part) => part.type === "media" - ? stateItem(part, unsupported) + ? stateItem(part, unsupported, false) : Effect.succeed({ type: "text", text: part.type === "text" ? part.text : encodeJson(part.value) }), ), }, @@ -184,23 +187,42 @@ const plain = (part: DecisionPart) => { return undefined } +/** A media part as a Chat Completions content part; `files` sends other media, such as PDFs, as `file` parts. */ const stateItem = ( part: DecisionPart, unsupported: (asset: Media.Asset, accepted: string) => AIError, + files: boolean, ): Effect.Effect => { if (part.type !== "media") return Effect.succeed(plain(part)) const asset = part.media - if (asset.kind !== "image" && asset.kind !== "audio" && asset.kind !== "video") + if (!files && asset.kind !== "image" && asset.kind !== "audio" && asset.kind !== "video") return Effect.fail(unsupported(asset, "images, audio and video")) return ProviderShared.requireInlineMedia(NAME, asset).pipe( Effect.map((inline) => { if (asset.kind === "image") return { type: "image_url", image_url: { url: inline.dataUrl } } if (asset.kind === "video") return { type: "video_url", video_url: { url: inline.dataUrl } } - return { type: "input_audio", input_audio: { data: inline.base64, format: inline.mime.slice("audio/".length) } } + if (asset.kind === "audio") { + const format = inline.mime.slice("audio/".length).toLowerCase() + return { type: "input_audio", input_audio: { data: inline.base64, format: audioFormats[format] ?? format } } + } + return { + type: "file", + file: { file_data: inline.dataUrl, ...(part.filename === undefined ? {} : { filename: part.filename }) }, + } }), ) } +/** OpenRouter's `input_audio` format names for MIME subtypes that aren't one themselves. */ +const audioFormats: Readonly> = { + "x-wav": "wav", + wave: "wav", + "vnd.wave": "wav", + "x-m4a": "m4a", + "x-aiff": "aiff", + "x-flac": "flac", +} + /** Map System One's answers, keyed by question name, back to the request's question order. */ export const toResponse = ( data: Response, @@ -237,6 +259,13 @@ export const toResponse = ( } const toAnswer = (question: DecisionQuestion, answer: typeof Answer.Type): DecisionAnswer => { + if (answer.type === "refusal") + return { + name: question.name, + type: "error", + reason: "refusal", + ...(answer.message === undefined ? {} : { message: answer.message }), + } if (answer.type === "noul") return { name: question.name, type: "predicate", probability: answer.noul } if (answer.type === "choice") { const choices = question.type === "choice" ? question.choices : [] diff --git a/packages/ai/test/decision.test.ts b/packages/ai/test/decision.test.ts index b31d4beef61..54fa1dcd082 100644 --- a/packages/ai/test/decision.test.ts +++ b/packages/ai/test/decision.test.ts @@ -292,6 +292,67 @@ describe("Decision", () => { ), ) + it.effect("sends PDFs as file parts and audio by its OpenRouter format name in the OpenRouter state", () => + ai.decision + .generate({ + model: OpenRouter.configure({ apiKey: "openrouter-key", baseURL: "https://openrouter.test/api/v1" }).decision( + "google/gemini-decision-1", + ), + input: [ + { type: "media", media: Media.base64("JVBE", "application/pdf"), filename: "invoice.pdf" }, + { type: "media", media: Media.base64("AAAA", "audio/x-wav") }, + ], + questions: [{ name: "refund", type: "predicate", instructions: "Is this a refund request?" }], + }) + .pipe( + Effect.provide( + client((input) => { + expect(JSON.parse(input.text).state).toEqual([ + { type: "file", file: { file_data: "data:application/pdf;base64,JVBE", filename: "invoice.pdf" } }, + { type: "input_audio", input_audio: { data: "AAAA", format: "wav" } }, + ]) + return Effect.succeed( + json(input, { model: "google/gemini-decision-1", answers: { refund: { type: "noul", noul: 0.9 } } }), + ) + }), + ), + ), + ) + + it.effect("returns a System One host's refusals as per-question errors", () => + ai.decision + .generate({ + model: OpenRouter.configure({ apiKey: "openrouter-key", baseURL: "https://openrouter.test/api/v1" }).decision( + "openai/gpt-6-luna-decisions", + ), + input: [{ type: "text", text: "hello" }], + questions: [ + { name: "greeting", type: "predicate", instructions: "Is this a greeting?" }, + { name: "unsafe", type: "predicate", instructions: "Something the model declines" }, + ], + }) + .pipe( + Effect.tap((response) => + Effect.sync(() => + expect(response.answers).toEqual([ + { name: "greeting", type: "predicate", probability: 0.99 }, + { name: "unsafe", type: "error", reason: "refusal" }, + ]), + ), + ), + Effect.provide( + client((input) => + Effect.succeed( + json(input, { + model: "openai/gpt-6-luna-decisions", + answers: { greeting: { type: "noul", noul: 0.99 }, unsafe: { type: "refusal" } }, + }), + ), + ), + ), + ), + ) + it.effect("puts media in a user message inside the Baseten state", () => ai.decision .generate({