fix(ai): accept System One refusals and send PDFs in the OpenRouter state (#63443)

This commit is contained in:
Aiden Cline
2026-11-11 21:00:45 -06:00
committed by GitHub
Unverified
parent 5c965609cf
commit 647e5a7b18
2 changed files with 94 additions and 4 deletions
+33 -4
View File
@@ -38,6 +38,8 @@ const Answer = Schema.Union([
probabilities: Schema.Record(Schema.String, Probability),
confidence: Probability,
}),
// OpenRouter's and OpenCode Zen's answer for a question the upstream model declined.
Schema.Struct({ type: Schema.Literal("refusal"), message: Schema.optional(Schema.String) }),
]).pipe(Schema.toTaggedUnion("type"))
const NativeUsage = Schema.StructWithRest(
Schema.Struct({
@@ -122,7 +124,8 @@ const placeMedia = Effect.fnUntraced(function* (request: DecisionRequest, mediaI
})
const media = parts.flatMap((part) => (part.type === "media" ? [part.media] : []))
if (media.length === 0) return { state: state(parts.map(plain)) }
if (mediaInput === "state") return { state: yield* Effect.forEach(parts, (part) => stateItem(part, unsupported)) }
if (mediaInput === "state")
return { state: yield* Effect.forEach(parts, (part) => stateItem(part, unsupported, true)) }
// Chat-style content in one user message: text and JSON as text parts, media as in `state`.
if (mediaInput === "state.messages")
return {
@@ -132,7 +135,7 @@ const placeMedia = Effect.fnUntraced(function* (request: DecisionRequest, mediaI
role: "user",
content: yield* Effect.forEach(parts, (part) =>
part.type === "media"
? stateItem(part, unsupported)
? stateItem(part, unsupported, false)
: Effect.succeed({ type: "text", text: part.type === "text" ? part.text : encodeJson(part.value) }),
),
},
@@ -184,23 +187,42 @@ const plain = (part: DecisionPart) => {
return undefined
}
/** A media part as a Chat Completions content part; `files` sends other media, such as PDFs, as `file` parts. */
const stateItem = (
part: DecisionPart,
unsupported: (asset: Media.Asset, accepted: string) => AIError,
files: boolean,
): Effect.Effect<unknown, AIError> => {
if (part.type !== "media") return Effect.succeed(plain(part))
const asset = part.media
if (asset.kind !== "image" && asset.kind !== "audio" && asset.kind !== "video")
if (!files && asset.kind !== "image" && asset.kind !== "audio" && asset.kind !== "video")
return Effect.fail(unsupported(asset, "images, audio and video"))
return ProviderShared.requireInlineMedia(NAME, asset).pipe(
Effect.map((inline) => {
if (asset.kind === "image") return { type: "image_url", image_url: { url: inline.dataUrl } }
if (asset.kind === "video") return { type: "video_url", video_url: { url: inline.dataUrl } }
return { type: "input_audio", input_audio: { data: inline.base64, format: inline.mime.slice("audio/".length) } }
if (asset.kind === "audio") {
const format = inline.mime.slice("audio/".length).toLowerCase()
return { type: "input_audio", input_audio: { data: inline.base64, format: audioFormats[format] ?? format } }
}
return {
type: "file",
file: { file_data: inline.dataUrl, ...(part.filename === undefined ? {} : { filename: part.filename }) },
}
}),
)
}
/** OpenRouter's `input_audio` format names for MIME subtypes that aren't one themselves. */
const audioFormats: Readonly<Record<string, string>> = {
"x-wav": "wav",
wave: "wav",
"vnd.wave": "wav",
"x-m4a": "m4a",
"x-aiff": "aiff",
"x-flac": "flac",
}
/** Map System One's answers, keyed by question name, back to the request's question order. */
export const toResponse = (
data: Response,
@@ -237,6 +259,13 @@ export const toResponse = (
}
const toAnswer = (question: DecisionQuestion, answer: typeof Answer.Type): DecisionAnswer => {
if (answer.type === "refusal")
return {
name: question.name,
type: "error",
reason: "refusal",
...(answer.message === undefined ? {} : { message: answer.message }),
}
if (answer.type === "noul") return { name: question.name, type: "predicate", probability: answer.noul }
if (answer.type === "choice") {
const choices = question.type === "choice" ? question.choices : []
+61
View File
@@ -292,6 +292,67 @@ describe("Decision", () => {
),
)
it.effect("sends PDFs as file parts and audio by its OpenRouter format name in the OpenRouter state", () =>
ai.decision
.generate({
model: OpenRouter.configure({ apiKey: "openrouter-key", baseURL: "https://openrouter.test/api/v1" }).decision(
"google/gemini-decision-1",
),
input: [
{ type: "media", media: Media.base64("JVBE", "application/pdf"), filename: "invoice.pdf" },
{ type: "media", media: Media.base64("AAAA", "audio/x-wav") },
],
questions: [{ name: "refund", type: "predicate", instructions: "Is this a refund request?" }],
})
.pipe(
Effect.provide(
client((input) => {
expect(JSON.parse(input.text).state).toEqual([
{ type: "file", file: { file_data: "data:application/pdf;base64,JVBE", filename: "invoice.pdf" } },
{ type: "input_audio", input_audio: { data: "AAAA", format: "wav" } },
])
return Effect.succeed(
json(input, { model: "google/gemini-decision-1", answers: { refund: { type: "noul", noul: 0.9 } } }),
)
}),
),
),
)
it.effect("returns a System One host's refusals as per-question errors", () =>
ai.decision
.generate({
model: OpenRouter.configure({ apiKey: "openrouter-key", baseURL: "https://openrouter.test/api/v1" }).decision(
"openai/gpt-6-luna-decisions",
),
input: [{ type: "text", text: "hello" }],
questions: [
{ name: "greeting", type: "predicate", instructions: "Is this a greeting?" },
{ name: "unsafe", type: "predicate", instructions: "Something the model declines" },
],
})
.pipe(
Effect.tap((response) =>
Effect.sync(() =>
expect(response.answers).toEqual([
{ name: "greeting", type: "predicate", probability: 0.99 },
{ name: "unsafe", type: "error", reason: "refusal" },
]),
),
),
Effect.provide(
client((input) =>
Effect.succeed(
json(input, {
model: "openai/gpt-6-luna-decisions",
answers: { greeting: { type: "noul", noul: 0.99 }, unsafe: { type: "refusal" } },
}),
),
),
),
),
)
it.effect("puts media in a user message inside the Baseten state", () =>
ai.decision
.generate({