refactor(ai): align media and decision responses with the generate API (#59452)

This commit is contained in:
Aiden Cline
2026-10-26 23:16:18 -05:00
committed by GitHub
Unverified
parent 8b17d2edb4
commit a5587e3110
36 changed files with 122 additions and 247 deletions
+14 -10
View File
@@ -255,7 +255,10 @@ const [refund, department] = response.answers
```
`input` is a JSON object or an ordered list of `text`, `json` and `media` parts. Answers come back in question order.
A question the host declines is `{ name, type: "error", reason: "refusal" }`; the other answers still come back.
A question the host declines is `{ name, type: "error", reason, message? }`, with `reason` such as `"refusal"`; the
other answers still come back.
`response.usage` has `inputTokens`, `outputTokens`, `totalTokens` and the host's reported `cost` when it sends one;
the host's raw usage stays in `providerMetadata`.
Media goes wherever the host reads it, and a kind the host has no place for fails the call. Host-specific fields go in
`providerOptions`, typed by the selected model (OpenAI `promptCacheKey` and `safetyIdentifier`, OpenRouter
`provider` routing, Baseten `max_len`).
@@ -628,7 +631,7 @@ const edit = Image.generate({
The default image format is WEBP; `outputFormat` also accepts PNG/JPEG and `responseFormat: "url"`
returns a signed URL. `aspect` lowers to Meta's `size`, which is only an aspect-ratio hint: Meta renders about 2.5
megapixels whatever you ask, so `resolution` is ignored with a notice. For conversational images, select
megapixels whatever you ask, so `resolution` is ignored. For conversational images, select
`meta.responses("muse-image-1.0")` with `tools: [Meta.imageGeneration({ reasoningStrength: "low" })]`.
Generated images are provider-executed tool results with file content. Retain `response.message` to
replay the signed image handle on the next request. Muse Image accepts only the `image_generation` tool.
@@ -668,16 +671,16 @@ const program = Effect.gen(function* () {
})
```
Common fields are portable in shape, not in support. A route that has to approximate `aspect` or `resolution`
returns what it used in `response.notices`; one that cannot honor them fails with a typed `AIError` before any network
call rather than dropping them, so check this table before swapping only the `model`:
Common fields are portable in shape, not in support. Some routes approximate `aspect` or `resolution` with the nearest
size they offer; others fail with a typed `AIError` before any network call rather than dropping them, so check this
table before swapping only the `model`:
| Provider | `n` | `aspect` | `resolution` | `images` | Moved to `providerOptions` |
| --------------------- | --- | -------------------- | --------------------- | -------------------------------- | -------------------------------------------------- |
| OpenAI | ✓¹ | nearest size, notice | nearest size, notice | ✓ | `size`, `outputFormat`, `mask` |
| OpenAI | ✓¹ | nearest size | nearest size | ✓ | `size`, `outputFormat`, `mask` |
| Google (Gemini) | 1 | ✓ | ✓ (`imageSize`) | ✓ (no public URLs) | `seed` |
| xAI | ✓ | ✓ | `1k`/`2k`, notice | ✓ | none |
| Meta | ✓ | ✓ (hint) | ignored, notice | ✓ | `size`, `outputFormat` |
| xAI | ✓ | ✓ | `1k`/`2k` | ✓ | none |
| Meta | ✓ | ✓ (hint) | ignored | ✓ | `size`, `outputFormat` |
| Z.ai | ✗ | width × height | width × height | ✗ | `size` |
| Alibaba | ✓ | width × height | width × height | ✓ | `size`, `seed` |
| Black Forest Labs | 1 | per model | width × height models | per model (1–8) | `width`, `height`, `seed`, `output_format`, `mask` |
@@ -963,8 +966,9 @@ failures (rate limits, provider 5xx, network errors) with backoff that honors `r
`poll.timeout`; submits and cancels never retry. Interrupting a wait (or aborting its `signal`) does not cancel the
provider job, which keeps running and billing: call `cancel()` to stop it. Failed,
cancelled, and expired generations fail typed with the provider's terminal document on `reason.body`; moderation
outcomes (Veo `raiMediaFilteredReasons`, xAI `respect_moderation`, Runway `SAFETY.*` codes) surface as `notices` when
a video is still returned and as a `ContentPolicy` reason when nothing is.
outcomes (Veo `raiMediaFilteredReasons`, xAI `respect_moderation`, Runway `SAFETY.*` codes) fail with a
`ContentPolicy` reason when no video comes back; when one still does, Veo and xAI keep the raw outcome in
`providerMetadata`.
Provider notes:
+26 -4
View File
@@ -2,7 +2,15 @@ import { Effect, Schema } from "effect"
import { DecisionClient, Service, type Execute } from "./decision-client.js"
import { Media } from "./media.js"
import { tryRequest } from "./media-model.js"
import { AIError, HttpOptions, ModelID, ProviderID, ProviderMetadata, ProviderOptions, Usage } from "./schema/index.js"
import {
AIError,
HttpOptions,
ModelID,
ProviderID,
ProviderMetadata,
ProviderOptions,
type OpenString,
} from "./schema/index.js"
// ---------------------------------------------------------------------------
// Input
@@ -100,11 +108,16 @@ export const ScoreAnswer = Schema.Struct({
})
export type ScoreAnswer = Schema.Schema.Type<typeof ScoreAnswer>
/** The host declined this one question; the other answers still came back. */
/** Why a question has no answer; hosts may report reasons beyond these. */
export type ErrorReason = OpenString<"refusal">
export const ErrorReason: Schema.Codec<ErrorReason> = Schema.Union([Schema.Literal("refusal"), Schema.String])
/** This one question has no answer; the other answers still came back. */
export const ErrorAnswer = Schema.Struct({
name: Schema.String,
type: Schema.Literal("error"),
reason: Schema.Literal("refusal"),
reason: ErrorReason,
message: Schema.optional(Schema.String),
})
export type ErrorAnswer = Schema.Schema.Type<typeof ErrorAnswer>
@@ -255,9 +268,18 @@ export type DecisionRequestInput<
readonly http?: HttpOptions.Input
}
/** What decision hosts report; their raw usage stays in the response's `providerMetadata`. */
export class DecisionUsage extends Schema.Class<DecisionUsage>("Decision.Usage")({
inputTokens: Schema.optional(Schema.Number),
outputTokens: Schema.optional(Schema.Number),
totalTokens: Schema.optional(Schema.Number),
/** USD the host says it charged for this request. */
cost: Schema.optional(Schema.Number),
}) {}
export class DecisionResponse extends Schema.Class<DecisionResponse>("Decision.Response")({
answers: Schema.Array(DecisionAnswer),
usage: Schema.optional(Usage),
usage: Schema.optional(DecisionUsage),
providerMetadata: Schema.optional(ProviderMetadata),
}) {}
+2
View File
@@ -188,7 +188,9 @@ export {
DecisionRequest,
DecisionResponse,
DecisionTextPart,
DecisionUsage,
ErrorAnswer,
ErrorReason,
PredicateAnswer,
PredicateQuestion,
ScoreAnswer,
-1
View File
@@ -22,7 +22,6 @@ export const ImageClient = {
...response.images.map((image, index) => ImageOutputEvent.make({ index, image })),
ImageFinishEvent.make({
usage: response.usage,
notices: response.notices,
providerMetadata: response.providerMetadata,
}),
],
-3
View File
@@ -99,7 +99,6 @@ export type ImageRequestInput<Model extends ImageModel = ImageModel> = Omit<
export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")({
images: Schema.Array(Media.AssetSchema),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}) {
get image() {
@@ -123,7 +122,6 @@ export const ImageOutputEvent = Schema.Struct({
export const ImageFinishEvent = Schema.Struct({
type: Schema.tag("finish"),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "Image.Event.Finish" })
@@ -153,7 +151,6 @@ const collectResponse = (events: ReadonlyArray<ImageEvent>): Effect.Effect<Image
new ImageResponse({
images: events.filter(ImageEvent.is.image).map((event) => event.image),
usage: finish.usage,
notices: finish.notices,
providerMetadata: finish.providerMetadata,
}),
)
+1 -9
View File
@@ -51,7 +51,7 @@ export const Source = Schema.Union([BytesSource, Base64Source, UrlSource, RefSou
export type Source = Schema.Schema.Type<typeof Source>
// ---------------------------------------------------------------------------
// Kind, Info, Notice
// Kind, Info
// ---------------------------------------------------------------------------
export type AspectRatio = `${number}:${number}`
@@ -84,14 +84,6 @@ export const Info = Schema.Struct({
}).annotate({ identifier: "Media.Info" })
export type Info = Schema.Schema.Type<typeof Info>
/** A provider-side partial result such as stripped audio or a moderated sample; never a silent drop. */
export const Notice = Schema.Struct({
type: Schema.Literals(["moderated", "filtered", "other"]),
message: Schema.String,
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "Media.Notice" })
export type Notice = Schema.Schema.Type<typeof Notice>
// ---------------------------------------------------------------------------
// Asset
// ---------------------------------------------------------------------------
+1 -10
View File
@@ -114,10 +114,8 @@ const decodeResult = Effect.fn("FalImages.decodeResult")(function* (
context: MediaProtocol.PollContext<FalQueue.Token>,
) {
const output = yield* decodeQueueResult(response)
const { images, seed, has_nsfw_concepts, ...rest } = output.value
const { images, seed, ...rest } = output.value
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
// With the safety checker on, flagged images come back blacked out rather than omitted.
const flagged = (has_nsfw_concepts ?? []).flatMap((value, index) => (value ? [index] : []))
return new ImageResponse({
images: images.map((image) => {
const info = { width: image.width ?? undefined, height: image.height ?? undefined }
@@ -127,13 +125,6 @@ const decodeResult = Effect.fn("FalImages.decodeResult")(function* (
Media.url(image.url, { mediaType: image.content_type ?? undefined, info })
)
}),
notices:
flagged.length === 0
? undefined
: flagged.map((index) => ({
type: "moderated" as const,
message: `${route.name} flagged image ${index} as NSFW`,
})),
providerMetadata: { fal: { requestId: context.token.requestID, seed: seed ?? undefined, ...rest } },
})
})
@@ -194,44 +194,11 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
}; inspect body for prompt feedback and candidate details`,
)
}
// Candidates that stopped for a safety or policy reason are partial results, not a silent drop.
const notices = [
...(decoded.promptFeedback === undefined
? []
: [
{
type: "filtered" as const,
message: `${route.name} reported prompt feedback`,
providerMetadata: { google: { promptFeedback: decoded.promptFeedback } },
},
]),
...candidates.flatMap((candidate, index) =>
candidate.finishReason === undefined || candidate.finishReason === "STOP"
? []
: [
{
type: "filtered" as const,
message: `${route.name} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${
candidate.finishMessage === undefined ? "" : `: ${candidate.finishMessage}`
}`,
providerMetadata: {
google: {
candidateIndex: candidate.index ?? index,
finishReason: candidate.finishReason,
finishMessage: candidate.finishMessage,
safetyRatings: candidate.safetyRatings,
},
},
},
],
),
]
const usage = decoded.usageMetadata
const outputTokens =
usage?.candidatesTokenCount === undefined ? undefined : usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0)
return new ImageResponse({
images,
notices: notices.length === 0 ? undefined : notices,
usage:
usage === undefined
? undefined
@@ -122,7 +122,6 @@ const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) =
return SpeechStream.finish(route, state, {
...output,
usage: GeminiGenerateContent.usage(state.usage),
notices: GeminiGenerateContent.notices(route.name, state),
providerMetadata: GeminiGenerateContent.providerMetadata(state),
detail: `finish reason: ${state.finishReason}`,
})
@@ -172,7 +172,6 @@ const finish = (state: State) => {
segments: state.segments.length === 0 ? undefined : state.segments,
words: state.words.length === 0 ? undefined : state.words,
usage: GeminiGenerateContent.usage(state.usage),
notices: GeminiGenerateContent.notices(route.name, state),
providerMetadata: GeminiGenerateContent.providerMetadata(state),
}),
])
+1 -6
View File
@@ -181,11 +181,6 @@ const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* (
}),
)
const reasons = generated?.raiMediaFilteredReasons ?? []
const notices = reasons.map((reason) => ({
type: "filtered" as const,
message: `${route.name} filtered media: ${reason}`,
providerMetadata: { google: { raiMediaFilteredReason: reason } },
}))
if (videos.length === 0 && (reasons.length > 0 || (generated?.raiMediaFilteredCount ?? 0) > 0))
return yield* output.contentPolicy(
`${route.name} filtered every video${reasons.length === 0 ? "" : `: ${reasons.join("; ")}`}`,
@@ -193,11 +188,11 @@ const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* (
if (videos.length === 0) return yield* output.invalid(`${route.name} operation completed without any video`)
return new VideoResponse({
videos,
notices: notices.length === 0 ? undefined : notices,
providerMetadata: {
google: {
operation: context.token.operation,
raiMediaFilteredCount: generated?.raiMediaFilteredCount,
raiMediaFilteredReasons: generated?.raiMediaFilteredReasons,
metadata: operation.metadata,
},
},
+1 -10
View File
@@ -18,7 +18,7 @@ export const DEFAULT_BASE_URL = "https://api.meta.ai/v1"
/**
* Provider-native options. Common fields (`n`, `aspect`, `images`) live on the request; Meta picks its own output
* resolution (about 2.5 megapixels), so `resolution` is ignored with a notice.
* resolution (about 2.5 megapixels), so `resolution` is ignored.
*/
export type ImageOptions = {
/** `WIDTHxHEIGHT` aspect ratio hint; it replaces `aspect`, so setting both fails. */
@@ -127,15 +127,6 @@ const decodeResponse = Effect.fn("MetaImages.decodeResponse")(function* (
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
return new ImageResponse({
images,
notices:
context.request.resolution === undefined
? undefined
: [
{
type: "other",
message: `${route.name} picks its own output resolution; ignored ${context.request.resolution}`,
},
],
usage:
decoded.usage === undefined
? undefined
+26 -5
View File
@@ -1,5 +1,5 @@
import { Effect, Schema } from "effect"
import { DecisionResponse, type DecisionAnswer, type DecisionPart, type DecisionRequestFor } from "../decision.js"
import { DecisionResponse, DecisionUsage, type DecisionAnswer, type DecisionPart, type DecisionRequestFor } from "../decision.js"
import { DecisionRoute } from "../route/decision.js"
import { Endpoint } from "../route/endpoint.js"
import { MediaProtocol } from "../route/media-protocol.js"
@@ -41,7 +41,7 @@ const Answer = Schema.Union([
probabilities: Schema.Array(Schema.Struct({ value: Schema.Int, label: Schema.String, probability: Probability })),
confidence: Probability,
}),
Schema.Struct({ type: Schema.Literal("refusal"), name: Schema.String }),
Schema.Struct({ type: Schema.Literal("refusal"), name: Schema.String, message: Schema.optional(Schema.String) }),
])
const Response = Schema.Struct({
model: Schema.String,
@@ -111,14 +111,35 @@ export const protocol = DecisionRoute.protocol({
const output = yield* decodeDocument(response)
const answers = output.value.answers.map(
(answer): DecisionAnswer =>
answer.type === "refusal" ? { name: answer.name, type: "error", reason: "refusal" } : answer,
answer.type === "refusal"
? {
name: answer.name,
type: "error",
reason: "refusal",
...(answer.message === undefined ? {} : { message: answer.message }),
}
: answer,
)
const byName = new Map(answers.map((answer) => [answer.name, answer]))
const ordered = context.request.questions.flatMap((question) => byName.get(question.name) ?? [])
const usage = OpenResponses.mapUsage(output.value.usage, context.providerMetadataKey)
return new DecisionResponse({
answers: ordered.length === answers.length ? ordered : answers,
usage: OpenResponses.mapUsage(output.value.usage, context.providerMetadataKey),
providerMetadata: { [context.providerMetadataKey]: { model: output.value.model } },
usage:
usage === undefined
? undefined
: new DecisionUsage({
inputTokens: usage.inputTokens,
outputTokens: usage.outputTokens,
totalTokens: usage.totalTokens,
cost: usage.cost,
}),
providerMetadata: {
[context.providerMetadataKey]: {
model: output.value.model,
...(output.value.usage === undefined ? {} : { usage: output.value.usage }),
},
},
})
}),
},
@@ -329,12 +329,9 @@ const step = (state: State, frame: Frame) => (typeof frame === "string" ? onEven
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
if (state.completed === 0) return Effect.fail(route.incomplete())
const provider = context.request.model.provider
const chosen = size(context.request)
const notice = chosen === undefined ? undefined : ImageSizing.notice(route.name, context.request, chosen)
return Effect.succeed([
ImageFinishEvent.make({
usage: usage(state.usage, provider),
notices: notice === undefined ? undefined : [notice],
providerMetadata: {
[provider]: {
outputFormat: state.format,
@@ -148,10 +148,6 @@ const decodeImageDocument = (identity: MediaProtocol.Identity) => {
const image = yield* MediaInput.decodedAsset(output.invalid, `${identity.name} result`, data, undefined)
return new ImageResponse({
images: [image],
notices:
document.finish_reason === "CONTENT_FILTERED"
? [{ type: "moderated", message: `${identity.name} blurred the image for violating its content policy` }]
: undefined,
providerMetadata: { stability: { seed: document.seed, finishReason: document.finish_reason } },
})
})
+6 -3
View File
@@ -1,6 +1,7 @@
import { Effect, Schema } from "effect"
import {
DecisionResponse,
DecisionUsage,
type DecisionAnswer,
type DecisionModelCompatibility,
type DecisionPart,
@@ -11,7 +12,7 @@ import type { Media } from "../media.js"
import { DecisionRoute } from "../route/decision.js"
import { Endpoint } from "../route/endpoint.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { AIError, UnsupportedOperationError, Usage, mergeJsonRecords } from "../schema/index.js"
import { AIError, UnsupportedOperationError, mergeJsonRecords } from "../schema/index.js"
import { encodeJson } from "../utils/json.js"
import { JsonObject, ProviderShared } from "./shared.js"
@@ -42,6 +43,7 @@ const NativeUsage = Schema.StructWithRest(
Schema.Struct({
input_tokens: Schema.optional(Schema.Number),
output_tokens: Schema.optional(Schema.Number),
cost: Schema.optional(Schema.Number),
}),
[JsonObject],
)
@@ -211,15 +213,16 @@ export const toResponse = (
usage:
data.usage === undefined
? undefined
: new Usage({
: new DecisionUsage({
inputTokens: data.usage.input_tokens,
outputTokens: data.usage.output_tokens,
totalTokens: ProviderShared.totalTokens(data.usage.input_tokens, data.usage.output_tokens, undefined),
providerMetadata: { [providerMetadataKey]: data.usage },
cost: data.usage.cost,
}),
providerMetadata: {
[providerMetadataKey]: {
model: data.model,
...(data.usage === undefined ? {} : { usage: data.usage }),
...(data.id === undefined ? {} : { responseId: data.id }),
...(data.provider === undefined ? {} : { provider: data.provider }),
...data.provider_metadata?.[providerMetadataKey],
@@ -99,18 +99,6 @@ export const withheld = (name: string, chunk: Chunk, frame: string) => {
})
}
/** Any finish reason other than `STOP` means the output may be cut short, so it is surfaced rather than dropped. */
export const notices = (name: string, state: Metadata): ReadonlyArray<Media.Notice> | undefined =>
state.finishReason === undefined || state.finishReason === "STOP"
? undefined
: [
{
type: contentFiltered(state.finishReason) ? "filtered" : "other",
message: `${name} finished with ${state.finishReason}`,
providerMetadata: { google: { finishReason: state.finishReason } },
},
]
export const usage = (usage: UsageMetadata | undefined): MediaUsage | undefined =>
usage === undefined
? undefined
@@ -1,5 +1,4 @@
import { Effect } from "effect"
import type { Media } from "../../media.js"
import type { MediaProtocol } from "../../route/media-protocol.js"
export interface Size {
@@ -71,14 +70,7 @@ export const nearest = (shape: Shape, sizes: ReadonlyArray<Size>): Size => {
})
}
/** Reports a size more than 2% off the requested aspect or more than √2 per edge off the requested tier. */
export const notice = (name: string, shape: Shape, size: Size): Media.Notice | undefined => {
const requested = missed(shape, size)
if (requested === undefined) return undefined
return { type: "other", message: `${name} doesn't offer ${requested}; used ${describe(size)}` }
}
/** Free width and height that must honor the shape; a size that misses it fails instead of returning a notice. */
/** Free width and height that must honor the shape; a size that misses it fails. */
export const exact = (route: MediaProtocol.Identity, shape: Shape, limits: Limits) => {
if (!isRequested(shape)) return Effect.succeed(undefined)
const size = fit(shape, limits)
@@ -92,6 +84,7 @@ export const exact = (route: MediaProtocol.Identity, shape: Shape, limits: Limit
)
}
/** A size more than 2% off the requested aspect or more than √2 per edge off the requested tier. */
const missed = (shape: Shape, size: Size) => {
const aspectMissed =
shape.aspect !== undefined && Math.abs(Math.log(size.width / size.height / aspectRatio(shape))) > 0.02
@@ -85,7 +85,6 @@ export const finish = (
readonly mediaType: string | undefined
readonly info?: Media.Info
readonly usage?: MediaUsage
readonly notices?: ReadonlyArray<Media.Notice>
readonly providerMetadata?: ProviderMetadata
readonly detail?: string
},
@@ -98,7 +97,6 @@ export const finish = (
SpeechFinishEvent.make({
audio: Media.bytes(concatBytes(state.chunks), output.mediaType, { info: output.info }),
usage: output.usage,
notices: output.notices,
providerMetadata: output.providerMetadata,
}),
])
-7
View File
@@ -95,12 +95,9 @@ const decodeDocument = route.decodeJson(XAIImageResponse)
const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.DecodeContext<Request>,
) {
const output = yield* decodeDocument(response)
const decoded = output.value
const requested = context.request.resolution
const used = resolution(context.request)
const images = yield* Effect.forEach(decoded.data, (item, index) =>
MediaInput.imageOutput(output.invalid, `${route.name} result ${index}`, item, item.mime_type ?? undefined, {
providerMetadata:
@@ -114,10 +111,6 @@ const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* (
// provider metadata.
return new ImageResponse({
images,
notices:
requested === undefined || requested.toLowerCase() === used
? undefined
: [{ type: "other", message: `${route.name} doesn't offer ${requested}; used ${used}` }],
providerMetadata: decoded.usage === undefined ? undefined : { xai: { usage: decoded.usage } },
})
})
+10 -8
View File
@@ -155,13 +155,10 @@ const decodeResult = Effect.fn("XAIVideo.decodeResult")(function* (
}
if (status !== "completed")
return yield* output.ended("expired", `${route.name} request ${context.token.requestID} expired`)
// `respect_moderation: false` marks a filtered result; a URL may still be present, so report it as a notice.
const notices =
decoded.video?.respect_moderation === false
? [{ type: "moderated" as const, message: `${route.name} flagged the generated video for moderation` }]
: undefined
// `respect_moderation: false` marks a filtered result; a URL may still be present, so only a missing one fails.
const moderated = decoded.video?.respect_moderation === false
const url = decoded.video?.url ?? undefined
if (url === undefined && notices !== undefined)
if (url === undefined && moderated)
return yield* output.contentPolicy(`${route.name} withheld the video for moderation`)
if (url === undefined) return yield* output.invalid(`${route.name} completed without a video URL`)
const duration = decoded.video?.duration ?? undefined
@@ -172,8 +169,13 @@ const decodeResult = Effect.fn("XAIVideo.decodeResult")(function* (
info: duration === undefined ? undefined : { durationSeconds: duration },
}),
],
notices,
providerMetadata: { xai: { requestId: context.token.requestID, model: decoded.model ?? undefined } },
providerMetadata: {
xai: {
requestId: context.token.requestID,
model: decoded.model ?? undefined,
respectModeration: decoded.video?.respect_moderation ?? undefined,
},
},
})
})
-12
View File
@@ -87,21 +87,9 @@ const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* (
const output = yield* decodeDocument(response)
const decoded = output.value
if (decoded.data.length === 0) return yield* output.invalid(`${route.name} returned no images`)
const filters = decoded.content_filter ?? []
return new ImageResponse({
// Z.ai returns only URLs and no content type; the media type resolves when the asset is materialized.
images: yield* Effect.forEach(decoded.data, (item) => MediaProtocol.expiringUrl(item.url, OUTPUT_RETENTION)),
// Z.ai reports applied content filters alongside a successful result; surface them instead of dropping them.
notices:
filters.length === 0
? undefined
: filters.map((filter) => ({
type: "moderated" as const,
message: `${route.name} applied a content filter${filter.role === undefined ? "" : ` for ${filter.role}`}${
filter.level === undefined ? "" : ` at level ${filter.level}`
}`,
providerMetadata: { zai: filter },
})),
providerMetadata: {
zai: {
created: decoded.created,
-1
View File
@@ -23,7 +23,6 @@ export const SpeechClient = {
SpeechFinishEvent.make({
audio: response.audio,
usage: response.usage,
notices: response.notices,
providerMetadata: response.providerMetadata,
}),
],
-3
View File
@@ -112,7 +112,6 @@ export class SpeechResponse extends Schema.Class<SpeechResponse>("Speech.Respons
audio: Media.AssetSchema,
timestamps: Schema.optional(Schema.Array(SpeechTimestamp)),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}) {}
@@ -131,7 +130,6 @@ export const SpeechFinishEvent = Schema.Struct({
type: Schema.tag("finish"),
audio: Media.AssetSchema,
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "Speech.Event.Finish" })
@@ -163,7 +161,6 @@ const collectResponse = (events: ReadonlyArray<SpeechEvent>): Effect.Effect<Spee
audio: finish.audio,
timestamps: timestamps.length === 0 ? undefined : timestamps,
usage: finish.usage,
notices: finish.notices,
providerMetadata: finish.providerMetadata,
}),
)
-1
View File
@@ -130,7 +130,6 @@ const transcriptFields = {
language: Schema.optional(Schema.String),
durationSeconds: Schema.optional(Schema.Number),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}
-1
View File
@@ -22,7 +22,6 @@ export const VideoClient = {
...response.videos.map((video, index) => VideoOutputEvent.make({ index, video })),
VideoFinishEvent.make({
usage: response.usage,
notices: response.notices,
providerMetadata: response.providerMetadata,
}),
],
-3
View File
@@ -113,7 +113,6 @@ export type VideoRequestInput<Model extends VideoModel = VideoModel> = Omit<
export class VideoResponse extends Schema.Class<VideoResponse>("Video.Response")({
videos: Schema.Array(Media.AssetSchema),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}) {
get video() {
@@ -130,7 +129,6 @@ export const VideoOutputEvent = Schema.Struct({
export const VideoFinishEvent = Schema.Struct({
type: Schema.tag("finish"),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "Video.Event.Finish" })
@@ -155,7 +153,6 @@ const collectResponse = (events: ReadonlyArray<VideoEvent>): Effect.Effect<Video
new VideoResponse({
videos: events.filter(VideoEvent.is.video).map((event) => event.video),
usage: finish.usage,
notices: finish.notices,
providerMetadata: finish.providerMetadata,
}),
)
+20 -7
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/http"
import { Decision, DecisionClient, Media } from "../src/effect.js"
import { Decision, DecisionClient, DecisionUsage, Media } from "../src/effect.js"
import {
Baseten,
CloudflareWorkersAI,
@@ -94,7 +94,9 @@ describe("Decision", () => {
expect(response.answers).toEqual(expectedAnswers)
expect(response.usage?.totalTokens).toBe(36)
expect(response.providerMetadata).toEqual({ typesafe: { model: "jev-1.14.0" } })
expect(response.providerMetadata).toEqual({
typesafe: { model: "jev-1.14.0", usage: { input_tokens: 30, output_tokens: 6 } },
})
}).pipe(
Effect.provide(
client((input) =>
@@ -159,7 +161,12 @@ describe("Decision", () => {
{ value: true, probability: 0.99 },
{ value: false, probability: 0.01 },
])
expect(response.usage?.providerMetadata).toEqual({ opencode: { input_tokens: 10, output_tokens: 2 } })
expect(response.usage).toEqual(
new DecisionUsage({ inputTokens: 10, outputTokens: 2, totalTokens: 12, cost: 0.0004 }),
)
expect(response.providerMetadata).toEqual({
opencode: { model: "jev-1.13.0", usage: { input_tokens: 10, output_tokens: 2, cost: 0.0004 } },
})
}),
),
Effect.provide(
@@ -180,7 +187,7 @@ describe("Decision", () => {
answers: {
greeting: { type: "choice", choice: "true", probabilities: { false: 0.01, true: 0.99 }, confidence: 1 },
},
usage: { input_tokens: 10, output_tokens: 2 },
usage: { input_tokens: 10, output_tokens: 2, cost: 0.0004 },
}),
)
}),
@@ -315,7 +322,9 @@ describe("Decision", () => {
Effect.sync(() => {
expect(response.answers).toEqual(expectedAnswers)
expect(response.usage?.inputTokens).toBe(40)
expect(response.providerMetadata).toEqual({ openai: { model: "gpt-6-luna" } })
expect(response.providerMetadata).toEqual({
openai: { model: "gpt-6-luna", usage: { input_tokens: 40, output_tokens: 0, total_tokens: 40 } },
})
}),
),
Effect.provide(
@@ -401,6 +410,7 @@ describe("Decision", () => {
questions: [
{ name: "greeting", type: "predicate", instructions: "Is this a greeting?" },
{ name: "unsafe", type: "predicate", instructions: "Something the model declines" },
{ name: "explained", type: "predicate", instructions: "Something the model declines with a reason" },
],
}).pipe(
Effect.tap((response) =>
@@ -408,6 +418,7 @@ describe("Decision", () => {
expect(response.answers).toEqual([
{ name: "greeting", type: "predicate", probability: 0.99 },
{ name: "unsafe", type: "error", reason: "refusal" },
{ name: "explained", type: "error", reason: "refusal", message: "I can't help with that." },
]),
),
),
@@ -419,6 +430,7 @@ describe("Decision", () => {
answers: [
{ type: "predicate", name: "greeting", probability: 0.99 },
{ type: "refusal", name: "unsafe" },
{ type: "refusal", name: "explained", message: "I can't help with that." },
],
}),
),
@@ -450,8 +462,9 @@ describe("Decision", () => {
Effect.tap((response) =>
Effect.sync(() => {
expect(response.answers).toEqual([{ name: "refund", type: "predicate", probability: 0.98 }])
expect(response.providerMetadata).toEqual({ "vercel-ai-gateway": { model: "typesafe-ai/jev" } })
expect(response.usage?.providerMetadata).toHaveProperty("vercel-ai-gateway")
expect(response.providerMetadata).toEqual({
"vercel-ai-gateway": { model: "typesafe-ai/jev", usage: { input_tokens: 10, output_tokens: 0 } },
})
}),
),
Effect.provide(
-16
View File
@@ -44,20 +44,4 @@ describe("ImageSizing", () => {
),
).toEqual({ width: 512, height: 512 })
})
test("reports a size that misses the requested aspect or tier", () => {
expect(ImageSizing.notice("OpenAI Images", { aspect: "16:9" }, { width: 1360, height: 768 })).toBeUndefined()
expect(ImageSizing.notice("OpenAI Images", { aspect: "3:2" }, { width: 1536, height: 1024 })).toBeUndefined()
expect(ImageSizing.notice("OpenAI Images", { aspect: "16:9" }, { width: 1536, height: 1024 })).toEqual({
type: "other",
message: "OpenAI Images doesn't offer 16:9; used 1536x1024 (3:2)",
})
expect(ImageSizing.notice("OpenAI Images", { resolution: "4K" }, { width: 2880, height: 2880 })).toEqual({
type: "other",
message: "OpenAI Images doesn't offer 4K; used 2880x2880 (1:1)",
})
expect(ImageSizing.notice("OpenAI Images", { resolution: "hd" }, { width: 1024, height: 1024 })?.message).toBe(
"OpenAI Images doesn't offer hd; used 1024x1024 (1:1)",
)
})
})
+3 -25
View File
@@ -273,7 +273,7 @@ describe("Image", () => {
),
)
it.effect("lowers aspect to the nearest OpenAI size and reports the substitution", () =>
it.effect("lowers aspect to the nearest OpenAI size", () =>
Effect.gen(function* () {
const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" })
const response = yield* Image.generate({
@@ -281,9 +281,6 @@ describe("Image", () => {
prompt: "A lighthouse",
aspect: "16:9",
})
expect(response.notices).toEqual([
{ type: "other", message: "OpenAI Images doesn't offer 16:9; used 1536x1024 (3:2)" },
])
const conflict = yield* Image.generate({
model: openai.image("gpt-image-2"),
prompt: "A lighthouse",
@@ -310,7 +307,6 @@ describe("Image", () => {
prompt: "A lighthouse",
aspect: "16:9",
})
expect(response.notices).toBeUndefined()
}).pipe(
Effect.provide(
layer((input) => {
@@ -779,7 +775,7 @@ describe("Image", () => {
),
)
it.effect("surfaces filtered Google candidates as notices next to the returned image", () =>
it.effect("returns the image next to filtered Google candidates", () =>
Image.generate({
model: Google.configure({ apiKey: "test", baseURL: "https://generativelanguage.test/v1beta" }).image(
"gemini-3.1-flash-image",
@@ -789,25 +785,7 @@ describe("Image", () => {
Effect.tap((response) =>
Effect.sync(() => {
expect(response.images).toHaveLength(1)
expect(response.notices).toEqual([
{
type: "filtered",
message: "Google Images reported prompt feedback",
providerMetadata: { google: { promptFeedback: { blockReason: "OTHER" } } },
},
{
type: "filtered",
message: "Google Images candidate 1 finished with IMAGE_SAFETY: Blocked.",
providerMetadata: {
google: {
candidateIndex: 1,
finishReason: "IMAGE_SAFETY",
finishMessage: "Blocked.",
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
},
},
},
])
expect(response.providerMetadata?.google).toMatchObject({ promptFeedback: { blockReason: "OTHER" } })
}),
),
Effect.provide(
@@ -130,7 +130,7 @@ describe("Decision recorded", () => {
plan: "pro",
}).pipe(
Effect.map((response) => {
expect(response.providerMetadata).toEqual({ openai: { model: "gpt-6-luna" } })
expect(response.providerMetadata?.openai).toMatchObject({ model: "gpt-6-luna", usage: { input_tokens: 430 } })
expect(response.usage?.outputTokens).toBe(0)
}),
),
@@ -39,7 +39,6 @@ describe("xAI Images", () => {
expect(response.images[1].mediaType).toBe("application/octet-stream")
expect(response.images[1].source).toEqual({ type: "url", url: "https://api.xai.test/image.jpg" })
expect(response.usage).toBeUndefined()
expect(response.notices).toEqual([{ type: "other", message: "xAI Images doesn't offer 4K; used 2k" }])
expect(response.providerMetadata).toEqual({ xai: { usage: { num_images: 2 } } })
}).pipe(
Effect.provide(
@@ -38,13 +38,6 @@ describe("Z.ai Images", () => {
url: "https://cdn.z.ai/generated.png",
expiresAt: 30 * 24 * 60 * 60 * 1000,
})
expect(response.notices).toEqual([
{
type: "moderated",
message: "Z.ai Images applied a content filter for future-role at level 4.5",
providerMetadata: { zai: { role: "future-role", level: 4.5 } },
},
])
expect(response.providerMetadata).toEqual({
zai: {
created: 1_760_335_349,
+1 -7
View File
@@ -248,13 +248,7 @@ describe("Speech", () => {
expect(truncated.reason).toMatchObject({ _tag: "InvalidProviderOutput", classification: "incomplete-stream" })
expect(yield* partial.audio.bytes()).toEqual(Uint8Array.from([1, 2]))
expect(partial.notices).toEqual([
{
type: "other",
message: "Google Speech finished with MAX_TOKENS",
providerMetadata: { google: { finishReason: "MAX_TOKENS" } },
},
])
expect(partial.providerMetadata?.google).toMatchObject({ finishReason: "MAX_TOKENS" })
expect(policy.reason).toMatchObject({ _tag: "ContentPolicy", body: withheld })
}),
)
+1 -7
View File
@@ -612,13 +612,7 @@ describe("Transcription", () => {
expect(truncated.reason).toMatchObject({ _tag: "InvalidProviderOutput", classification: "incomplete-stream" })
expect(partial.text).toBe("Hello")
expect(partial.notices).toEqual([
{
type: "other",
message: "Google Transcription finished with MAX_TOKENS",
providerMetadata: { google: { finishReason: "MAX_TOKENS" } },
},
])
expect(partial.providerMetadata?.google).toMatchObject({ finishReason: "MAX_TOKENS" })
expect(policy.reason).toMatchObject({ _tag: "ContentPolicy", body: withheld })
}),
)
+6 -14
View File
@@ -61,15 +61,8 @@ describe("Video / Google Veo", () => {
expect(response.video.headers).toEqual({ "x-goog-api-key": "test" })
expect(JSON.stringify(response.video)).not.toContain("x-goog-api-key")
expect(Media.from(response.video.source).headers).toBeUndefined()
expect(response.notices).toEqual([
{
type: "filtered",
message: "Google Veo filtered media: audio filtered",
providerMetadata: { google: { raiMediaFilteredReason: "audio filtered" } },
},
])
expect(response.providerMetadata).toEqual({
google: { operation, raiMediaFilteredCount: 1, metadata: undefined },
google: { operation, raiMediaFilteredCount: 1, raiMediaFilteredReasons: ["audio filtered"], metadata: undefined },
})
expect(yield* response.video.bytes()).toEqual(Uint8Array.from([9, 9, 9]))
})
@@ -314,8 +307,9 @@ describe("Video / xAI", () => {
])
expect(response.video.source).toEqual({ type: "url", url: "https://vidgen.x.ai/out.mp4", mediaType: "video/mp4" })
expect(response.video.info).toEqual({ durationSeconds: 10 })
expect(response.notices).toBeUndefined()
expect(response.providerMetadata).toEqual({ xai: { requestId: "req_1", model: "grok-imagine-video-1.5" } })
expect(response.providerMetadata).toEqual({
xai: { requestId: "req_1", model: "grok-imagine-video-1.5", respectModeration: true },
})
}),
)
@@ -393,7 +387,7 @@ describe("Video / xAI", () => {
)
}
it.effect("flags moderated results as a notice and withheld videos as a content policy failure", () =>
it.effect("returns moderated results and fails withheld videos as a content policy failure", () =>
Effect.gen(function* () {
const flagged = yield* Video.generate({ model, prompt: "x" }).pipe(
Effect.provide(
@@ -409,9 +403,7 @@ describe("Video / xAI", () => {
),
),
)
expect(flagged.notices).toEqual([
{ type: "moderated", message: "xAI Video flagged the generated video for moderation" },
])
expect(flagged.providerMetadata).toEqual({ xai: { requestId: "req_1", respectModeration: false } })
const withheld = yield* Video.generate({ model, prompt: "x" }).pipe(
Effect.flip,
Effect.provide(