diff --git a/packages/ai/README.md b/packages/ai/README.md index 45d69c2999c..7c052afa7cf 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -255,7 +255,10 @@ const [refund, department] = response.answers ``` `input` is a JSON object or an ordered list of `text`, `json` and `media` parts. Answers come back in question order. -A question the host declines is `{ name, type: "error", reason: "refusal" }`; the other answers still come back. +A question the host declines is `{ name, type: "error", reason, message? }`, with `reason` such as `"refusal"`; the +other answers still come back. +`response.usage` has `inputTokens`, `outputTokens`, `totalTokens` and the host's reported `cost` when it sends one; +the host's raw usage stays in `providerMetadata`. Media goes wherever the host reads it, and a kind the host has no place for fails the call. Host-specific fields go in `providerOptions`, typed by the selected model (OpenAI `promptCacheKey` and `safetyIdentifier`, OpenRouter `provider` routing, Baseten `max_len`). @@ -628,7 +631,7 @@ const edit = Image.generate({ The default image format is WEBP; `outputFormat` also accepts PNG/JPEG and `responseFormat: "url"` returns a signed URL. `aspect` lowers to Meta's `size`, which is only an aspect-ratio hint: Meta renders about 2.5 -megapixels whatever you ask, so `resolution` is ignored with a notice. For conversational images, select +megapixels whatever you ask, so `resolution` is ignored. For conversational images, select `meta.responses("muse-image-1.0")` with `tools: [Meta.imageGeneration({ reasoningStrength: "low" })]`. Generated images are provider-executed tool results with file content. Retain `response.message` to replay the signed image handle on the next request. Muse Image accepts only the `image_generation` tool. @@ -668,16 +671,16 @@ const program = Effect.gen(function* () { }) ``` -Common fields are portable in shape, not in support. A route that has to approximate `aspect` or `resolution` -returns what it used in `response.notices`; one that cannot honor them fails with a typed `AIError` before any network -call rather than dropping them, so check this table before swapping only the `model`: +Common fields are portable in shape, not in support. Some routes approximate `aspect` or `resolution` with the nearest +size they offer; others fail with a typed `AIError` before any network call rather than dropping them, so check this +table before swapping only the `model`: | Provider | `n` | `aspect` | `resolution` | `images` | Moved to `providerOptions` | | --------------------- | --- | -------------------- | --------------------- | -------------------------------- | -------------------------------------------------- | -| OpenAI | ✓¹ | nearest size, notice | nearest size, notice | ✓ | `size`, `outputFormat`, `mask` | +| OpenAI | ✓¹ | nearest size | nearest size | ✓ | `size`, `outputFormat`, `mask` | | Google (Gemini) | 1 | ✓ | ✓ (`imageSize`) | ✓ (no public URLs) | `seed` | -| xAI | ✓ | ✓ | `1k`/`2k`, notice | ✓ | none | -| Meta | ✓ | ✓ (hint) | ignored, notice | ✓ | `size`, `outputFormat` | +| xAI | ✓ | ✓ | `1k`/`2k` | ✓ | none | +| Meta | ✓ | ✓ (hint) | ignored | ✓ | `size`, `outputFormat` | | Z.ai | ✗ | width × height | width × height | ✗ | `size` | | Alibaba | ✓ | width × height | width × height | ✓ | `size`, `seed` | | Black Forest Labs | 1 | per model | width × height models | per model (1–8) | `width`, `height`, `seed`, `output_format`, `mask` | @@ -963,8 +966,9 @@ failures (rate limits, provider 5xx, network errors) with backoff that honors `r `poll.timeout`; submits and cancels never retry. Interrupting a wait (or aborting its `signal`) does not cancel the provider job, which keeps running and billing: call `cancel()` to stop it. Failed, cancelled, and expired generations fail typed with the provider's terminal document on `reason.body`; moderation -outcomes (Veo `raiMediaFilteredReasons`, xAI `respect_moderation`, Runway `SAFETY.*` codes) surface as `notices` when -a video is still returned and as a `ContentPolicy` reason when nothing is. +outcomes (Veo `raiMediaFilteredReasons`, xAI `respect_moderation`, Runway `SAFETY.*` codes) fail with a +`ContentPolicy` reason when no video comes back; when one still does, Veo and xAI keep the raw outcome in +`providerMetadata`. Provider notes: diff --git a/packages/ai/src/decision.ts b/packages/ai/src/decision.ts index 9417b541827..2d1df43d39d 100644 --- a/packages/ai/src/decision.ts +++ b/packages/ai/src/decision.ts @@ -2,7 +2,15 @@ import { Effect, Schema } from "effect" import { DecisionClient, Service, type Execute } from "./decision-client.js" import { Media } from "./media.js" import { tryRequest } from "./media-model.js" -import { AIError, HttpOptions, ModelID, ProviderID, ProviderMetadata, ProviderOptions, Usage } from "./schema/index.js" +import { + AIError, + HttpOptions, + ModelID, + ProviderID, + ProviderMetadata, + ProviderOptions, + type OpenString, +} from "./schema/index.js" // --------------------------------------------------------------------------- // Input @@ -100,11 +108,16 @@ export const ScoreAnswer = Schema.Struct({ }) export type ScoreAnswer = Schema.Schema.Type -/** The host declined this one question; the other answers still came back. */ +/** Why a question has no answer; hosts may report reasons beyond these. */ +export type ErrorReason = OpenString<"refusal"> +export const ErrorReason: Schema.Codec = Schema.Union([Schema.Literal("refusal"), Schema.String]) + +/** This one question has no answer; the other answers still came back. */ export const ErrorAnswer = Schema.Struct({ name: Schema.String, type: Schema.Literal("error"), - reason: Schema.Literal("refusal"), + reason: ErrorReason, + message: Schema.optional(Schema.String), }) export type ErrorAnswer = Schema.Schema.Type @@ -255,9 +268,18 @@ export type DecisionRequestInput< readonly http?: HttpOptions.Input } +/** What decision hosts report; their raw usage stays in the response's `providerMetadata`. */ +export class DecisionUsage extends Schema.Class("Decision.Usage")({ + inputTokens: Schema.optional(Schema.Number), + outputTokens: Schema.optional(Schema.Number), + totalTokens: Schema.optional(Schema.Number), + /** USD the host says it charged for this request. */ + cost: Schema.optional(Schema.Number), +}) {} + export class DecisionResponse extends Schema.Class("Decision.Response")({ answers: Schema.Array(DecisionAnswer), - usage: Schema.optional(Usage), + usage: Schema.optional(DecisionUsage), providerMetadata: Schema.optional(ProviderMetadata), }) {} diff --git a/packages/ai/src/effect.ts b/packages/ai/src/effect.ts index 325c02ebcc1..97ec8a5c585 100644 --- a/packages/ai/src/effect.ts +++ b/packages/ai/src/effect.ts @@ -188,7 +188,9 @@ export { DecisionRequest, DecisionResponse, DecisionTextPart, + DecisionUsage, ErrorAnswer, + ErrorReason, PredicateAnswer, PredicateQuestion, ScoreAnswer, diff --git a/packages/ai/src/image-client.ts b/packages/ai/src/image-client.ts index 070c0129709..ab6d23ac350 100644 --- a/packages/ai/src/image-client.ts +++ b/packages/ai/src/image-client.ts @@ -22,7 +22,6 @@ export const ImageClient = { ...response.images.map((image, index) => ImageOutputEvent.make({ index, image })), ImageFinishEvent.make({ usage: response.usage, - notices: response.notices, providerMetadata: response.providerMetadata, }), ], diff --git a/packages/ai/src/image.ts b/packages/ai/src/image.ts index cd456563875..f91078f8b4e 100644 --- a/packages/ai/src/image.ts +++ b/packages/ai/src/image.ts @@ -99,7 +99,6 @@ export type ImageRequestInput = Omit< export class ImageResponse extends Schema.Class("Image.Response")({ images: Schema.Array(Media.AssetSchema), usage: Schema.optional(MediaUsage), - notices: Schema.optional(Schema.Array(Media.Notice)), providerMetadata: Schema.optional(ProviderMetadata), }) { get image() { @@ -123,7 +122,6 @@ export const ImageOutputEvent = Schema.Struct({ export const ImageFinishEvent = Schema.Struct({ type: Schema.tag("finish"), usage: Schema.optional(MediaUsage), - notices: Schema.optional(Schema.Array(Media.Notice)), providerMetadata: Schema.optional(ProviderMetadata), }).annotate({ identifier: "Image.Event.Finish" }) @@ -153,7 +151,6 @@ const collectResponse = (events: ReadonlyArray): Effect.Effect event.image), usage: finish.usage, - notices: finish.notices, providerMetadata: finish.providerMetadata, }), ) diff --git a/packages/ai/src/media.ts b/packages/ai/src/media.ts index 0de96d0d21e..3fed4e3c1b2 100644 --- a/packages/ai/src/media.ts +++ b/packages/ai/src/media.ts @@ -51,7 +51,7 @@ export const Source = Schema.Union([BytesSource, Base64Source, UrlSource, RefSou export type Source = Schema.Schema.Type // --------------------------------------------------------------------------- -// Kind, Info, Notice +// Kind, Info // --------------------------------------------------------------------------- export type AspectRatio = `${number}:${number}` @@ -84,14 +84,6 @@ export const Info = Schema.Struct({ }).annotate({ identifier: "Media.Info" }) export type Info = Schema.Schema.Type -/** A provider-side partial result such as stripped audio or a moderated sample; never a silent drop. */ -export const Notice = Schema.Struct({ - type: Schema.Literals(["moderated", "filtered", "other"]), - message: Schema.String, - providerMetadata: Schema.optional(ProviderMetadata), -}).annotate({ identifier: "Media.Notice" }) -export type Notice = Schema.Schema.Type - // --------------------------------------------------------------------------- // Asset // --------------------------------------------------------------------------- diff --git a/packages/ai/src/protocols/fal-images.ts b/packages/ai/src/protocols/fal-images.ts index 87edcccf253..b1d5570fa99 100644 --- a/packages/ai/src/protocols/fal-images.ts +++ b/packages/ai/src/protocols/fal-images.ts @@ -114,10 +114,8 @@ const decodeResult = Effect.fn("FalImages.decodeResult")(function* ( context: MediaProtocol.PollContext, ) { const output = yield* decodeQueueResult(response) - const { images, seed, has_nsfw_concepts, ...rest } = output.value + const { images, seed, ...rest } = output.value if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`) - // With the safety checker on, flagged images come back blacked out rather than omitted. - const flagged = (has_nsfw_concepts ?? []).flatMap((value, index) => (value ? [index] : [])) return new ImageResponse({ images: images.map((image) => { const info = { width: image.width ?? undefined, height: image.height ?? undefined } @@ -127,13 +125,6 @@ const decodeResult = Effect.fn("FalImages.decodeResult")(function* ( Media.url(image.url, { mediaType: image.content_type ?? undefined, info }) ) }), - notices: - flagged.length === 0 - ? undefined - : flagged.map((index) => ({ - type: "moderated" as const, - message: `${route.name} flagged image ${index} as NSFW`, - })), providerMetadata: { fal: { requestId: context.token.requestID, seed: seed ?? undefined, ...rest } }, }) }) diff --git a/packages/ai/src/protocols/google-images.ts b/packages/ai/src/protocols/google-images.ts index c6ecb41e9e1..cd704f8e155 100644 --- a/packages/ai/src/protocols/google-images.ts +++ b/packages/ai/src/protocols/google-images.ts @@ -194,44 +194,11 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* ( }; inspect body for prompt feedback and candidate details`, ) } - // Candidates that stopped for a safety or policy reason are partial results, not a silent drop. - const notices = [ - ...(decoded.promptFeedback === undefined - ? [] - : [ - { - type: "filtered" as const, - message: `${route.name} reported prompt feedback`, - providerMetadata: { google: { promptFeedback: decoded.promptFeedback } }, - }, - ]), - ...candidates.flatMap((candidate, index) => - candidate.finishReason === undefined || candidate.finishReason === "STOP" - ? [] - : [ - { - type: "filtered" as const, - message: `${route.name} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${ - candidate.finishMessage === undefined ? "" : `: ${candidate.finishMessage}` - }`, - providerMetadata: { - google: { - candidateIndex: candidate.index ?? index, - finishReason: candidate.finishReason, - finishMessage: candidate.finishMessage, - safetyRatings: candidate.safetyRatings, - }, - }, - }, - ], - ), - ] const usage = decoded.usageMetadata const outputTokens = usage?.candidatesTokenCount === undefined ? undefined : usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0) return new ImageResponse({ images, - notices: notices.length === 0 ? undefined : notices, usage: usage === undefined ? undefined diff --git a/packages/ai/src/protocols/google-speech.ts b/packages/ai/src/protocols/google-speech.ts index 2f9593060f4..2f385f09d16 100644 --- a/packages/ai/src/protocols/google-speech.ts +++ b/packages/ai/src/protocols/google-speech.ts @@ -122,7 +122,6 @@ const finish = (state: State, context: MediaProtocol.ResponseContext) = return SpeechStream.finish(route, state, { ...output, usage: GeminiGenerateContent.usage(state.usage), - notices: GeminiGenerateContent.notices(route.name, state), providerMetadata: GeminiGenerateContent.providerMetadata(state), detail: `finish reason: ${state.finishReason}`, }) diff --git a/packages/ai/src/protocols/google-transcription.ts b/packages/ai/src/protocols/google-transcription.ts index 8eab87ccf5f..53f4882dd42 100644 --- a/packages/ai/src/protocols/google-transcription.ts +++ b/packages/ai/src/protocols/google-transcription.ts @@ -172,7 +172,6 @@ const finish = (state: State) => { segments: state.segments.length === 0 ? undefined : state.segments, words: state.words.length === 0 ? undefined : state.words, usage: GeminiGenerateContent.usage(state.usage), - notices: GeminiGenerateContent.notices(route.name, state), providerMetadata: GeminiGenerateContent.providerMetadata(state), }), ]) diff --git a/packages/ai/src/protocols/google-video.ts b/packages/ai/src/protocols/google-video.ts index e04c03ebfb1..dfc4a20462b 100644 --- a/packages/ai/src/protocols/google-video.ts +++ b/packages/ai/src/protocols/google-video.ts @@ -181,11 +181,6 @@ const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* ( }), ) const reasons = generated?.raiMediaFilteredReasons ?? [] - const notices = reasons.map((reason) => ({ - type: "filtered" as const, - message: `${route.name} filtered media: ${reason}`, - providerMetadata: { google: { raiMediaFilteredReason: reason } }, - })) if (videos.length === 0 && (reasons.length > 0 || (generated?.raiMediaFilteredCount ?? 0) > 0)) return yield* output.contentPolicy( `${route.name} filtered every video${reasons.length === 0 ? "" : `: ${reasons.join("; ")}`}`, @@ -193,11 +188,11 @@ const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* ( if (videos.length === 0) return yield* output.invalid(`${route.name} operation completed without any video`) return new VideoResponse({ videos, - notices: notices.length === 0 ? undefined : notices, providerMetadata: { google: { operation: context.token.operation, raiMediaFilteredCount: generated?.raiMediaFilteredCount, + raiMediaFilteredReasons: generated?.raiMediaFilteredReasons, metadata: operation.metadata, }, }, diff --git a/packages/ai/src/protocols/meta-images.ts b/packages/ai/src/protocols/meta-images.ts index 615e2e32f04..cabf118babd 100644 --- a/packages/ai/src/protocols/meta-images.ts +++ b/packages/ai/src/protocols/meta-images.ts @@ -18,7 +18,7 @@ export const DEFAULT_BASE_URL = "https://api.meta.ai/v1" /** * Provider-native options. Common fields (`n`, `aspect`, `images`) live on the request; Meta picks its own output - * resolution (about 2.5 megapixels), so `resolution` is ignored with a notice. + * resolution (about 2.5 megapixels), so `resolution` is ignored. */ export type ImageOptions = { /** `WIDTHxHEIGHT` aspect ratio hint; it replaces `aspect`, so setting both fails. */ @@ -127,15 +127,6 @@ const decodeResponse = Effect.fn("MetaImages.decodeResponse")(function* ( if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`) return new ImageResponse({ images, - notices: - context.request.resolution === undefined - ? undefined - : [ - { - type: "other", - message: `${route.name} picks its own output resolution; ignored ${context.request.resolution}`, - }, - ], usage: decoded.usage === undefined ? undefined diff --git a/packages/ai/src/protocols/openai-decisions.ts b/packages/ai/src/protocols/openai-decisions.ts index 1dae596f4c7..5be7c5dc9ca 100644 --- a/packages/ai/src/protocols/openai-decisions.ts +++ b/packages/ai/src/protocols/openai-decisions.ts @@ -1,5 +1,5 @@ import { Effect, Schema } from "effect" -import { DecisionResponse, type DecisionAnswer, type DecisionPart, type DecisionRequestFor } from "../decision.js" +import { DecisionResponse, DecisionUsage, type DecisionAnswer, type DecisionPart, type DecisionRequestFor } from "../decision.js" import { DecisionRoute } from "../route/decision.js" import { Endpoint } from "../route/endpoint.js" import { MediaProtocol } from "../route/media-protocol.js" @@ -41,7 +41,7 @@ const Answer = Schema.Union([ probabilities: Schema.Array(Schema.Struct({ value: Schema.Int, label: Schema.String, probability: Probability })), confidence: Probability, }), - Schema.Struct({ type: Schema.Literal("refusal"), name: Schema.String }), + Schema.Struct({ type: Schema.Literal("refusal"), name: Schema.String, message: Schema.optional(Schema.String) }), ]) const Response = Schema.Struct({ model: Schema.String, @@ -111,14 +111,35 @@ export const protocol = DecisionRoute.protocol({ const output = yield* decodeDocument(response) const answers = output.value.answers.map( (answer): DecisionAnswer => - answer.type === "refusal" ? { name: answer.name, type: "error", reason: "refusal" } : answer, + answer.type === "refusal" + ? { + name: answer.name, + type: "error", + reason: "refusal", + ...(answer.message === undefined ? {} : { message: answer.message }), + } + : answer, ) const byName = new Map(answers.map((answer) => [answer.name, answer])) const ordered = context.request.questions.flatMap((question) => byName.get(question.name) ?? []) + const usage = OpenResponses.mapUsage(output.value.usage, context.providerMetadataKey) return new DecisionResponse({ answers: ordered.length === answers.length ? ordered : answers, - usage: OpenResponses.mapUsage(output.value.usage, context.providerMetadataKey), - providerMetadata: { [context.providerMetadataKey]: { model: output.value.model } }, + usage: + usage === undefined + ? undefined + : new DecisionUsage({ + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + totalTokens: usage.totalTokens, + cost: usage.cost, + }), + providerMetadata: { + [context.providerMetadataKey]: { + model: output.value.model, + ...(output.value.usage === undefined ? {} : { usage: output.value.usage }), + }, + }, }) }), }, diff --git a/packages/ai/src/protocols/openai-images.ts b/packages/ai/src/protocols/openai-images.ts index c096f535dbb..d7d2c920a4e 100644 --- a/packages/ai/src/protocols/openai-images.ts +++ b/packages/ai/src/protocols/openai-images.ts @@ -329,12 +329,9 @@ const step = (state: State, frame: Frame) => (typeof frame === "string" ? onEven const finish = (state: State, context: MediaProtocol.ResponseContext) => { if (state.completed === 0) return Effect.fail(route.incomplete()) const provider = context.request.model.provider - const chosen = size(context.request) - const notice = chosen === undefined ? undefined : ImageSizing.notice(route.name, context.request, chosen) return Effect.succeed([ ImageFinishEvent.make({ usage: usage(state.usage, provider), - notices: notice === undefined ? undefined : [notice], providerMetadata: { [provider]: { outputFormat: state.format, diff --git a/packages/ai/src/protocols/stability-images.ts b/packages/ai/src/protocols/stability-images.ts index 05debb08b68..db154e456fa 100644 --- a/packages/ai/src/protocols/stability-images.ts +++ b/packages/ai/src/protocols/stability-images.ts @@ -148,10 +148,6 @@ const decodeImageDocument = (identity: MediaProtocol.Identity) => { const image = yield* MediaInput.decodedAsset(output.invalid, `${identity.name} result`, data, undefined) return new ImageResponse({ images: [image], - notices: - document.finish_reason === "CONTENT_FILTERED" - ? [{ type: "moderated", message: `${identity.name} blurred the image for violating its content policy` }] - : undefined, providerMetadata: { stability: { seed: document.seed, finishReason: document.finish_reason } }, }) }) diff --git a/packages/ai/src/protocols/system-one.ts b/packages/ai/src/protocols/system-one.ts index fd4b9382bbc..9c81e587d81 100644 --- a/packages/ai/src/protocols/system-one.ts +++ b/packages/ai/src/protocols/system-one.ts @@ -1,6 +1,7 @@ import { Effect, Schema } from "effect" import { DecisionResponse, + DecisionUsage, type DecisionAnswer, type DecisionModelCompatibility, type DecisionPart, @@ -11,7 +12,7 @@ import type { Media } from "../media.js" import { DecisionRoute } from "../route/decision.js" import { Endpoint } from "../route/endpoint.js" import { MediaProtocol } from "../route/media-protocol.js" -import { AIError, UnsupportedOperationError, Usage, mergeJsonRecords } from "../schema/index.js" +import { AIError, UnsupportedOperationError, mergeJsonRecords } from "../schema/index.js" import { encodeJson } from "../utils/json.js" import { JsonObject, ProviderShared } from "./shared.js" @@ -42,6 +43,7 @@ const NativeUsage = Schema.StructWithRest( Schema.Struct({ input_tokens: Schema.optional(Schema.Number), output_tokens: Schema.optional(Schema.Number), + cost: Schema.optional(Schema.Number), }), [JsonObject], ) @@ -211,15 +213,16 @@ export const toResponse = ( usage: data.usage === undefined ? undefined - : new Usage({ + : new DecisionUsage({ inputTokens: data.usage.input_tokens, outputTokens: data.usage.output_tokens, totalTokens: ProviderShared.totalTokens(data.usage.input_tokens, data.usage.output_tokens, undefined), - providerMetadata: { [providerMetadataKey]: data.usage }, + cost: data.usage.cost, }), providerMetadata: { [providerMetadataKey]: { model: data.model, + ...(data.usage === undefined ? {} : { usage: data.usage }), ...(data.id === undefined ? {} : { responseId: data.id }), ...(data.provider === undefined ? {} : { provider: data.provider }), ...data.provider_metadata?.[providerMetadataKey], diff --git a/packages/ai/src/protocols/utils/gemini-generate-content.ts b/packages/ai/src/protocols/utils/gemini-generate-content.ts index 55a7e3f38d0..9291be7cc28 100644 --- a/packages/ai/src/protocols/utils/gemini-generate-content.ts +++ b/packages/ai/src/protocols/utils/gemini-generate-content.ts @@ -99,18 +99,6 @@ export const withheld = (name: string, chunk: Chunk, frame: string) => { }) } -/** Any finish reason other than `STOP` means the output may be cut short, so it is surfaced rather than dropped. */ -export const notices = (name: string, state: Metadata): ReadonlyArray | undefined => - state.finishReason === undefined || state.finishReason === "STOP" - ? undefined - : [ - { - type: contentFiltered(state.finishReason) ? "filtered" : "other", - message: `${name} finished with ${state.finishReason}`, - providerMetadata: { google: { finishReason: state.finishReason } }, - }, - ] - export const usage = (usage: UsageMetadata | undefined): MediaUsage | undefined => usage === undefined ? undefined diff --git a/packages/ai/src/protocols/utils/image-sizing.ts b/packages/ai/src/protocols/utils/image-sizing.ts index 38b53f2f2f8..fac08065df9 100644 --- a/packages/ai/src/protocols/utils/image-sizing.ts +++ b/packages/ai/src/protocols/utils/image-sizing.ts @@ -1,5 +1,4 @@ import { Effect } from "effect" -import type { Media } from "../../media.js" import type { MediaProtocol } from "../../route/media-protocol.js" export interface Size { @@ -71,14 +70,7 @@ export const nearest = (shape: Shape, sizes: ReadonlyArray): Size => { }) } -/** Reports a size more than 2% off the requested aspect or more than √2 per edge off the requested tier. */ -export const notice = (name: string, shape: Shape, size: Size): Media.Notice | undefined => { - const requested = missed(shape, size) - if (requested === undefined) return undefined - return { type: "other", message: `${name} doesn't offer ${requested}; used ${describe(size)}` } -} - -/** Free width and height that must honor the shape; a size that misses it fails instead of returning a notice. */ +/** Free width and height that must honor the shape; a size that misses it fails. */ export const exact = (route: MediaProtocol.Identity, shape: Shape, limits: Limits) => { if (!isRequested(shape)) return Effect.succeed(undefined) const size = fit(shape, limits) @@ -92,6 +84,7 @@ export const exact = (route: MediaProtocol.Identity, shape: Shape, limits: Limit ) } +/** A size more than 2% off the requested aspect or more than √2 per edge off the requested tier. */ const missed = (shape: Shape, size: Size) => { const aspectMissed = shape.aspect !== undefined && Math.abs(Math.log(size.width / size.height / aspectRatio(shape))) > 0.02 diff --git a/packages/ai/src/protocols/utils/speech-stream.ts b/packages/ai/src/protocols/utils/speech-stream.ts index 2b0188e4311..c9cb6fe9753 100644 --- a/packages/ai/src/protocols/utils/speech-stream.ts +++ b/packages/ai/src/protocols/utils/speech-stream.ts @@ -85,7 +85,6 @@ export const finish = ( readonly mediaType: string | undefined readonly info?: Media.Info readonly usage?: MediaUsage - readonly notices?: ReadonlyArray readonly providerMetadata?: ProviderMetadata readonly detail?: string }, @@ -98,7 +97,6 @@ export const finish = ( SpeechFinishEvent.make({ audio: Media.bytes(concatBytes(state.chunks), output.mediaType, { info: output.info }), usage: output.usage, - notices: output.notices, providerMetadata: output.providerMetadata, }), ]) diff --git a/packages/ai/src/protocols/xai-images.ts b/packages/ai/src/protocols/xai-images.ts index 966ef4f4426..4c97e612460 100644 --- a/packages/ai/src/protocols/xai-images.ts +++ b/packages/ai/src/protocols/xai-images.ts @@ -95,12 +95,9 @@ const decodeDocument = route.decodeJson(XAIImageResponse) const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* ( response: HttpClientResponse.HttpClientResponse, - context: MediaProtocol.DecodeContext, ) { const output = yield* decodeDocument(response) const decoded = output.value - const requested = context.request.resolution - const used = resolution(context.request) const images = yield* Effect.forEach(decoded.data, (item, index) => MediaInput.imageOutput(output.invalid, `${route.name} result ${index}`, item, item.mime_type ?? undefined, { providerMetadata: @@ -114,10 +111,6 @@ const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* ( // provider metadata. return new ImageResponse({ images, - notices: - requested === undefined || requested.toLowerCase() === used - ? undefined - : [{ type: "other", message: `${route.name} doesn't offer ${requested}; used ${used}` }], providerMetadata: decoded.usage === undefined ? undefined : { xai: { usage: decoded.usage } }, }) }) diff --git a/packages/ai/src/protocols/xai-video.ts b/packages/ai/src/protocols/xai-video.ts index 4d1649630db..3e1d53059e2 100644 --- a/packages/ai/src/protocols/xai-video.ts +++ b/packages/ai/src/protocols/xai-video.ts @@ -155,13 +155,10 @@ const decodeResult = Effect.fn("XAIVideo.decodeResult")(function* ( } if (status !== "completed") return yield* output.ended("expired", `${route.name} request ${context.token.requestID} expired`) - // `respect_moderation: false` marks a filtered result; a URL may still be present, so report it as a notice. - const notices = - decoded.video?.respect_moderation === false - ? [{ type: "moderated" as const, message: `${route.name} flagged the generated video for moderation` }] - : undefined + // `respect_moderation: false` marks a filtered result; a URL may still be present, so only a missing one fails. + const moderated = decoded.video?.respect_moderation === false const url = decoded.video?.url ?? undefined - if (url === undefined && notices !== undefined) + if (url === undefined && moderated) return yield* output.contentPolicy(`${route.name} withheld the video for moderation`) if (url === undefined) return yield* output.invalid(`${route.name} completed without a video URL`) const duration = decoded.video?.duration ?? undefined @@ -172,8 +169,13 @@ const decodeResult = Effect.fn("XAIVideo.decodeResult")(function* ( info: duration === undefined ? undefined : { durationSeconds: duration }, }), ], - notices, - providerMetadata: { xai: { requestId: context.token.requestID, model: decoded.model ?? undefined } }, + providerMetadata: { + xai: { + requestId: context.token.requestID, + model: decoded.model ?? undefined, + respectModeration: decoded.video?.respect_moderation ?? undefined, + }, + }, }) }) diff --git a/packages/ai/src/protocols/zai-images.ts b/packages/ai/src/protocols/zai-images.ts index faa53acc757..ab5aed1908e 100644 --- a/packages/ai/src/protocols/zai-images.ts +++ b/packages/ai/src/protocols/zai-images.ts @@ -87,21 +87,9 @@ const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* ( const output = yield* decodeDocument(response) const decoded = output.value if (decoded.data.length === 0) return yield* output.invalid(`${route.name} returned no images`) - const filters = decoded.content_filter ?? [] return new ImageResponse({ // Z.ai returns only URLs and no content type; the media type resolves when the asset is materialized. images: yield* Effect.forEach(decoded.data, (item) => MediaProtocol.expiringUrl(item.url, OUTPUT_RETENTION)), - // Z.ai reports applied content filters alongside a successful result; surface them instead of dropping them. - notices: - filters.length === 0 - ? undefined - : filters.map((filter) => ({ - type: "moderated" as const, - message: `${route.name} applied a content filter${filter.role === undefined ? "" : ` for ${filter.role}`}${ - filter.level === undefined ? "" : ` at level ${filter.level}` - }`, - providerMetadata: { zai: filter }, - })), providerMetadata: { zai: { created: decoded.created, diff --git a/packages/ai/src/speech-client.ts b/packages/ai/src/speech-client.ts index fc9f309e548..1eb389a57aa 100644 --- a/packages/ai/src/speech-client.ts +++ b/packages/ai/src/speech-client.ts @@ -23,7 +23,6 @@ export const SpeechClient = { SpeechFinishEvent.make({ audio: response.audio, usage: response.usage, - notices: response.notices, providerMetadata: response.providerMetadata, }), ], diff --git a/packages/ai/src/speech.ts b/packages/ai/src/speech.ts index 0c0535ed9a1..087abd92f2c 100644 --- a/packages/ai/src/speech.ts +++ b/packages/ai/src/speech.ts @@ -112,7 +112,6 @@ export class SpeechResponse extends Schema.Class("Speech.Respons audio: Media.AssetSchema, timestamps: Schema.optional(Schema.Array(SpeechTimestamp)), usage: Schema.optional(MediaUsage), - notices: Schema.optional(Schema.Array(Media.Notice)), providerMetadata: Schema.optional(ProviderMetadata), }) {} @@ -131,7 +130,6 @@ export const SpeechFinishEvent = Schema.Struct({ type: Schema.tag("finish"), audio: Media.AssetSchema, usage: Schema.optional(MediaUsage), - notices: Schema.optional(Schema.Array(Media.Notice)), providerMetadata: Schema.optional(ProviderMetadata), }).annotate({ identifier: "Speech.Event.Finish" }) @@ -163,7 +161,6 @@ const collectResponse = (events: ReadonlyArray): Effect.Effect VideoOutputEvent.make({ index, video })), VideoFinishEvent.make({ usage: response.usage, - notices: response.notices, providerMetadata: response.providerMetadata, }), ], diff --git a/packages/ai/src/video.ts b/packages/ai/src/video.ts index f9cba5030c2..f5ab9af022d 100644 --- a/packages/ai/src/video.ts +++ b/packages/ai/src/video.ts @@ -113,7 +113,6 @@ export type VideoRequestInput = Omit< export class VideoResponse extends Schema.Class("Video.Response")({ videos: Schema.Array(Media.AssetSchema), usage: Schema.optional(MediaUsage), - notices: Schema.optional(Schema.Array(Media.Notice)), providerMetadata: Schema.optional(ProviderMetadata), }) { get video() { @@ -130,7 +129,6 @@ export const VideoOutputEvent = Schema.Struct({ export const VideoFinishEvent = Schema.Struct({ type: Schema.tag("finish"), usage: Schema.optional(MediaUsage), - notices: Schema.optional(Schema.Array(Media.Notice)), providerMetadata: Schema.optional(ProviderMetadata), }).annotate({ identifier: "Video.Event.Finish" }) @@ -155,7 +153,6 @@ const collectResponse = (events: ReadonlyArray): Effect.Effect