diff --git a/packages/ai/src/protocols/anthropic-messages.ts b/packages/ai/src/protocols/anthropic-messages.ts index 5a80e96b98d..062b644a8c5 100644 --- a/packages/ai/src/protocols/anthropic-messages.ts +++ b/packages/ai/src/protocols/anthropic-messages.ts @@ -536,11 +536,7 @@ const ANTHROPIC_BREAKPOINT_CAP = 4 const cacheControl = (breakpoints: Cache.Breakpoints, cache: CacheHint | undefined) => { if (cache?.type !== "ephemeral" && cache?.type !== "persistent") return undefined - if (breakpoints.remaining <= 0) { - breakpoints.dropped += 1 - return undefined - } - breakpoints.remaining -= 1 + if (!Cache.take(breakpoints)) return undefined return { type: "ephemeral" as const, ...(cache.ttl === undefined ? {} : { ttl: cache.ttl }) } } @@ -706,109 +702,57 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMReq } satisfies AnthropicVideoBlock } const fileId = fileIdFromMetadata(part.metadata) + const image = (source: AnthropicImageBlock["source"]): AnthropicImageBlock => { + const transformations = transformationsFromMetadata(part.metadata) + return { + type: "image", + source, + ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), + ...(transformations === undefined ? {} : { transformations }), + } + } + const document = (source: AnthropicDocumentBlock["source"]): AnthropicDocumentBlock => { + const title = documentTitleFromPart(part) + const context = documentContextFromMetadata(part.metadata) + const citations = citationsFromMetadata(part.metadata) + return { + type: "document", + source, + ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), + ...(title === undefined ? {} : { title }), + ...(context === undefined ? {} : { context }), + ...(citations === undefined ? {} : { citations }), + } + } // SDK file sources: FileImageSource:2350 / FileDocumentSource:2344 {type:"file", file_id} if (fileId) { - if (mime.startsWith("image/")) - return { - type: "image" as const, - source: { type: "file" as const, file_id: fileId }, - ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), - ...(transformationsFromMetadata(part.metadata) === undefined - ? {} - : { transformations: transformationsFromMetadata(part.metadata)! }), - } satisfies AnthropicImageBlock - return { - type: "document" as const, - source: { type: "file" as const, file_id: fileId }, - ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), - ...(documentTitleFromPart(part) === undefined ? {} : { title: documentTitleFromPart(part)! }), - ...(documentContextFromMetadata(part.metadata) === undefined - ? {} - : { context: documentContextFromMetadata(part.metadata)! }), - ...(citationsFromMetadata(part.metadata) === undefined - ? {} - : { citations: citationsFromMetadata(part.metadata)! }), - } satisfies AnthropicDocumentBlock + if (mime.startsWith("image/")) return image({ type: "file", file_id: fileId }) + return document({ type: "file", file_id: fileId }) } const rawString = ProviderShared.mediaUrl(part.media)?.trim() // SDK URL sources: URLImageSource:3817 / URLPDFSource:3823 {type:"url", url} if (rawString && isHttpUrl(rawString)) { - if (mime.startsWith("image/")) - return { - type: "image" as const, - source: { type: "url" as const, url: rawString }, - ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), - ...(transformationsFromMetadata(part.metadata) === undefined - ? {} - : { transformations: transformationsFromMetadata(part.metadata)! }), - } satisfies AnthropicImageBlock - if (mime === "application/pdf") - return { - type: "document" as const, - source: { type: "url" as const, url: rawString }, - ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), - ...(documentTitleFromPart(part) === undefined ? {} : { title: documentTitleFromPart(part)! }), - ...(documentContextFromMetadata(part.metadata) === undefined - ? {} - : { context: documentContextFromMetadata(part.metadata)! }), - ...(citationsFromMetadata(part.metadata) === undefined - ? {} - : { citations: citationsFromMetadata(part.metadata)! }), - } satisfies AnthropicDocumentBlock + if (mime.startsWith("image/")) return image({ type: "url", url: rawString }) + if (mime === "application/pdf") return document({ type: "url", url: rawString }) } const media = yield* ProviderShared.requireInlineMedia("Anthropic Messages", part.media) // SDK PlainTextSource:2716 {type:"text", media_type:"text/plain", data} - if (mime === "text/plain") { - const textData = Buffer.from(media.base64, "base64").toString("utf8") - return { - type: "document" as const, - source: { type: "text" as const, media_type: "text/plain" as const, data: textData }, - ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), - ...(documentTitleFromPart(part) === undefined ? {} : { title: documentTitleFromPart(part)! }), - ...(documentContextFromMetadata(part.metadata) === undefined - ? {} - : { context: documentContextFromMetadata(part.metadata)! }), - ...(citationsFromMetadata(part.metadata) === undefined - ? {} - : { citations: citationsFromMetadata(part.metadata)! }), - } satisfies AnthropicDocumentBlock - } + if (mime === "text/plain") + return document({ + type: "text", + media_type: "text/plain", + data: Buffer.from(media.base64, "base64").toString("utf8"), + }) if (media.mime === "application/pdf") - return { - type: "document" as const, - source: { - type: "base64" as const, - media_type: "application/pdf" as const, - data: media.base64, - }, - ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), - ...(documentTitleFromPart(part) === undefined ? {} : { title: documentTitleFromPart(part)! }), - ...(documentContextFromMetadata(part.metadata) === undefined - ? {} - : { context: documentContextFromMetadata(part.metadata)! }), - ...(citationsFromMetadata(part.metadata) === undefined - ? {} - : { citations: citationsFromMetadata(part.metadata)! }), - } satisfies AnthropicDocumentBlock + return document({ type: "base64", media_type: "application/pdf", data: media.base64 }) if (!media.mime.startsWith("image/")) return yield* invalid(`Anthropic Messages does not support media type ${part.media.mediaType}`) - return { - type: "image" as const, - source: { - type: "base64" as const, - media_type: media.mime, - data: media.base64, - }, - ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), - ...(transformationsFromMetadata(part.metadata) === undefined - ? {} - : { transformations: transformationsFromMetadata(part.metadata)! }), - } satisfies AnthropicImageBlock + return image({ type: "base64", media_type: media.mime, data: media.base64 }) }) // Tool results may carry structured text, images, and documents. Keep media as provider-native @@ -1082,11 +1026,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques cache_control: cacheControl(breakpoints, part.cache), })) const messages = yield* lowerMessages(flattened.request, breakpoints) - if (breakpoints.dropped > 0) { - yield* Effect.logWarning( - `Anthropic Messages: dropped ${breakpoints.dropped} cache breakpoint(s); the API allows at most ${ANTHROPIC_BREAKPOINT_CAP} per request.`, - ) - } + yield* Cache.warnDropped(breakpoints, "Anthropic Messages", ANTHROPIC_BREAKPOINT_CAP) const output_config = updates.effort === undefined && format === undefined ? undefined : { effort: updates.effort, format } const maxTokens = generation?.maxTokens ?? DEFAULT_MAX_TOKENS diff --git a/packages/ai/src/protocols/bedrock-converse.ts b/packages/ai/src/protocols/bedrock-converse.ts index 1c56b08555f..45f95cbe86b 100644 --- a/packages/ai/src/protocols/bedrock-converse.ts +++ b/packages/ai/src/protocols/bedrock-converse.ts @@ -24,6 +24,7 @@ import { classifyProviderFailure } from "../provider-error.js" import { JsonObject, optionalArray, ProviderShared } from "./shared.js" import { BedrockAuth } from "./utils/bedrock-auth.js" import { BedrockCache } from "./utils/bedrock-cache.js" +import { warnDropped } from "./utils/cache.js" import { BedrockMedia } from "./utils/bedrock-media.js" import { claudeVersion, supportsThinkingBlockBinding, THINKING_BINDING_BETA } from "./utils/claude-model.js" import { Lifecycle } from "./utils/lifecycle.js" @@ -607,11 +608,7 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: })() const system = lowerSystem(breakpoints, request.system) const messages = yield* lowerMessages(flattened.request, breakpoints) - if (breakpoints.dropped > 0) { - yield* Effect.logWarning( - `Bedrock Converse: dropped ${breakpoints.dropped} cache breakpoint(s); the API allows at most ${BedrockCache.BEDROCK_BREAKPOINT_CAP} per request.`, - ) - } + yield* warnDropped(breakpoints, "Bedrock Converse", BedrockCache.BEDROCK_BREAKPOINT_CAP) const inferenceConfig = (() => { if ( maxTokens === undefined && diff --git a/packages/ai/src/protocols/gemini.ts b/packages/ai/src/protocols/gemini.ts index 7fcd6405837..1296b272cca 100644 --- a/packages/ai/src/protocols/gemini.ts +++ b/packages/ai/src/protocols/gemini.ts @@ -299,6 +299,9 @@ const lowerContentPart = Effect.fnUntraced(function* (part: TextPart | MediaPart const providerMetadata = (key: string, metadata: Record): ProviderMetadata => ({ [key]: metadata }) +const signatureMetadata = (key: string, signature: string | undefined) => + signature ? providerMetadata(key, { thoughtSignature: signature }) : undefined + const thoughtSignature = (metadata: ProviderMetadata | undefined, key: string) => { const value = metadata?.[key] return typeof value?.thoughtSignature === "string" ? value.thoughtSignature : undefined @@ -595,18 +598,14 @@ const finish = (state: ParserState): ReadonlyArray => { lifecycle, events, state.reasoningId, - state.reasoningSignature === undefined - ? undefined - : providerMetadata(state.providerMetadataKey, { thoughtSignature: state.reasoningSignature }), + signatureMetadata(state.providerMetadataKey, state.reasoningSignature), ) if (state.textId !== undefined) lifecycle = Lifecycle.textEnd( lifecycle, events, state.textId, - state.textSignature === undefined - ? undefined - : providerMetadata(state.providerMetadataKey, { thoughtSignature: state.textSignature }), + signatureMetadata(state.providerMetadataKey, state.textSignature), ) Lifecycle.finish(lifecycle, events, { reason: { @@ -683,6 +682,19 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => { let nextTextId = nextState.nextTextId // Supplier ids must be tracked across chunks of the same response, not just within one event's parts. const seenCallIds = new Set(nextState.seenCallIds) + const signed = (signature: string | undefined) => signatureMetadata(state.providerMetadataKey, signature) + const endReasoning = () => { + if (reasoningId === undefined) return + lifecycle = Lifecycle.reasoningEnd(lifecycle, events, reasoningId, signed(reasoningSignature)) + reasoningId = undefined + reasoningSignature = undefined + } + const endText = () => { + if (textId === undefined) return + lifecycle = Lifecycle.textEnd(lifecycle, events, textId, signed(textSignature)) + textId = undefined + textSignature = undefined + } for (const input of candidate.content.parts ?? []) { if ( @@ -710,63 +722,27 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => { events.push( LLMEvent.media({ media: Media.base64(part.inlineData.data, part.inlineData.mimeType), - providerMetadata: signature - ? providerMetadata(state.providerMetadataKey, { thoughtSignature: signature }) - : undefined, + providerMetadata: signed(signature), }), ) continue } if ("text" in part && part.text.length > 0) { if (part.thought) { - if (textId !== undefined) { - lifecycle = Lifecycle.textEnd( - lifecycle, - events, - textId, - textSignature - ? providerMetadata(state.providerMetadataKey, { thoughtSignature: textSignature }) - : undefined, - ) - textId = undefined - textSignature = undefined - } + endText() if (reasoningId === undefined) { reasoningId = `reasoning-${nextReasoningId}` nextReasoningId += 1 } - lifecycle = Lifecycle.reasoningDelta( - lifecycle, - events, - reasoningId, - part.text, - signature ? providerMetadata(state.providerMetadataKey, { thoughtSignature: signature }) : undefined, - ) + lifecycle = Lifecycle.reasoningDelta(lifecycle, events, reasoningId, part.text, signed(signature)) continue } - if (reasoningId !== undefined) { - lifecycle = Lifecycle.reasoningEnd( - lifecycle, - events, - reasoningId, - reasoningSignature - ? providerMetadata(state.providerMetadataKey, { thoughtSignature: reasoningSignature }) - : undefined, - ) - reasoningId = undefined - reasoningSignature = undefined - } + endReasoning() if (textId === undefined) { textId = `text-${nextTextId}` nextTextId += 1 } - lifecycle = Lifecycle.textDelta( - lifecycle, - events, - textId, - part.text, - textSignature ? providerMetadata(state.providerMetadataKey, { thoughtSignature: textSignature }) : undefined, - ) + lifecycle = Lifecycle.textDelta(lifecycle, events, textId, part.text, signed(textSignature)) textSignature = undefined continue } @@ -781,37 +757,15 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => { const duplicate = supplied !== undefined && seenCallIds.has(supplied) if (supplied !== undefined) seenCallIds.add(supplied) const id = supplied !== undefined && !duplicate ? supplied : `tool_${crypto.randomUUID().replaceAll("-", "")}` - if (reasoningId !== undefined) { - lifecycle = Lifecycle.reasoningEnd( - lifecycle, - events, - reasoningId, - reasoningSignature - ? providerMetadata(state.providerMetadataKey, { thoughtSignature: reasoningSignature }) - : undefined, - ) - reasoningId = undefined - reasoningSignature = undefined - } - if (textId !== undefined) { - lifecycle = Lifecycle.textEnd( - lifecycle, - events, - textId, - textSignature ? providerMetadata(state.providerMetadataKey, { thoughtSignature: textSignature }) : undefined, - ) - textId = undefined - textSignature = undefined - } + endReasoning() + endText() lifecycle = Lifecycle.stepStart(lifecycle, events) events.push( LLMEvent.toolCall({ id, name: part.functionCall.name, input, - providerMetadata: part.thoughtSignature - ? providerMetadata(state.providerMetadataKey, { thoughtSignature: part.thoughtSignature }) - : undefined, + providerMetadata: signed(signature), }), ) hasToolCalls = true diff --git a/packages/ai/src/protocols/utils/bedrock-cache.ts b/packages/ai/src/protocols/utils/bedrock-cache.ts index b5695b24b97..ec5344aaeed 100644 --- a/packages/ai/src/protocols/utils/bedrock-cache.ts +++ b/packages/ai/src/protocols/utils/bedrock-cache.ts @@ -1,6 +1,6 @@ import { Schema } from "effect" import { CacheTTL, type CacheHint, type LLMRequest } from "../../schema/index.js" -import { newBreakpoints } from "./cache.js" +import { newBreakpoints, take } from "./cache.js" // Bedrock cache markers are positional: emit a `cachePoint` block immediately // after the content the caller wants treated as a cacheable prefix. Bedrock @@ -47,11 +47,7 @@ export type Breakpoints = ReturnType export const block = (breakpoints: Breakpoints, cache: CacheHint | undefined): CachePointBlock | undefined => { if (!breakpoints.supported) return undefined if (cache?.type !== "ephemeral" && cache?.type !== "persistent") return undefined - if (breakpoints.remaining <= 0) { - breakpoints.dropped += 1 - return undefined - } - breakpoints.remaining -= 1 + if (!take(breakpoints)) return undefined // Older Claude releases cache for five minutes only, so a one-hour request falls back to the default. const ttl = cache.ttl === "1h" && !breakpoints.ttl1h ? undefined : cache.ttl return { cachePoint: { type: "default", ...(ttl === undefined ? {} : { ttl }) } } diff --git a/packages/ai/src/protocols/utils/cache.ts b/packages/ai/src/protocols/utils/cache.ts index bc5052f6ff0..4a46b6f4f8e 100644 --- a/packages/ai/src/protocols/utils/cache.ts +++ b/packages/ai/src/protocols/utils/cache.ts @@ -1,4 +1,5 @@ // Shared counter and lifetime mapping for provider cache lowering. +import { Effect } from "effect" import type { CacheHint, CacheTTL } from "../../schema/index.js" export interface Breakpoints { @@ -8,11 +9,27 @@ export interface Breakpoints { export const newBreakpoints = (cap: number): Breakpoints => ({ remaining: cap, dropped: 0 }) +// Spends one breakpoint, or records it as dropped once the budget is gone. +export const take = (breakpoints: Breakpoints) => { + if (breakpoints.remaining <= 0) { + breakpoints.dropped += 1 + return false + } + breakpoints.remaining -= 1 + return true +} + +export const warnDropped = (breakpoints: Breakpoints, protocol: string, cap: number) => + breakpoints.dropped === 0 + ? Effect.void + : Effect.logWarning( + `${protocol}: dropped ${breakpoints.dropped} cache breakpoint(s); the API allows at most ${cap} per request.`, + ) + export const cacheControl = () => { const breakpoints = newBreakpoints(4) return (cache: CacheHint | undefined) => { - if (cache === undefined || breakpoints.remaining === 0) return undefined - breakpoints.remaining -= 1 + if (cache === undefined || !take(breakpoints)) return undefined return { type: "ephemeral" as const, ttl: cache.ttl } } } diff --git a/packages/ai/src/schema/events.ts b/packages/ai/src/schema/events.ts index 7364704896c..8243b612cc9 100644 --- a/packages/ai/src/schema/events.ts +++ b/packages/ai/src/schema/events.ts @@ -486,11 +486,10 @@ const appendEvent = (state: ResponseState, event: LLMEvent): ResponseState => { } } -const textContent = (text: string, providerMetadata: ProviderMetadata | undefined): ContentPart => - providerMetadata === undefined ? { type: "text", text } : { type: "text", text, providerMetadata } +type ContentKind = "text" | "reasoning" -const reasoningContent = (text: string, providerMetadata: ProviderMetadata | undefined): ContentPart => - providerMetadata === undefined ? { type: "reasoning", text } : { type: "reasoning", text, providerMetadata } +const contentPart = (type: ContentKind, text: string, providerMetadata: ProviderMetadata | undefined): ContentPart => + providerMetadata === undefined ? { type, text } : { type, text, providerMetadata } const contentWith = (state: ResponseState, content: ReadonlyArray): ResponseState => ({ ...state, @@ -505,72 +504,64 @@ const replaceContent = (state: ResponseState, index: number, part: ContentPart) state.message.content.map((item, itemIndex) => (itemIndex === index ? part : item)), ) -const ensureText = (state: ResponseState, id: string, providerMetadata?: ProviderMetadata): ResponseState => { - if (state.textParts[id]) return state - return { - ...appendContent(state, textContent("", providerMetadata)), - textParts: { - ...state.textParts, - [id]: { contentIndex: state.message.content.length, text: "", providerMetadata }, - }, - } +const contentParts = (state: ResponseState, type: ContentKind) => + type === "text" ? state.textParts : state.reasoningParts + +const withContentParts = ( + state: ResponseState, + type: ContentKind, + parts: Readonly>, +): ResponseState => (type === "text" ? { ...state, textParts: parts } : { ...state, reasoningParts: parts }) + +const ensureContent = ( + state: ResponseState, + type: ContentKind, + id: string, + providerMetadata?: ProviderMetadata, +): ResponseState => { + const parts = contentParts(state, type) + if (parts[id]) return state + return withContentParts(appendContent(state, contentPart(type, "", providerMetadata)), type, { + ...parts, + [id]: { contentIndex: state.message.content.length, text: "", providerMetadata }, + }) } -const reduceTextDelta = (state: ResponseState, event: TextDelta): ResponseState => { - const started = ensureText(state, event.id, event.providerMetadata) - const current = started.textParts[event.id] +const reduceContentDelta = ( + state: ResponseState, + type: ContentKind, + event: TextDelta | ReasoningDelta, +): ResponseState => { + const started = ensureContent(state, type, event.id, event.providerMetadata) + const parts = contentParts(started, type) + const current = parts[event.id] if (!current) return started const text = current.text + event.text const providerMetadata = event.providerMetadata ?? current.providerMetadata - return { - ...replaceContent(started, current.contentIndex, textContent(text, providerMetadata)), - textParts: { ...started.textParts, [event.id]: { ...current, text, providerMetadata } }, - } -} - -const reduceTextEnd = (state: ResponseState, event: TextEnd): ResponseState => { - const current = state.textParts[event.id] - if (!current) return state - const text = event.text ?? current.text - const providerMetadata = event.providerMetadata ?? current.providerMetadata - return { - ...replaceContent(state, current.contentIndex, textContent(text, providerMetadata)), - textParts: { ...state.textParts, [event.id]: { ...current, text, providerMetadata } }, - } -} - -const ensureReasoning = (state: ResponseState, id: string, providerMetadata?: ProviderMetadata): ResponseState => { - if (state.reasoningParts[id]) return state - return { - ...appendContent(state, reasoningContent("", providerMetadata)), - reasoningParts: { - ...state.reasoningParts, - [id]: { contentIndex: state.message.content.length, text: "", providerMetadata }, + return withContentParts( + replaceContent(started, current.contentIndex, contentPart(type, text, providerMetadata)), + type, + { + ...parts, + [event.id]: { ...current, text, providerMetadata }, }, - } + ) } -const reduceReasoningDelta = (state: ResponseState, event: ReasoningDelta): ResponseState => { - const started = ensureReasoning(state, event.id, event.providerMetadata) - const current = started.reasoningParts[event.id] - if (!current) return started - const text = current.text + event.text - const providerMetadata = event.providerMetadata ?? current.providerMetadata - return { - ...replaceContent(started, current.contentIndex, reasoningContent(text, providerMetadata)), - reasoningParts: { ...started.reasoningParts, [event.id]: { ...current, text, providerMetadata } }, - } -} - -const reduceReasoningEnd = (state: ResponseState, event: ReasoningEnd): ResponseState => { - const current = state.reasoningParts[event.id] +const reduceContentEnd = (state: ResponseState, type: ContentKind, event: TextEnd | ReasoningEnd): ResponseState => { + const parts = contentParts(state, type) + const current = parts[event.id] if (!current) return state const text = event.text ?? current.text const providerMetadata = event.providerMetadata ?? current.providerMetadata - return { - ...replaceContent(state, current.contentIndex, reasoningContent(text, providerMetadata)), - reasoningParts: { ...state.reasoningParts, [event.id]: { ...current, text, providerMetadata } }, - } + return withContentParts( + replaceContent(state, current.contentIndex, contentPart(type, text, providerMetadata)), + type, + { + ...parts, + [event.id]: { ...current, text, providerMetadata }, + }, + ) } const reduceToolInputStart = (state: ResponseState, event: ToolInputStart): ResponseState => ({ @@ -642,17 +633,17 @@ const reduceResponseState = (state: ResponseState, event: LLMEvent): ResponseSta case "compaction": return appendContent(next, event) case "text-start": - return ensureText(next, event.id, event.providerMetadata) + return ensureContent(next, "text", event.id, event.providerMetadata) case "text-delta": - return reduceTextDelta(next, event) + return reduceContentDelta(next, "text", event) case "text-end": - return reduceTextEnd(next, event) + return reduceContentEnd(next, "text", event) case "reasoning-start": - return ensureReasoning(next, event.id, event.providerMetadata) + return ensureContent(next, "reasoning", event.id, event.providerMetadata) case "reasoning-delta": - return reduceReasoningDelta(next, event) + return reduceContentDelta(next, "reasoning", event) case "reasoning-end": - return reduceReasoningEnd(next, event) + return reduceContentEnd(next, "reasoning", event) case "tool-input-start": return reduceToolInputStart(next, event) case "tool-input-delta":