diff --git a/packages/ai/example/tutorial.ts b/packages/ai/example/tutorial.ts index ef58009f629..d4101475b27 100644 --- a/packages/ai/example/tutorial.ts +++ b/packages/ai/example/tutorial.ts @@ -184,7 +184,7 @@ const FakeProtocol = Protocol.make({ Effect.succeed({ model: request.model.id, input: request.messages - .flatMap((message) => message.content) + .flatMap((message) => [...message.content]) .filter((part) => part.type === "text") .map((part) => part.text) .join("\n"), diff --git a/packages/ai/src/cache-policy.ts b/packages/ai/src/cache-policy.ts index c24a4267495..b50992c47a2 100644 --- a/packages/ai/src/cache-policy.ts +++ b/packages/ai/src/cache-policy.ts @@ -12,10 +12,13 @@ // count against the four-breakpoint budget; auto only fills remaining slots. import { CacheHint, type CachePolicy, type CachePolicyObject, type CacheTTL } from "./schema/options.js" import { + AssistantMessage, type LLMRequest, - Message, + type Message, SystemMessage, ToolDefinition, + ToolMessage, + UserMessage, type ContentPart, type ToolEntry, } from "./schema/messages.js" @@ -148,18 +151,7 @@ const markMessageAt = ( const existing = target.content[markAt] if (!existing || !isCacheablePart(existing) || existing.cache || budget.remaining === 0) return messages budget.remaining -= 1 - const next = - target.role === "system" - ? new SystemMessage({ - ...target, - content: target.content.map((part, i) => - i === markAt && part.type === "text" ? { ...part, cache: hint } : part, - ), - }) - : Message.make({ - ...target, - content: target.content.map((part, i) => (i === markAt ? { ...existing, cache: hint } : part)), - }) + const next = cachePartAt(target, markAt, hint) // Single pass over `messages`, substituting the one updated entry. Long // conversations call this on every request, so avoid `.map()` here — its // closure dispatch and identity copies show up in profiling. @@ -168,6 +160,22 @@ const markMessageAt = ( return result } +const cachePartAt = (message: Message, index: number, hint: CacheHint): Message => { + switch (message.role) { + case "system": + return new SystemMessage({ ...message, content: cacheContentAt(message.content, index, hint) }) + case "user": + return new UserMessage({ ...message, content: cacheContentAt(message.content, index, hint) }) + case "assistant": + return new AssistantMessage({ ...message, content: cacheContentAt(message.content, index, hint) }) + case "tool": + return new ToolMessage({ ...message, content: cacheContentAt(message.content, index, hint) }) + } +} + +const cacheContentAt = (content: ReadonlyArray, index: number, hint: CacheHint) => + content.map((part, i) => (i === index ? { ...part, cache: hint } : part)) + const markMessages = ( messages: ReadonlyArray, strategy: NonNullable, diff --git a/packages/ai/src/llm.ts b/packages/ai/src/llm.ts index 23ef2fe82e9..6dc070b1aae 100644 --- a/packages/ai/src/llm.ts +++ b/packages/ai/src/llm.ts @@ -44,7 +44,7 @@ export type LLMRequestInput = TextR /** Exactly one of `prompt`, a single user message, or `messages`, the full conversation. */ export type PromptOrMessages = - | { readonly prompt: Message.ContentInput; readonly messages?: never } + | { readonly prompt: Message.UserContentInput; readonly messages?: never } | { readonly messages: ReadonlyArray; readonly prompt?: never } interface TextRequestFields diff --git a/packages/ai/src/protocols/anthropic-messages.ts b/packages/ai/src/protocols/anthropic-messages.ts index 94714ac9690..d6eee0f4eb3 100644 --- a/packages/ai/src/protocols/anthropic-messages.ts +++ b/packages/ai/src/protocols/anthropic-messages.ts @@ -838,7 +838,6 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest, breakpoi content.push(yield* lowerMedia(part, request, breakpoints)) continue } - return yield* ProviderShared.unsupportedContent("Anthropic Messages", "user", ["text", "media"]) } if (content.length > 0) messages.push({ role: "user", content }) continue @@ -896,8 +895,6 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest, breakpoi const content: AnthropicToolResultBlock[] = [] for (const part of message.content) { - if (!ProviderShared.supportsContent(part, ["tool-result"])) - return yield* ProviderShared.unsupportedContent("Anthropic Messages", "tool", ["tool-result"]) content.push({ type: "tool_result", tool_use_id: scrubToolCallID(part.id), diff --git a/packages/ai/src/protocols/bedrock-converse.ts b/packages/ai/src/protocols/bedrock-converse.ts index 45f95cbe86b..6495524c681 100644 --- a/packages/ai/src/protocols/bedrock-converse.ts +++ b/packages/ai/src/protocols/bedrock-converse.ts @@ -384,8 +384,6 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest, breakpoi if (message.role === "user") { const content: BedrockUserBlock[] = [] for (const part of message.content) { - if (!ProviderShared.supportsContent(part, ["text", "media"])) - return yield* ProviderShared.unsupportedContent("Bedrock Converse", "user", ["text", "media"]) if (part.type === "text") { content.push(...textWithCache(breakpoints, part.text, part.cache)) continue @@ -437,8 +435,6 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest, breakpoi const content: BedrockUserBlock[] = [] for (const part of message.content) { - if (!ProviderShared.supportsContent(part, ["tool-result"])) - return yield* ProviderShared.unsupportedContent("Bedrock Converse", "tool", ["tool-result"]) const result = yield* lowerToolResult(part, documentNames, normalizeID) const images: BedrockMedia.ImageBlock[] = hoistImages ? result.toolResult.content.filter((item) => "image" in item) diff --git a/packages/ai/src/protocols/gemini.ts b/packages/ai/src/protocols/gemini.ts index 1296b272cca..0ffbf3e43ec 100644 --- a/packages/ai/src/protocols/gemini.ts +++ b/packages/ai/src/protocols/gemini.ts @@ -340,8 +340,6 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest) { if (message.role === "user") { const parts: Array> = [] for (const part of message.content) { - if (!ProviderShared.supportsContent(part, ["text", "media"])) - return yield* ProviderShared.unsupportedContent("Gemini", "user", ["text", "media"]) parts.push(yield* lowerContentPart(part)) } contents.push({ role: "user", parts }) @@ -395,8 +393,6 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest) { const parts: Array> = [] for (const part of message.content) { - if (!ProviderShared.supportsContent(part, ["tool-result"])) - return yield* ProviderShared.unsupportedContent("Gemini", "tool", ["tool-result"]) if (part.result.type !== "content") { parts.push({ functionResponse: { diff --git a/packages/ai/src/protocols/meta-responses.ts b/packages/ai/src/protocols/meta-responses.ts index 08493da9022..44ac6a39c8d 100644 --- a/packages/ai/src/protocols/meta-responses.ts +++ b/packages/ai/src/protocols/meta-responses.ts @@ -1,7 +1,7 @@ import { Effect, Schema } from "effect" import { Base64 } from "effect/encoding" import { Protocol } from "../route/protocol.js" -import { LLMEvent, type LLMRequest, Message, ToolResultPart } from "../schema/index.js" +import { AssistantMessage, LLMEvent, type LLMRequest, ToolResultPart } from "../schema/index.js" import { OpenResponses } from "./open-responses.js" import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js" import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js" @@ -65,9 +65,9 @@ const fromRequest = Effect.fn("MetaResponses.fromRequest")(function* (request: L { ...request, messages: request.messages.map((message) => - message.role === "system" + message.role !== "assistant" ? message - : Message.make({ + : new AssistantMessage({ ...message, content: message.content.map((part) => { if ( diff --git a/packages/ai/src/protocols/mistral-chat.ts b/packages/ai/src/protocols/mistral-chat.ts index 4c1fbfbab8f..6e9975c2662 100644 --- a/packages/ai/src/protocols/mistral-chat.ts +++ b/packages/ai/src/protocols/mistral-chat.ts @@ -17,6 +17,8 @@ import { type MediaPart, type ToolCallPart, type ToolDefinition, + type ToolMessage, + type UserMessage, } from "../schema/index.js" import { classifyProviderFailure } from "../provider-error.js" import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js" @@ -227,7 +229,7 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart) { return yield* ProviderShared.invalidRequest(`Mistral Chat does not support media type ${part.media.mediaType}`) }) -const lowerUser = Effect.fnUntraced(function* (message: LLMRequest["messages"][number]) { +const lowerUser = Effect.fnUntraced(function* (message: UserMessage) { const content: MistralUserContent[] = [] for (const part of message.content) { if (part.type === "text") { @@ -238,7 +240,6 @@ const lowerUser = Effect.fnUntraced(function* (message: LLMRequest["messages"][n content.push(yield* lowerMedia(part)) continue } - return yield* ProviderShared.unsupportedContent("Mistral Chat", "user", ["text", "media"]) } if (content.every((part) => part.type === "text")) return { role: "user" as const, content: content.map((part) => part.text).join("") } @@ -285,14 +286,9 @@ const lowerAssistant = Effect.fnUntraced(function* ( } }) -const lowerToolResults = Effect.fnUntraced(function* ( - message: LLMRequest["messages"][number], - normalizeID: (id: string) => string, -) { +const lowerToolResults = Effect.fnUntraced(function* (message: ToolMessage, normalizeID: (id: string) => string) { const output: MistralMessage[] = [] for (const part of message.content) { - if (part.type !== "tool-result") - return yield* ProviderShared.unsupportedContent("Mistral Chat", "tool", ["tool-result"]) if (part.result.type !== "content") { output.push({ role: "tool", diff --git a/packages/ai/src/protocols/open-responses.ts b/packages/ai/src/protocols/open-responses.ts index fb81be7eea7..5172546ecaa 100644 --- a/packages/ai/src/protocols/open-responses.ts +++ b/packages/ai/src/protocols/open-responses.ts @@ -16,6 +16,7 @@ import { type ToolCallPart, type ToolDefinition, type ToolResultPart, + type UserMessage, } from "../schema/index.js" import type { Media } from "../media.js" import { @@ -606,13 +607,12 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMReq }) const lowerUserContent = Effect.fnUntraced(function* ( - part: LLMRequest["messages"][number]["content"][number], + part: UserMessage["content"][number], request: LLMRequest, adapter: ProviderAdapter, ) { if (part.type === "text") return { type: "input_text" as const, text: part.text } - if (part.type === "media") return yield* lowerMedia(part, request, adapter) - return yield* ProviderShared.unsupportedContent(adapter.name, "user", ["text", "media"]) + return yield* lowerMedia(part, request, adapter) }) // Tool results may carry structured text, images, and files. Keep media as provider-native @@ -794,8 +794,6 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest, adapter: } for (const part of message.content) { - if (!ProviderShared.supportsContent(part, ["tool-result"])) - return yield* ProviderShared.unsupportedContent(adapter.name, "tool", ["tool-result"]) input.push({ type: customTool(part) ? "custom_tool_call_output" : "function_call_output", call_id: part.id, diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index 5f33bfc4b97..cd76637892c 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -509,7 +509,6 @@ const lowerUserMessage = Effect.fnUntraced(function* (message: UserMessage, opti content.push(yield* lowerMedia(part)) continue } - return yield* ProviderShared.unsupportedContent("OpenAI Chat", "user", ["text", "media"]) } if (content.every((part) => part.type === "text" && part.cache_control === undefined)) return { @@ -580,8 +579,6 @@ const lowerToolMessages = Effect.fnUntraced(function* (message: ToolMessage, opt const messages: OpenAIChatMessage[] = [] const attachments: OpenAIChatUserContent[] = [] for (const part of message.content) { - if (!ProviderShared.supportsContent(part, ["tool-result"])) - return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"]) if (part.result.type !== "content") { messages.push( toolMessage( diff --git a/packages/ai/src/protocols/shared.ts b/packages/ai/src/protocols/shared.ts index 4bd9c9dc40a..6913ab77168 100644 --- a/packages/ai/src/protocols/shared.ts +++ b/packages/ai/src/protocols/shared.ts @@ -7,15 +7,18 @@ import { InvalidRequestError, UnsupportedOperationError, AIError, + AssistantMessage, type LLMRequest, Message, ToolDefinition, + ToolMessage, type ContentPart, type MediaPart, type OpenString, type ProviderID, type RequestEncoding, type TextPart, + type ToolCallPart, type ToolEntry, type ToolResultPart, } from "../schema/index.js" @@ -348,14 +351,15 @@ export const flattenTools = (tools: ReadonlyArray, path: ReadonlyArra export const flattenToolRequest = (request: LLMRequest) => { const messages = request.messages.map((message) => { - if (message.role === "system") return message - const content = message.content.map((part) => { - if ((part.type !== "tool-call" && part.type !== "tool-result") || part.namespace === undefined) return part - return { ...part, name: `${part.namespace}_${part.name}`, namespace: undefined } - }) - return content.every((part, index) => part === message.content[index]) - ? message - : Message.make({ ...message, content }) + if (message.role === "assistant") { + const content = message.content.map((part) => + part.type === "tool-call" || part.type === "tool-result" ? flattenToolName(part) : part, + ) + return sameParts(content, message.content) ? message : new AssistantMessage({ ...message, content }) + } + if (message.role !== "tool") return message + const content = message.content.map(flattenToolName) + return sameParts(content, message.content) ? message : new ToolMessage({ ...message, content }) }) return { tools: flattenTools(request.tools), @@ -365,6 +369,12 @@ export const flattenToolRequest = (request: LLMRequest) => { } } +const flattenToolName = (part: Part): Part => + part.namespace === undefined ? part : { ...part, name: `${part.namespace}_${part.name}`, namespace: undefined } + +const sameParts = (next: ReadonlyArray, previous: ReadonlyArray) => + next.every((part, index) => part === previous[index]) + export const matchToolChoice = ( route: string, toolChoice: NonNullable, diff --git a/packages/ai/src/protocols/utils/responses-compaction.ts b/packages/ai/src/protocols/utils/responses-compaction.ts index 439c1c860f6..d276e5246ec 100644 --- a/packages/ai/src/protocols/utils/responses-compaction.ts +++ b/packages/ai/src/protocols/utils/responses-compaction.ts @@ -6,7 +6,8 @@ import { CompactionResponse, type LLMRequest, Message, - type ContentPart, + type MediaPart, + type TextPart, mergeJsonRecords, } from "../../schema/index.js" import type { CompactOperation } from "../../route/client.js" @@ -174,7 +175,7 @@ function toMessage(item: (typeof Response.Type.output)[number], model: LLMReques return Message.make({ role: item.role, providerMetadata: { [key]: { itemId: item.id, type: item.type, status: item.status, phase: item.phase } }, - content: item.content.map((part): ContentPart => { + content: item.content.map((part): TextPart | MediaPart => { if (part.type === "input_text" || part.type === "output_text") return { type: "text", text: part.text } if (part.type === "input_image") return { diff --git a/packages/ai/src/schema/events.ts b/packages/ai/src/schema/events.ts index 7d85e19332a..725e1f73a01 100644 --- a/packages/ai/src/schema/events.ts +++ b/packages/ai/src/schema/events.ts @@ -9,7 +9,6 @@ import { ToolOutput, ToolResultPart, ToolResultValue, - type ContentPart, } from "./messages.js" import { ProviderMetadata } from "./options.js" import { Media } from "../media.js" @@ -486,18 +485,20 @@ const appendEvent = (state: ResponseState, event: LLMEvent): ResponseState => { } type ContentKind = "text" | "reasoning" +type AssistantPart = AssistantMessage["content"][number] -const contentPart = (type: ContentKind, text: string, providerMetadata: ProviderMetadata | undefined): ContentPart => +const contentPart = (type: ContentKind, text: string, providerMetadata: ProviderMetadata | undefined): AssistantPart => providerMetadata === undefined ? { type, text } : { type, text, providerMetadata } -const contentWith = (state: ResponseState, content: ReadonlyArray): ResponseState => ({ +const contentWith = (state: ResponseState, content: ReadonlyArray): ResponseState => ({ ...state, message: Message.assistant(content), }) -const appendContent = (state: ResponseState, part: ContentPart) => contentWith(state, [...state.message.content, part]) +const appendContent = (state: ResponseState, part: AssistantPart) => + contentWith(state, [...state.message.content, part]) -const replaceContent = (state: ResponseState, index: number, part: ContentPart) => +const replaceContent = (state: ResponseState, index: number, part: AssistantPart) => contentWith( state, state.message.content.map((item, itemIndex) => (itemIndex === index ? part : item)), @@ -600,7 +601,7 @@ const reduceToolInputEnd = (state: ResponseState, event: ToolInputEnd): Response } } -const toolCallContent = (event: ToolCall): ContentPart => +const toolCallContent = (event: ToolCall): AssistantPart => ToolCallPart.make({ id: event.id, name: event.name, @@ -610,7 +611,7 @@ const toolCallContent = (event: ToolCall): ContentPart => ...(event.providerMetadata === undefined ? {} : { providerMetadata: event.providerMetadata }), }) -const toolResultContent = (event: ToolResult): ContentPart => +const toolResultContent = (event: ToolResult): AssistantPart => ToolResultPart.make({ id: event.id, name: event.name, diff --git a/packages/ai/src/schema/messages.ts b/packages/ai/src/schema/messages.ts index 0b194f516e7..f328006564a 100644 --- a/packages/ai/src/schema/messages.ts +++ b/packages/ai/src/schema/messages.ts @@ -246,21 +246,25 @@ export class SystemMessage extends Schema.Class("LLM.SystemMessag export class UserMessage extends Schema.Class("LLM.UserMessage")({ id: Schema.optional(Schema.String), role: Schema.Literal("user"), - content: Schema.Array(ContentPart), + content: Schema.Array(Schema.Union([TextPart, MediaPart]).pipe(Schema.toTaggedUnion("type"))), providerMetadata: Schema.optional(ProviderMetadata), }) {} export class AssistantMessage extends Schema.Class("LLM.AssistantMessage")({ id: Schema.optional(Schema.String), role: Schema.Literal("assistant"), - content: Schema.Array(ContentPart), + content: Schema.Array( + Schema.Union([TextPart, ReasoningPart, ToolCallPart, ToolResultPart, MediaPart, CompactionPart]).pipe( + Schema.toTaggedUnion("type"), + ), + ), providerMetadata: Schema.optional(ProviderMetadata), }) {} export class ToolMessage extends Schema.Class("LLM.ToolMessage")({ id: Schema.optional(Schema.String), role: Schema.Literal("tool"), - content: Schema.Array(ContentPart), + content: Schema.Array(ToolResultPart), }) {} export type Message = SystemMessage | UserMessage | AssistantMessage | ToolMessage @@ -268,16 +272,26 @@ export type Message = SystemMessage | UserMessage | AssistantMessage | ToolMessa export type ContentPartInput = Exclude | MediaPartInput export namespace Message { - export type ContentInput = string | ContentPartInput | ReadonlyArray - export type SystemContentInput = string | TextPart | ReadonlyArray + export type ContentInput = + | string + | Part + | ReadonlyArray + export type SystemContentInput = ContentInput + export type UserContentInput = ContentInput + export type AssistantContentInput = ContentInput< + Exclude | MediaPartInput + > type RoleInput = Omit & { readonly content: Content } export type SystemInput = RoleInput< ConstructorParameters[0], - string | SystemMessage["content"][number] | SystemMessage["content"] + ContentInput + > + export type UserInput = RoleInput[0], UserContentInput> + export type AssistantInput = RoleInput[0], AssistantContentInput> + export type ToolInput = RoleInput< + ConstructorParameters[0], + ToolResultPart | ReadonlyArray > - export type UserInput = RoleInput[0], ContentInput> - export type AssistantInput = RoleInput[0], ContentInput> - export type ToolInput = RoleInput[0], ContentInput> export type Input = SystemInput | UserInput | AssistantInput | ToolInput } @@ -296,8 +310,12 @@ const content = ( input: string | Part | ReadonlyArray, ): Array => (typeof input === "string" ? [text(input)] : isParts(input) ? [...input] : [input]) -const contentWithMedia = (input: Message.ContentInput): ContentPart[] => - content(input).map((part) => (part.type === "media" ? mediaPart(part) : part)) +function contentWithMedia(input: Message.UserContentInput): UserMessage["content"] +function contentWithMedia(input: Message.AssistantContentInput): AssistantMessage["content"] +function contentWithMedia(input: Message.ContentInput): ReadonlyArray +function contentWithMedia(input: Message.ContentInput): ReadonlyArray { + return content(input).map((part) => (part.type === "media" ? mediaPart(part) : part)) +} function make(input: SystemMessage | Message.SystemInput): SystemMessage function make(input: UserMessage | Message.UserInput): UserMessage @@ -319,7 +337,7 @@ function make(input: Message | Message.Input): Message { case "tool": return input instanceof ToolMessage ? input - : new ToolMessage({ ...input, content: contentWithMedia(input.content) }) + : new ToolMessage({ ...input, content: isParts(input.content) ? [...input.content] : [input.content] }) } // Untyped callers can still pass an unknown role; reject it the way the schema does. return Schema.decodeUnknownSync(messageSchema)(input) @@ -332,8 +350,8 @@ export const Message = withStatics(messageSchema, { mediaPart({ type: "media", media, ...options }), content: contentWithMedia, make, - user: (input: Message.ContentInput) => make({ role: "user", content: input }), - assistant: (input: Message.ContentInput) => make({ role: "assistant", content: input }), + user: (input: Message.UserContentInput) => make({ role: "user", content: input }), + assistant: (input: Message.AssistantContentInput) => make({ role: "assistant", content: input }), /** * Add an operator-authored instruction at this chronological point in the * conversation. This is distinct from the initial `LLMRequest.system` diff --git a/packages/ai/test/adapter.test.ts b/packages/ai/test/adapter.test.ts index 0eb16843795..c84e35c90b7 100644 --- a/packages/ai/test/adapter.test.ts +++ b/packages/ai/test/adapter.test.ts @@ -54,7 +54,7 @@ const fakeProtocol = Protocol.make({ Effect.succeed({ body: [ ...request.messages - .flatMap((message) => message.content) + .flatMap((message) => [...message.content]) .filter((part) => part.type === "text") .map((part) => part.text), ...request.tools.map((tool) => `tool:${tool.name}:${tool.description}`), diff --git a/packages/ai/test/continuation-scenarios.ts b/packages/ai/test/continuation-scenarios.ts index 7203a11b266..538cf09b3a5 100644 --- a/packages/ai/test/continuation-scenarios.ts +++ b/packages/ai/test/continuation-scenarios.ts @@ -1,12 +1,13 @@ import { ai, - type ContentPart, + type AssistantMessage, type LanguageModel, Media, Message, ToolCallPart, ToolDefinition, ToolResultPart, + type UserMessage, } from "../src/effect.js" export const basicContinuation = ["system", "user-text", "assistant-text", "user-follow-up"] as const @@ -55,8 +56,8 @@ export function continuationRequest(input: { }) { const features = new Set(input.features) const messages = [] - const firstUser: ContentPart[] = [] - const firstAssistant: ContentPart[] = [] + const firstUser: Array = [] + const firstAssistant: Array = [] if (features.has("user-text")) firstUser.push({ type: "text", text: "What is shown here?" }) if (features.has("user-image")) diff --git a/packages/ai/test/lib/tool-runtime.ts b/packages/ai/test/lib/tool-runtime.ts index b15a9cd3c0c..2fe6918dc35 100644 --- a/packages/ai/test/lib/tool-runtime.ts +++ b/packages/ai/test/lib/tool-runtime.ts @@ -4,7 +4,7 @@ import { LLMEvent, type LLMRequest, Message, - type ContentPart, + type AssistantMessage, type ProviderMetadata, type ToolCallPart, ToolResultPart, @@ -82,7 +82,7 @@ const indexStep = (event: LLMEvent, index: number): LLMEvent => { } const stepState = (events: ReadonlyArray) => { - const assistantContent: ContentPart[] = [] + const assistantContent: Array = [] const toolCalls: ToolCallPart[] = [] let reason: Extract["reason"] = { normalized: "unknown" } let usage: Usage | undefined @@ -116,7 +116,7 @@ const stepState = (events: ReadonlyArray) => { } const appendText = ( - content: ContentPart[], + content: Array, type: "text" | "reasoning", text: string, providerMetadata?: ProviderMetadata, diff --git a/packages/ai/test/recorded-scenarios.ts b/packages/ai/test/recorded-scenarios.ts index 1031f99a5d9..1ca0ca4a26d 100644 --- a/packages/ai/test/recorded-scenarios.ts +++ b/packages/ai/test/recorded-scenarios.ts @@ -2,7 +2,7 @@ import { expect } from "bun:test" import { Effect, Schema, Stream } from "effect" import { ai, - type ContentPart, + type AssistantMessage, type FinishReason, type LanguageModel, type LLM, @@ -154,7 +154,7 @@ const encryptedReasoningOptions = { type AssistantTextExpectation = string | RegExp -type UserStep = { readonly type: "user"; readonly content: Message.ContentInput } +type UserStep = { readonly type: "user"; readonly content: Message.UserContentInput } type AssistantStep = { readonly type: "assistant" readonly text?: AssistantTextExpectation @@ -171,7 +171,7 @@ type AssistantStep = { } type ConversationStep = UserStep | AssistantStep -const user = (content: Message.ContentInput): ConversationStep => ({ type: "user", content }) +const user = (content: Message.UserContentInput): ConversationStep => ({ type: "user", content }) const assistant = { expectText: ( @@ -213,7 +213,7 @@ const assertAssistantToolCall = (response: LLMResponse, expected: NonNullable { - const content: ContentPart[] = [] + const content: Array = [] if (step.reasoning === "openai-encrypted") { const reasoning = response.events.find( (event): event is Extract => diff --git a/packages/core/src/plugin/provider/claude-agent/session.ts b/packages/core/src/plugin/provider/claude-agent/session.ts index f276eab07bd..c054b06f416 100644 --- a/packages/core/src/plugin/provider/claude-agent/session.ts +++ b/packages/core/src/plugin/provider/claude-agent/session.ts @@ -1,5 +1,6 @@ export * as ClaudeSessionHooks from "./session.js" +import { AssistantMessage, ToolMessage, type ToolResultPart } from "@opencode/ai/effect" import type { Context } from "@opencode/plugin/effect/plugin" import { Effect, Schema } from "effect" import { RequestKind } from "./language-model/request-kind.js" @@ -8,6 +9,16 @@ import { ClaudeProvider } from "./provider.js" // OpenCode records stopped tools (including a plain permission rejection) with this structured error. const isAborted = Schema.is(Schema.Struct({ error: Schema.Struct({ type: Schema.Literal("aborted") }) })) +const flagToolError = (part: ToolResultPart): ToolResultPart => { + if (part.result.type !== "error") return part + const flags = { + ...part.providerMetadata?.[ClaudeProvider.ID], + toolError: true, + toolAborted: isAborted(part.result.value), + } + return { ...part, providerMetadata: { ...part.providerMetadata, [ClaudeProvider.ID]: flags } } +} + export const register = Effect.fn("ClaudeSessionHooks.register")(function* (ctx: Context) { yield* ctx.session.hook( "model.request", @@ -29,17 +40,11 @@ export const register = Effect.fn("ClaudeSessionHooks.register")(function* (ctx: delete event.options.maxTokens // Preserve tool error status before the AI SDK bridge flattens error results to plain text. event.messages = event.messages.map((message) => { - if (message.role === "system") return message - const content = message.content.map((part) => { - if (part.type !== "tool-result" || part.result.type !== "error") return part - const flags = { - ...part.providerMetadata?.[ClaudeProvider.ID], - toolError: true, - toolAborted: isAborted(part.result.value), - } - return { ...part, providerMetadata: { ...part.providerMetadata, [ClaudeProvider.ID]: flags } } - }) - return { ...message, content } + if (message.role === "tool") + return new ToolMessage({ ...message, content: message.content.map(flagToolError) }) + if (message.role !== "assistant") return message + const content = message.content.map((part) => (part.type === "tool-result" ? flagToolError(part) : part)) + return new AssistantMessage({ ...message, content }) }) }), { providerID: ClaudeProvider.ID }, diff --git a/packages/core/src/session/model-request.ts b/packages/core/src/session/model-request.ts index eaf5ce8bf3e..3ec29fd9abb 100644 --- a/packages/core/src/session/model-request.ts +++ b/packages/core/src/session/model-request.ts @@ -2,14 +2,20 @@ export * as SessionModelRequest from "./model-request.js" import { ai, + AssistantMessage, + type ContentPart, GenerationOptions, type GenerationOptionsFields, LanguageModel, type LLM, type LLMRequest, type Media, + type MediaPart, Message, SystemPart, + ToolMessage, + type ToolResultPart, + UserMessage, } from "@opencode/ai/effect" import type { StreamOptions } from "@opencode/ai/route" import type { @@ -161,29 +167,46 @@ const mediaBytes = (media: Media.Asset) => { const replaceMedia = ( messages: LLMRequest["messages"], replace: (media: { mime: string; name: string | undefined; bytes: () => number }) => string | undefined, -) => - messages.map((message) => { - if (message.role === "system") return message - const content = message.content.map((part) => { - if (part.type === "media") { - const text = replace({ mime: part.media.mediaType, name: part.filename, bytes: () => mediaBytes(part.media) }) - return text === undefined ? part : Message.text(text) - } - if (part.type !== "tool-result" || part.result.type !== "content") return part - const result = part.result - const value = result.value.map((item): Content => { - if (item.type !== "file") return item - const text = replace({ mime: item.mime, name: item.name, bytes: () => item.uri.length }) - return text === undefined ? item : { type: "text", text } - }) - return value.every((item, index) => item === result.value[index]) - ? part - : { ...part, result: { ...result, value } } +) => { + const media = (part: MediaPart) => { + const text = replace({ mime: part.media.mediaType, name: part.filename, bytes: () => mediaBytes(part.media) }) + return text === undefined ? part : Message.text(text) + } + const toolResult = (part: ToolResultPart) => { + if (part.result.type !== "content") return part + const result = part.result + const value = result.value.map((item): Content => { + if (item.type !== "file") return item + const text = replace({ mime: item.mime, name: item.name, bytes: () => item.uri.length }) + return text === undefined ? item : { type: "text", text } }) - return content.every((part, index) => part === message.content[index]) - ? message - : Message.make({ ...message, content }) + return value.every((item, index) => item === result.value[index]) ? part : { ...part, result: { ...result, value } } + } + return messages.map((message) => { + switch (message.role) { + case "system": + return message + case "user": { + const content = message.content.map((part) => (part.type === "media" ? media(part) : part)) + return sameParts(content, message.content) ? message : new UserMessage({ ...message, content }) + } + case "assistant": { + const content = message.content.map((part) => { + if (part.type === "media") return media(part) + return part.type === "tool-result" ? toolResult(part) : part + }) + return sameParts(content, message.content) ? message : new AssistantMessage({ ...message, content }) + } + case "tool": { + const content = message.content.map(toolResult) + return sameParts(content, message.content) ? message : new ToolMessage({ ...message, content }) + } + } }) +} + +const sameParts = (next: ReadonlyArray, previous: ReadonlyArray) => + next.every((part, index) => part === previous[index]) export const unsupportedParts = ( messages: LLMRequest["messages"], diff --git a/packages/core/src/session/runner/to-llm-message.ts b/packages/core/src/session/runner/to-llm-message.ts index 86dd40f36f6..47c6fc4a14b 100644 --- a/packages/core/src/session/runner/to-llm-message.ts +++ b/packages/core/src/session/runner/to-llm-message.ts @@ -4,8 +4,10 @@ import { ReasoningEfforts, ToolCallPart, ToolResultPart, - type ContentPart, + type AssistantMessage, + type MediaPart, type ProviderMetadata, + type TextPart, } from "@opencode/ai/effect" import type { Model } from "@opencode/schema/model" import { Option, Schema } from "effect" @@ -17,7 +19,7 @@ import type { FileAttachment } from "@opencode/schema/prompt" const imageMimes = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]) -const media = (file: FileAttachment): ContentPart => ({ +const media = (file: FileAttachment): MediaPart => ({ type: "media", media: Media.base64(file.data, file.mime), filename: file.name, @@ -34,7 +36,7 @@ const attachmentLocation = (file: FileAttachment) => { } } -const textAttachment = (file: FileAttachment): ContentPart => ({ +const textAttachment = (file: FileAttachment): TextPart => ({ type: "text", text: `\n\n${[ `Attached file: ${file.name ?? (file.source.type === "uri" ? file.source.uri : "inline attachment")}`, @@ -46,7 +48,7 @@ const textAttachment = (file: FileAttachment): ContentPart => ({ .join("\n")}`, }) -const directoryAttachment = (file: FileAttachment): ContentPart => ({ +const directoryAttachment = (file: FileAttachment): TextPart => ({ type: "text", text: `\n\n${[ `Attached directory: ${attachmentLocation(file) ?? file.name ?? (file.source.type === "uri" ? file.source.uri : "directory")}`, @@ -58,7 +60,7 @@ const directoryAttachment = (file: FileAttachment): ContentPart => ({ .join("\n")}`, }) -const attachmentContent = (file: FileAttachment): ContentPart[] => { +const attachmentContent = (file: FileAttachment): Array => { if (file.mime === "text/plain") return [textAttachment(file)] if (file.mime === "application/x-directory") return [directoryAttachment(file)] if (imageMimes.has(file.mime) || file.mime === "application/pdf") { @@ -100,7 +102,7 @@ const toolInput = (tool: SessionMessage.AssistantTool) => ? Option.getOrElse(decodeToolInput(tool.state.input), () => tool.state.input) : tool.state.input -const toolCall = (tool: SessionMessage.AssistantTool, providerMetadata: ProviderMetadata | undefined): ContentPart => +const toolCall = (tool: SessionMessage.AssistantTool, providerMetadata: ProviderMetadata | undefined): ToolCallPart => ToolCallPart.make({ id: tool.id, name: tool.name, @@ -144,7 +146,7 @@ const assistant = (message: SessionMessage.Assistant, model: Model.Ref, provider const reasoningInterrupted = message.content.some( (item) => item.type === "reasoning" && item.time !== undefined && item.time.completed === undefined, ) - const content = message.content.flatMap((item): ContentPart[] => { + const content = message.content.flatMap((item): Array => { if (item.type === "text") return [ { diff --git a/packages/core/test/session-model-request.test.ts b/packages/core/test/session-model-request.test.ts index b6c105fc642..a39ed2a1c9b 100644 --- a/packages/core/test/session-model-request.test.ts +++ b/packages/core/test/session-model-request.test.ts @@ -149,14 +149,15 @@ describe("SessionModelRequest.unsupportedParts", () => { ], }, }) - const unsupported = Message.make({ role: "tool", content: [tool, Message.text("caption")] }) + const untouched = ToolResultPart.make({ id: "call_3", name: "read", result: "done" }) + const unsupported = Message.make({ role: "tool", content: [tool, untouched] }) const result = unsupportedParts([text, supported, unsupported], capabilities(["text"])) expect(result[0]).toBe(text) expect(result[1]).toBe(supported) expect(result[2]).not.toBe(unsupported) expect(result[2]).toBeInstanceOf(ToolMessage) - expect(result[2]?.content[1]).toEqual(Message.text("caption")) + expect(result[2]?.content[1]).toEqual(untouched) expect(result[2]?.content[0]).toMatchObject({ type: "tool-result", result: { diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index f5435eb5c9b..669988427da 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -1791,7 +1791,7 @@ describe("SessionRunnerLLM", () => { expect( s.requests .at(-1) - ?.messages.flatMap((message) => message.content) + ?.messages.flatMap((message) => [...message.content]) .some((part) => part.type === "compaction"), ).toBe(false) expect(s.requests.at(-1)?.messages[0]?.content).toEqual([Message.text("Original request")])