fix(ai): place Anthropic system updates before the next assistant turn (#52568)

packages/ai/src/protocols/anthropic-messages.ts

@@ -584,10 +584,7 @@ const serverToolResultType = (name: string): AnthropicServerToolResultType | und
   return undefined
 }
 
-const lowerServerToolResult = Effect.fnUntraced(function* (
-  part: ToolResultPart,
-  providerMetadataKey: string,
-) {
+const lowerServerToolResult = Effect.fnUntraced(function* (part: ToolResultPart, providerMetadataKey: string) {
   const wireType = serverToolResultType(part.name)
   if (!wireType)
     return yield* invalid(`Anthropic Messages does not know how to round-trip server tool result for ${part.name}`)
@@ -657,10 +654,7 @@ const citationsFromMetadata = (metadata: MediaPart["metadata"]): AnthropicDocume
 
 const isHttpUrl = (value: string) => /^https?:\/\//i.test(value.trim())
 
-const lowerMedia = Effect.fnUntraced(function* (
-  part: MediaPart,
-  breakpoints?: Cache.Breakpoints,
-) {
+const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, breakpoints?: Cache.Breakpoints) {
   const mime = part.media.mediaType.toLowerCase()
   const cacheControlValue = breakpoints ? cacheControl(breakpoints, part.cache) : undefined
   const fileId = fileIdFromMetadata(part.metadata)
@@ -804,9 +798,6 @@ const requireThinkingSignature = (request: LLMRequest) => {
   return true
 }
 
-// Mid-conversation system messages became available with Opus 4.8 and version
-// 5 of the other supported Claude families. Treat later family versions as
-// compatible without assuming that every Anthropic Messages model is Claude.
 // Opus 4.8 and every Claude 5 model accept mid-conversation system messages; later versions inherit support.
 const supportsNativeSystemUpdates = (request: LLMRequest) => {
   const version = claudeVersion(String(request.model.id))
@@ -815,25 +806,9 @@ const supportsNativeSystemUpdates = (request: LLMRequest) => {
   return version.major >= 5
 }
 
-const endsInServerToolUse = (message: LLMRequest["messages"][number]) => {
-  const last = message.content.at(-1)
-  return message.role === "assistant" && last?.type === "tool-call" && last.providerExecuted === true
-}
-
-const canUseNativeSystemUpdate = (request: LLMRequest, index: number) => {
-  const previous = request.messages[index - 1]
-  const next = request.messages[index + 1]
-  // Vertex currently rejects/404s for a system message after local tool results,
-  // so fold it into the user tool-result turn across continuations and history.
-  if (request.model.route.id === "google-vertex-messages" && previous?.role === "tool") return false
-  return (
-    previous !== undefined &&
-    previous.role !== "system" &&
-    (previous.role === "user" || previous.role === "tool" || endsInServerToolUse(previous)) &&
-    next?.role !== "system" &&
-    (next === undefined || next.role === "assistant")
-  )
-}
+// Native system messages must follow a user turn (tool results count) or a paused server-tool turn.
+const acceptsNativeSystemAfter = (message: AnthropicMessage | undefined) =>
+  message?.role === "user" || (message?.role === "assistant" && message.content.at(-1)?.type === "server_tool_use")
 
 const splitsLocalToolResults = (messages: LLMRequest["messages"], index: number) => {
   const pending = new Set<string>()
@@ -862,12 +837,34 @@ const lowerNativeSystemUpdate = Effect.fnUntraced(function* (
   }
 })
 
-const lowerMessages = Effect.fnUntraced(function* (
-  request: LLMRequest,
+const lowerWrappedSystemUpdate = Effect.fnUntraced(function* (
+  message: LLMRequest["messages"][number],
   breakpoints: Cache.Breakpoints,
 ) {
+  const part = yield* ProviderShared.wrappedSystemUpdate("Anthropic Messages", message)
+  return { type: "text" as const, text: part.text, cache_control: cacheControl(breakpoints, part.cache) }
+})
+
+const appendToUserTurn = (messages: AnthropicMessage[], block: AnthropicUserBlock) => {
+  const last = messages.at(-1)
+  if (last?.role === "user") messages[messages.length - 1] = { role: "user", content: [...last.content, block] }
+  else messages.push({ role: "user", content: [block] })
+}
+
+const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest, breakpoints: Cache.Breakpoints) {
   const messages: AnthropicMessage[] = []
   const providerMetadataKey = request.model.route.providerMetadataKey ?? String(request.model.provider)
+  // Text updates stay where they are unless a user turn follows them; then they move after the latest
+  // user turn, the nearest spot where Anthropic accepts a native system message.
+  const holdUpdates = supportsNativeSystemUpdates(request)
+  const held: Array<LLMRequest["messages"][number]> = []
+  const releaseHeld = Effect.fnUntraced(function* () {
+    const native = acceptsNativeSystemAfter(messages.findLast((message) => message.role !== "system"))
+    for (const update of held.splice(0)) {
+      if (native) messages.push(yield* lowerNativeSystemUpdate(update, breakpoints))
+      else appendToUserTurn(messages, yield* lowerWrappedSystemUpdate(update, breakpoints))
+    }
+  })
 
   for (const [index, message] of request.messages.entries()) {
     if (message.role === "system") {
@@ -879,16 +876,8 @@ const lowerMessages = Effect.fnUntraced(function* (
       }
       if (splitsLocalToolResults(request.messages, index))
         return yield* invalid("Anthropic Messages system updates cannot split a local tool call from its tool result")
-      if (supportsNativeSystemUpdates(request) && canUseNativeSystemUpdate(request, index)) {
-        messages.push(yield* lowerNativeSystemUpdate(message, breakpoints))
-        continue
-      }
-      const part = yield* ProviderShared.wrappedSystemUpdate("Anthropic Messages", message)
-      const block = { type: "text" as const, text: part.text, cache_control: cacheControl(breakpoints, part.cache) }
-      const previous = messages.at(-1)
-      if (previous?.role === "user")
-        messages[messages.length - 1] = { role: "user", content: [...previous.content, block] }
-      else messages.push({ role: "user", content: [block] })
+      if (holdUpdates) held.push(message)
+      else appendToUserTurn(messages, yield* lowerWrappedSystemUpdate(message, breakpoints))
       continue
     }
 
@@ -964,7 +953,9 @@ const lowerMessages = Effect.fnUntraced(function* (
           `Anthropic Messages assistant messages only support text, reasoning, and tool-call content for now`,
         )
       }
-      if (content.length > 0) messages.push({ role: "assistant", content })
+      if (content.length === 0) continue
+      yield* releaseHeld()
+      messages.push({ role: "assistant", content })
       continue
     }
 
@@ -985,6 +976,7 @@ const lowerMessages = Effect.fnUntraced(function* (
       messages[messages.length - 1] = { role: "user", content: [...previous.content, ...content] }
     else messages.push({ role: "user", content })
   }
+  yield* releaseHeld()
 
   return messages
 })

packages/ai/test/effort-updates.test.ts

@@ -171,6 +171,33 @@ describe("Anthropic Messages effort updates", () => {
     }),
   )
 
+  it.effect("releases a held system update next to an effort marker as one valid section", () =>
+    Effect.gen(function* () {
+      const prepared = yield* compileRequest(
+        LLM.request({
+          model: opus5,
+          messages: [
+            Message.user("Fix it."),
+            Message.assistant("Done."),
+            lowFromHigh,
+            Message.system("Update."),
+            Message.user("Next."),
+          ],
+          providerOptions: { effort: "low" },
+          cache: "none",
+        }),
+      )
+
+      expect(prepared.body.messages).toEqual([
+        { role: "user", content: [{ type: "text", text: "Fix it." }] },
+        { role: "assistant", content: [{ type: "text", text: "Done." }] },
+        { role: "system", content: [], output_config: { effort: "low" } },
+        { role: "user", content: [{ type: "text", text: "Next." }] },
+        { role: "system", content: [{ type: "text", text: "Update.", cache_control: undefined }] },
+      ])
+    }),
+  )
+
   it.effect("falls back to a plain top-level effort when history drifted from the current effort", () =>
     Effect.gen(function* () {
       const drifted = yield* compileRequest(

packages/ai/test/provider/anthropic-messages.test.ts

@@ -431,42 +431,115 @@ describe("Anthropic Messages route", () => {
         (yield* compileRequest(
           LLM.request({
             model: opus48,
-            messages: [Message.user("Before."), Message.system("One."), Message.system("Two.")],
+            messages: [
+              Message.user("Start."),
+              Message.assistant("One."),
+              Message.system("Update."),
+              Message.assistant("Two."),
+            ],
             cache: "none",
           }),
         )).body.messages,
       ).toEqual([
-        {
-          role: "user",
-          content: [
-            { type: "text", text: "Before." },
-            { type: "text", text: "<system-update>\nOne.\n</system-update>" },
-            { type: "text", text: "<system-update>\nTwo.\n</system-update>" },
-          ],
-        },
+        { role: "user", content: [{ type: "text", text: "Start." }] },
+        { role: "assistant", content: [{ type: "text", text: "One." }] },
+        { role: "user", content: [{ type: "text", text: "<system-update>\nUpdate.\n</system-update>" }] },
+        { role: "assistant", content: [{ type: "text", text: "Two." }] },
       ])
     }),
   )
 
-  it.effect("keeps a terminal Vertex system update in the tool-result turn", () =>
+  it.effect("moves system updates to the next assistant turn and sends consecutive updates together", () =>
+    Effect.gen(function* () {
+      const lower = (messages: ReadonlyArray<Message>) =>
+        compileRequest(LLM.request({ model: opus48, messages: [...messages], cache: "none" })).pipe(
+          Effect.map((prepared) => prepared.body.messages),
+        )
+      const system = (text: string) => ({ role: "system", content: [{ type: "text", text, cache_control: undefined }] })
+      const user = (text: string) => ({ role: "user", content: [{ type: "text", text }] })
+      const assistant = (text: string) => ({ role: "assistant", content: [{ type: "text", text }] })
+
+      expect(
+        yield* lower([
+          Message.user("Fix it."),
+          Message.assistant("Done."),
+          Message.system("Update."),
+          Message.user("Next."),
+        ]),
+      ).toEqual([user("Fix it."), assistant("Done."), user("Next."), system("Update.")])
+      expect(yield* lower([Message.user("Before."), Message.system("One."), Message.system("Two.")])).toEqual([
+        user("Before."),
+        system("One."),
+        system("Two."),
+      ])
+      expect(
+        yield* lower([
+          Message.user("Fix it."),
+          Message.assistant("Done."),
+          Message.system("One."),
+          Message.user("Next."),
+          Message.system("Two."),
+          Message.assistant("After."),
+        ]),
+      ).toEqual([
+        user("Fix it."),
+        assistant("Done."),
+        user("Next."),
+        system("One."),
+        system("Two."),
+        assistant("After."),
+      ])
+      expect(
+        yield* lower([
+          Message.user("Use the tool."),
+          Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: {} })]),
+          Message.tool({ id: "call_1", name: "lookup", result: "Done." }),
+          Message.system("Update."),
+          Message.user("Also check tests."),
+        ]),
+      ).toEqual([
+        user("Use the tool."),
+        { role: "assistant", content: [{ type: "tool_use", id: "call_1", name: "lookup", input: {} }] },
+        { role: "user", content: [{ type: "tool_result", tool_use_id: "call_1", content: '"Done."' }] },
+        user("Also check tests."),
+        system("Update."),
+      ])
+    }),
+  )
+
+  it.effect("keeps wrapped system updates in place for models without native system updates", () =>
     Effect.gen(function* () {
       const prepared = yield* compileRequest(
         LLM.request({
-          model: vertexOpus48,
+          model,
           messages: [
-            Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: {} })]),
-            Message.tool({ id: "call_1", name: "lookup", result: "Done." }),
-            Message.system("Operator update."),
+            Message.user("Fix it."),
+            Message.assistant("Done."),
+            Message.system("Update."),
+            Message.user("Next."),
           ],
           cache: "none",
         }),
       )
 
       expect(prepared.body.messages).toEqual([
-        {
-          role: "assistant",
-          content: [{ type: "tool_use", id: "call_1", name: "lookup", input: {} }],
-        },
+        { role: "user", content: [{ type: "text", text: "Fix it." }] },
+        { role: "assistant", content: [{ type: "text", text: "Done." }] },
+        { role: "user", content: [{ type: "text", text: "<system-update>\nUpdate.\n</system-update>" }] },
+        { role: "user", content: [{ type: "text", text: "Next." }] },
+      ])
+    }),
+  )
+
+  it.effect("sends Vertex system updates after local tool results as native system messages", () =>
+    Effect.gen(function* () {
+      const toolTurn = [
+        Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: {} })]),
+        Message.tool({ id: "call_1", name: "lookup", result: "Done." }),
+        Message.system("Operator update."),
+      ]
+      const lowered = [
+        { role: "assistant", content: [{ type: "tool_use", id: "call_1", name: "lookup", input: {} }] },
         {
           role: "user",
           content: [
@@ -477,55 +550,23 @@ describe("Anthropic Messages route", () => {
               is_error: undefined,
               cache_control: undefined,
             },
-            {
-              type: "text",
-              text: "<system-update>\nOperator update.\n</system-update>",
-              cache_control: undefined,
-            },
           ],
         },
-      ])
-    }),
-  )
+        { role: "system", content: [{ type: "text", text: "Operator update.", cache_control: undefined }] },
+      ]
 
-  it.effect("preserves folded tool-result system updates across multi-turn Vertex history", () =>
-    Effect.gen(function* () {
-      const prepared = yield* compileRequest(
+      const terminal = yield* compileRequest(LLM.request({ model: vertexOpus48, messages: toolTurn, cache: "none" }))
+      const history = yield* compileRequest(
         LLM.request({
           model: vertexOpus48,
-          messages: [
-            Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: {} })]),
-            Message.tool({ id: "call_1", name: "lookup", result: "Done." }),
-            Message.system("Operator update."),
-            Message.assistant("Acknowledged."),
-            Message.user("Next step."),
-          ],
+          messages: [...toolTurn, Message.assistant("Acknowledged."), Message.user("Next step.")],
           cache: "none",
         }),
       )
 
-      expect(prepared.body.messages).toEqual([
-        {
-          role: "assistant",
-          content: [{ type: "tool_use", id: "call_1", name: "lookup", input: {} }],
-        },
-        {
-          role: "user",
-          content: [
-            {
-              type: "tool_result",
-              tool_use_id: "call_1",
-              content: '"Done."',
-              is_error: undefined,
-              cache_control: undefined,
-            },
-            {
-              type: "text",
-              text: "<system-update>\nOperator update.\n</system-update>",
-              cache_control: undefined,
-            },
-          ],
-        },
+      expect(terminal.body.messages).toEqual(lowered)
+      expect(history.body.messages).toEqual([
+        ...lowered,
         { role: "assistant", content: [{ type: "text", text: "Acknowledged." }] },
         { role: "user", content: [{ type: "text", text: "Next step." }] },
       ])