diff --git a/packages/core/src/session/model-request.ts b/packages/core/src/session/model-request.ts index 293c93a5e61..129456de6f7 100644 --- a/packages/core/src/session/model-request.ts +++ b/packages/core/src/session/model-request.ts @@ -188,30 +188,31 @@ export const unsupportedParts = ( export const boundImages = (messages: LLMRequest["messages"]) => { const isImage = (mime: string) => mime.toLowerCase().startsWith("image/") - const imageBytes = messages.reduce( - (total, message) => - total + - message.content.reduce((sum, part) => { - if (part.type === "media" && isImage(part.media.mediaType)) return sum + mediaBytes(part.media) - if (part.type !== "tool-result" || part.result.type !== "content") return sum - return ( - sum + - part.result.value.reduce( - (bytes: number, item: Content) => - bytes + (item.type === "file" && isImage(item.mime) ? Buffer.byteLength(item.uri) : 0), - 0, - ) - ) - }, 0), - 0, + const sizes = messages.flatMap((message) => + message.content.flatMap((part) => { + if (part.type === "media") return isImage(part.media.mediaType) ? [mediaBytes(part.media)] : [] + if (part.type !== "tool-result" || part.result.type !== "content") return [] + return part.result.value.flatMap((item: Content) => + item.type === "file" && isImage(item.mime) ? [Buffer.byteLength(item.uri)] : [], + ) + }), ) - if (imageBytes <= IMAGE_BYTES_TRIGGER) return messages + // Replay the images in order, dropping the oldest down to the target whenever the kept payload passes the trigger. + // A new image then moves the cut only every (trigger - target) bytes instead of on every request, so the replaced + // prefix, and with it the provider's prompt cache, stays the same between cuts. + let cut = 0 + let kept = 0 + for (const size of sizes) { + kept += size + if (kept <= IMAGE_BYTES_TRIGGER) continue + while (kept > IMAGE_BYTES_TARGET) kept -= sizes[cut++]! + } + if (cut === 0) return messages - let removed = 0 + let index = 0 return replaceMedia(messages, (media) => { - if (!isImage(media.mime) || imageBytes - removed <= IMAGE_BYTES_TARGET) return - removed += media.bytes() - return IMAGE_REMOVED + if (!isImage(media.mime)) return + return index++ < cut ? IMAGE_REMOVED : undefined }) } diff --git a/packages/core/test/session-model-request.test.ts b/packages/core/test/session-model-request.test.ts index fb1837a5956..d1b2559f3bb 100644 --- a/packages/core/test/session-model-request.test.ts +++ b/packages/core/test/session-model-request.test.ts @@ -203,6 +203,21 @@ describe("SessionModelRequest.boundImages", () => { expect(result[3]).toBe(third) }) + test("keeps the replaced prefix stable as new images arrive, so the prompt cache survives", () => { + const image = (name: string) => + Message.user({ type: "media", media: Media.base64("a".repeat(3 * 1024 * 1024), "image/png"), filename: name }) + const removed = (messages: ReturnType) => + messages.filter((message) => message.content[0]?.type === "text").length + const history = Array.from({ length: 9 }, (_, i) => image(`${i}.png`)) + + // 27 MiB passes the 25 MiB trigger: the oldest four go, leaving 15 MiB. + expect(removed(boundImages(history))).toBe(4) + // Each later image used to move the cut by one; now the cut holds until the kept payload passes the trigger again. + expect(removed(boundImages([...history, image("9.png")]))).toBe(4) + expect(removed(boundImages([...history, image("9.png"), image("10.png"), image("11.png")]))).toBe(4) + expect(removed(boundImages([...history, ...Array.from({ length: 4 }, (_, i) => image(`${9 + i}.png`))]))).toBe(8) + }) + test("replaces images nested in tool results", () => { const image = "a".repeat(13 * 1024 * 1024) const result = boundImages([