diff --git a/packages/core/src/image/photon.ts b/packages/core/src/image/photon.ts index 9723edba926..7357c0bf225 100644 --- a/packages/core/src/image/photon.ts +++ b/packages/core/src/image/photon.ts @@ -5,7 +5,27 @@ import { fileURLToPath } from "node:url" import { FileSystem } from "../filesystem.js" import { DecodeError, ResizerUnavailableError, SizeError, type Limits } from "../image.js" -const JPEG_QUALITIES = [80, 85, 70, 55, 40] +const PNG_PASSTHROUGH_MAX_BASE64_BYTES = 384 * 1024 +const SMALL_PNG_MAX_PIXELS = 256 * 256 +const JPEG_QUALITIES = [80, 70, 55, 40] + +// Base64 uses four bytes per three input bytes, including padding. +const base64Length = (bytes: Uint8Array) => Math.ceil(bytes.length / 3) * 4 + +// Samples an evenly spaced grid of up to 64×64 pixels. +function hasTransparency(pixels: Uint8Array, width: number, height: number) { + const columns = Math.min(64, width) + const rows = Math.min(64, height) + let transparent = 0 + for (let row = 0; row < rows; row++) { + const y = Math.floor(((row + 0.5) * height) / rows) + for (let column = 0; column < columns; column++) { + const x = Math.floor(((column + 0.5) * width) / columns) + if (pixels[(y * width + x) * 4 + 3]! < 250) transparent++ + } + } + return transparent / (columns * rows) > 0.05 +} export const make = Effect.gen(function* () { const loadPhoton = yield* Effect.cached( @@ -20,11 +40,8 @@ export const make = Effect.gen(function* () { ? Effect.fail(new ResizerUnavailableError()) : Effect.tryPromise({ try: async () => { - ;( - globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string } - ).__OPENCODE_PHOTON_WASM_PATH = path.isAbsolute(photonWasm) - ? photonWasm - : fileURLToPath(new URL(photonWasm, import.meta.url)) + ;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH = + path.isAbsolute(photonWasm) ? photonWasm : fileURLToPath(new URL(photonWasm, import.meta.url)) return await import("@silvia-odwyer/photon-node") }, catch: () => new ResizerUnavailableError(), @@ -44,7 +61,9 @@ export const make = Effect.gen(function* () { const width = decoded.get_width() const height = decoded.get_height() const bytes = Buffer.byteLength(content.content, "utf-8") - if (width <= limits.maxWidth && height <= limits.maxHeight && bytes <= limits.maxBase64Bytes) return content + const fits = width <= limits.maxWidth && height <= limits.maxHeight && bytes <= limits.maxBase64Bytes + const smallPng = bytes <= PNG_PASSTHROUGH_MAX_BASE64_BYTES || width * height <= SMALL_PNG_MAX_PIXELS + if (fits && (!limits.autoResize || content.mime !== "image/png" || smallPng)) return content if (!limits.autoResize) return yield* new SizeError({ resource, @@ -55,6 +74,20 @@ export const make = Effect.gen(function* () { maxHeight: limits.maxHeight, maxBytes: limits.maxBase64Bytes, }) + const transparent = content.mime !== "image/jpeg" && hasTransparency(decoded.get_raw_pixels(), width, height) + // An image that already fits only changes when JPEG makes it smaller. + if (fits) { + if (transparent) return content + const jpeg = decoded.get_bytes_jpeg(JPEG_QUALITIES[0]!) + return base64Length(jpeg) < bytes + ? { + ...content, + content: Buffer.from(jpeg).toString("base64"), + encoding: "base64" as const, + mime: "image/jpeg", + } + : content + } const scale = Math.min(1, limits.maxWidth / width, limits.maxHeight / height) const sizes = Array.from({ length: 32 }).reduce>((acc) => { const previous = acc.at(-1) ?? { @@ -71,16 +104,22 @@ export const make = Effect.gen(function* () { return acc.some((item) => item.width === next.width && item.height === next.height) ? acc : [...acc, next] }, []) for (const size of sizes) { - const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3) + const resized = + size.width === width && size.height === height + ? decoded + : photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3) try { - const encoders: Array Uint8Array]> = [ - ["image/png", () => resized.get_bytes()], - ...JPEG_QUALITIES.map((quality) => ["image/jpeg", () => resized.get_bytes_jpeg(quality)] as const), - ] + const pngEncoder = ["image/png", () => resized.get_bytes()] as const + const jpegEncoders = JPEG_QUALITIES.map( + (quality) => ["image/jpeg", () => resized.get_bytes_jpeg(quality)] as const, + ) + const pngFirst = transparent || size.width * size.height <= SMALL_PNG_MAX_PIXELS + const encoders: Array Uint8Array]> = pngFirst + ? [pngEncoder, ...jpegEncoders] + : [...jpegEncoders, pngEncoder] for (const [mime, encode] of encoders) { const candidate = encode() - // Base64 uses four bytes per three input bytes, including padding. - if (Math.ceil(candidate.length / 3) * 4 <= limits.maxBase64Bytes) + if (base64Length(candidate) <= limits.maxBase64Bytes) return { ...content, content: Buffer.from(candidate).toString("base64"), @@ -89,7 +128,7 @@ export const make = Effect.gen(function* () { } } } finally { - resized.free() + if (resized !== decoded) resized.free() } } return yield* new SizeError({ diff --git a/packages/core/test/tool-read.test.ts b/packages/core/test/tool-read.test.ts index 4bf3af248d0..795e7ac028a 100644 --- a/packages/core/test/tool-read.test.ts +++ b/packages/core/test/tool-read.test.ts @@ -426,6 +426,83 @@ describe("ReadTool", () => { }), ) + it.effect("stores large opaque PNG screenshots as JPEG", () => + Effect.gen(function* () { + const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node")) + const width = 2000 + const height = 1125 + const opaquePixels = Uint8Array.from({ length: width * height * 4 }, (_, index) => + index % 4 === 3 ? 255 : (index * 31 + (index >> 7)) % 256, + ) + const opaqueSource = new photon.PhotonImage(opaquePixels, width, height) + const opaqueBase64 = Buffer.from(opaqueSource.get_bytes()).toString("base64") + opaqueSource.free() + readResult = { + type: "file", + uri: "file:///screenshot.png", + name: "screenshot.png", + content: opaqueBase64, + encoding: "base64", + mime: "image/png", + } + const registry = yield* Tool.Service + const result = yield* executeTool(registry, { + sessionID, + ...toolIdentity, + call: { type: "tool-call", id: "call-transcode-screenshot", name: "read", input: { path: "screenshot.png" } }, + }) + + expect(result.status).toBe("completed") + if (result.status !== "completed") return + const media = result.content?.[1] + expect(media?.type).toBe("file") + if (media?.type !== "file") return + expect(media.mime).toBe("image/jpeg") + const b64 = media.uri.split(",")[1] ?? "" + expect(b64.length).toBeLessThan(opaqueBase64.length) + const resized = photon.PhotonImage.new_from_byteslice(Buffer.from(b64, "base64")) + expect(resized.get_width()).toBe(width) + expect(resized.get_height()).toBe(height) + resized.free() + }), + ) + + it.effect("keeps fitting PNGs that are small, transparent, or smaller than their JPEG", () => + Effect.gen(function* () { + const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node")) + const png = (width: number, height: number, pixel: (x: number, y: number, channel: number) => number) => { + const pixels = new Uint8Array(width * height * 4) + for (let index = 0; index < pixels.length; index++) { + const offset = index >> 2 + pixels[index] = pixel(offset % width, Math.floor(offset / width), index % 4) + } + const source = new photon.PhotonImage(pixels, width, height) + const content = Buffer.from(source.get_bytes()).toString("base64") + source.free() + return { uri: "file:///image.png", content, encoding: "base64" as const, mime: "image/png" } + } + let seed = 1 + const blocks = Array.from({ length: 500 * 375 }, () => { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0 + return (seed >>> 31) * 255 + }) + const cases = { + transparent: png(600, 600, (x, y, channel) => + channel === 3 ? (y < 120 ? 0 : 255) : (((y * 600 + x) * 4 + channel) * 31 + x) % 256, + ), + small: png(1000, 1000, (_x, y, channel) => (channel === 3 ? 255 : y % 256)), + diagram: png(2000, 1500, (x, y, channel) => + channel === 3 ? 255 : blocks[Math.floor(y / 4) * 500 + Math.floor(x / 4)]!, + ), + } + expect(cases.transparent.content.length).toBeGreaterThan(384 * 1024) + expect(cases.small.content.length).toBeLessThanOrEqual(384 * 1024) + expect(cases.diagram.content.length).toBeGreaterThan(384 * 1024) + const image = yield* Image.Service + for (const content of Object.values(cases)) expect(yield* image.normalize("image.png", content)).toEqual(content) + }), + ) + it.effect("accepts PNG candidates at the exact base64 limit and skips them one byte below", () => Effect.gen(function* () { const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node")) diff --git a/packages/gui-extensions/src/browser/chromium.ts b/packages/gui-extensions/src/browser/chromium.ts index 4e665fa8c87..cb6d73a6804 100644 --- a/packages/gui-extensions/src/browser/chromium.ts +++ b/packages/gui-extensions/src/browser/chromium.ts @@ -967,7 +967,7 @@ export function createBrowserPage( if (bounds.width * bounds.height * (scale * pixelRatio) ** 2 > 16_000_000) throw new Error("Screenshot exceeds 16 megapixels; capture an element or use a smaller maxWidth.") - const format = action.format ?? "png" + const format = action.format ?? "jpeg" const capture = await cdp.send("Page.captureScreenshot", { format,