fix(core): store opaque screenshots as JPEG before PNG (#56990)

packages/core/src/image/photon.ts

@@ -5,7 +5,27 @@ import { fileURLToPath } from "node:url"
 import { FileSystem } from "../filesystem.js"
 import { DecodeError, ResizerUnavailableError, SizeError, type Limits } from "../image.js"
 
-const JPEG_QUALITIES = [80, 85, 70, 55, 40]
+const PNG_PASSTHROUGH_MAX_BASE64_BYTES = 384 * 1024
+const SMALL_PNG_MAX_PIXELS = 256 * 256
+const JPEG_QUALITIES = [80, 70, 55, 40]
+
+// Base64 uses four bytes per three input bytes, including padding.
+const base64Length = (bytes: Uint8Array) => Math.ceil(bytes.length / 3) * 4
+
+// Samples an evenly spaced grid of up to 64×64 pixels.
+function hasTransparency(pixels: Uint8Array, width: number, height: number) {
+  const columns = Math.min(64, width)
+  const rows = Math.min(64, height)
+  let transparent = 0
+  for (let row = 0; row < rows; row++) {
+    const y = Math.floor(((row + 0.5) * height) / rows)
+    for (let column = 0; column < columns; column++) {
+      const x = Math.floor(((column + 0.5) * width) / columns)
+      if (pixels[(y * width + x) * 4 + 3]! < 250) transparent++
+    }
+  }
+  return transparent / (columns * rows) > 0.05
+}
 
 export const make = Effect.gen(function* () {
   const loadPhoton = yield* Effect.cached(
@@ -20,11 +40,8 @@ export const make = Effect.gen(function* () {
       ? Effect.fail(new ResizerUnavailableError())
       : Effect.tryPromise({
           try: async () => {
-            ;(
-              globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }
-            ).__OPENCODE_PHOTON_WASM_PATH = path.isAbsolute(photonWasm)
-              ? photonWasm
-              : fileURLToPath(new URL(photonWasm, import.meta.url))
+            ;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
+              path.isAbsolute(photonWasm) ? photonWasm : fileURLToPath(new URL(photonWasm, import.meta.url))
             return await import("@silvia-odwyer/photon-node")
           },
           catch: () => new ResizerUnavailableError(),
@@ -44,7 +61,9 @@ export const make = Effect.gen(function* () {
       const width = decoded.get_width()
       const height = decoded.get_height()
       const bytes = Buffer.byteLength(content.content, "utf-8")
-      if (width <= limits.maxWidth && height <= limits.maxHeight && bytes <= limits.maxBase64Bytes) return content
+      const fits = width <= limits.maxWidth && height <= limits.maxHeight && bytes <= limits.maxBase64Bytes
+      const smallPng = bytes <= PNG_PASSTHROUGH_MAX_BASE64_BYTES || width * height <= SMALL_PNG_MAX_PIXELS
+      if (fits && (!limits.autoResize || content.mime !== "image/png" || smallPng)) return content
       if (!limits.autoResize)
         return yield* new SizeError({
           resource,
@@ -55,6 +74,20 @@ export const make = Effect.gen(function* () {
           maxHeight: limits.maxHeight,
           maxBytes: limits.maxBase64Bytes,
         })
+      const transparent = content.mime !== "image/jpeg" && hasTransparency(decoded.get_raw_pixels(), width, height)
+      // An image that already fits only changes when JPEG makes it smaller.
+      if (fits) {
+        if (transparent) return content
+        const jpeg = decoded.get_bytes_jpeg(JPEG_QUALITIES[0]!)
+        return base64Length(jpeg) < bytes
+          ? {
+              ...content,
+              content: Buffer.from(jpeg).toString("base64"),
+              encoding: "base64" as const,
+              mime: "image/jpeg",
+            }
+          : content
+      }
       const scale = Math.min(1, limits.maxWidth / width, limits.maxHeight / height)
       const sizes = Array.from({ length: 32 }).reduce<Array<{ width: number; height: number }>>((acc) => {
         const previous = acc.at(-1) ?? {
@@ -71,16 +104,22 @@ export const make = Effect.gen(function* () {
         return acc.some((item) => item.width === next.width && item.height === next.height) ? acc : [...acc, next]
       }, [])
       for (const size of sizes) {
-        const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3)
+        const resized =
+          size.width === width && size.height === height
+            ? decoded
+            : photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3)
         try {
-          const encoders: Array<readonly [mime: string, encode: () => Uint8Array]> = [
-            ["image/png", () => resized.get_bytes()],
-            ...JPEG_QUALITIES.map((quality) => ["image/jpeg", () => resized.get_bytes_jpeg(quality)] as const),
-          ]
+          const pngEncoder = ["image/png", () => resized.get_bytes()] as const
+          const jpegEncoders = JPEG_QUALITIES.map(
+            (quality) => ["image/jpeg", () => resized.get_bytes_jpeg(quality)] as const,
+          )
+          const pngFirst = transparent || size.width * size.height <= SMALL_PNG_MAX_PIXELS
+          const encoders: Array<readonly [mime: string, encode: () => Uint8Array]> = pngFirst
+            ? [pngEncoder, ...jpegEncoders]
+            : [...jpegEncoders, pngEncoder]
           for (const [mime, encode] of encoders) {
             const candidate = encode()
-            // Base64 uses four bytes per three input bytes, including padding.
-            if (Math.ceil(candidate.length / 3) * 4 <= limits.maxBase64Bytes)
+            if (base64Length(candidate) <= limits.maxBase64Bytes)
               return {
                 ...content,
                 content: Buffer.from(candidate).toString("base64"),
@@ -89,7 +128,7 @@ export const make = Effect.gen(function* () {
               }
           }
         } finally {
-          resized.free()
+          if (resized !== decoded) resized.free()
         }
       }
       return yield* new SizeError({

packages/core/test/tool-read.test.ts

@@ -426,6 +426,83 @@ describe("ReadTool", () => {
     }),
   )
 
+  it.effect("stores large opaque PNG screenshots as JPEG", () =>
+    Effect.gen(function* () {
+      const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
+      const width = 2000
+      const height = 1125
+      const opaquePixels = Uint8Array.from({ length: width * height * 4 }, (_, index) =>
+        index % 4 === 3 ? 255 : (index * 31 + (index >> 7)) % 256,
+      )
+      const opaqueSource = new photon.PhotonImage(opaquePixels, width, height)
+      const opaqueBase64 = Buffer.from(opaqueSource.get_bytes()).toString("base64")
+      opaqueSource.free()
+      readResult = {
+        type: "file",
+        uri: "file:///screenshot.png",
+        name: "screenshot.png",
+        content: opaqueBase64,
+        encoding: "base64",
+        mime: "image/png",
+      }
+      const registry = yield* Tool.Service
+      const result = yield* executeTool(registry, {
+        sessionID,
+        ...toolIdentity,
+        call: { type: "tool-call", id: "call-transcode-screenshot", name: "read", input: { path: "screenshot.png" } },
+      })
+
+      expect(result.status).toBe("completed")
+      if (result.status !== "completed") return
+      const media = result.content?.[1]
+      expect(media?.type).toBe("file")
+      if (media?.type !== "file") return
+      expect(media.mime).toBe("image/jpeg")
+      const b64 = media.uri.split(",")[1] ?? ""
+      expect(b64.length).toBeLessThan(opaqueBase64.length)
+      const resized = photon.PhotonImage.new_from_byteslice(Buffer.from(b64, "base64"))
+      expect(resized.get_width()).toBe(width)
+      expect(resized.get_height()).toBe(height)
+      resized.free()
+    }),
+  )
+
+  it.effect("keeps fitting PNGs that are small, transparent, or smaller than their JPEG", () =>
+    Effect.gen(function* () {
+      const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
+      const png = (width: number, height: number, pixel: (x: number, y: number, channel: number) => number) => {
+        const pixels = new Uint8Array(width * height * 4)
+        for (let index = 0; index < pixels.length; index++) {
+          const offset = index >> 2
+          pixels[index] = pixel(offset % width, Math.floor(offset / width), index % 4)
+        }
+        const source = new photon.PhotonImage(pixels, width, height)
+        const content = Buffer.from(source.get_bytes()).toString("base64")
+        source.free()
+        return { uri: "file:///image.png", content, encoding: "base64" as const, mime: "image/png" }
+      }
+      let seed = 1
+      const blocks = Array.from({ length: 500 * 375 }, () => {
+        seed = (Math.imul(seed, 1103515245) + 12345) >>> 0
+        return (seed >>> 31) * 255
+      })
+      const cases = {
+        transparent: png(600, 600, (x, y, channel) =>
+          channel === 3 ? (y < 120 ? 0 : 255) : (((y * 600 + x) * 4 + channel) * 31 + x) % 256,
+        ),
+        small: png(1000, 1000, (_x, y, channel) => (channel === 3 ? 255 : y % 256)),
+        diagram: png(2000, 1500, (x, y, channel) =>
+          channel === 3 ? 255 : blocks[Math.floor(y / 4) * 500 + Math.floor(x / 4)]!,
+        ),
+      }
+      expect(cases.transparent.content.length).toBeGreaterThan(384 * 1024)
+      expect(cases.small.content.length).toBeLessThanOrEqual(384 * 1024)
+      expect(cases.diagram.content.length).toBeGreaterThan(384 * 1024)
+      const image = yield* Image.Service
+      for (const content of Object.values(cases)) expect(yield* image.normalize("image.png", content)).toEqual(content)
+    }),
+  )
+
   it.effect("accepts PNG candidates at the exact base64 limit and skips them one byte below", () =>
     Effect.gen(function* () {
       const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))

packages/gui-extensions/src/browser/chromium.ts

@@ -967,7 +967,7 @@ export function createBrowserPage(
 
         if (bounds.width * bounds.height * (scale * pixelRatio) ** 2 > 16_000_000)
           throw new Error("Screenshot exceeds 16 megapixels; capture an element or use a smaller maxWidth.")
-        const format = action.format ?? "png"
+        const format = action.format ?? "jpeg"
 
         const capture = await cdp.send("Page.captureScreenshot", {
           format,