From c54677bece9feda2e4f5240159761d04c8595973 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:04:36 -0500 Subject: [PATCH] fix(core): tell HEIC, QuickTime and M4A apart from MP4 (#58887) --- packages/core/src/mime.ts | 29 +++++++++++++++++++---- packages/core/src/session/prompt.ts | 5 +++- packages/core/test/mime.test.ts | 17 +++++++++++++ packages/core/test/session-prompt.test.ts | 23 ++++++++++++++++-- 4 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 packages/core/test/mime.test.ts diff --git a/packages/core/src/mime.ts b/packages/core/src/mime.ts index 35f449ca86f..15f24023fa9 100644 --- a/packages/core/src/mime.ts +++ b/packages/core/src/mime.ts @@ -8,11 +8,26 @@ export function detect(bytes: Uint8Array) { if (startsWith(bytes, [0x25, 0x50, 0x44, 0x46, 0x2d])) return "application/pdf" if (startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes.subarray(8), [0x57, 0x45, 0x42, 0x50])) return "image/webp" - if ( - startsWith(bytes.subarray(4), [0x66, 0x74, 0x79, 0x70]) && - (startsWith(bytes.subarray(8), [0x61, 0x76, 0x69, 0x66]) || startsWith(bytes.subarray(8), [0x61, 0x76, 0x69, 0x73])) - ) - return "image/avif" + if (bytes.length >= 12 && ascii(bytes, 4, 8) === "ftyp") { + // AVIF, HEIC, QuickTime and M4A share MP4's container; the major brand tells them apart, and generic HEIF brands + // list `avif`/`avis` among the compatible brands that follow the minor version. + const brand = ascii(bytes, 8, 12) + const boxEnd = Math.min(new DataView(bytes.buffer, bytes.byteOffset).getUint32(0), bytes.length) + const compatible = Array.from({ length: Math.floor((boxEnd - 16) / 4) }, (_, index) => + ascii(bytes, 16 + index * 4, 20 + index * 4), + ) + const generic = brand === "mif1" || brand === "msf1" + if ( + brand === "avif" || + brand === "avis" || + (generic && (compatible.includes("avif") || compatible.includes("avis"))) + ) + return "image/avif" + if (generic || ["heic", "heix", "hevc", "hevx"].includes(brand)) return "image/heic" + if (brand === "qt ") return "video/quicktime" + if (brand === "M4A ") return "audio/mp4" + return "video/mp4" + } return isText(bytes) ? "text/plain" : "application/octet-stream" } @@ -26,6 +41,10 @@ export const isTextual = (mime: string) => mime === "application/javascript" || mime === "application/x-javascript" +function ascii(bytes: Uint8Array, start: number, end: number) { + return String.fromCharCode(...bytes.subarray(start, end)) +} + function startsWith(bytes: Uint8Array, prefix: number[]) { return prefix.every((value, index) => bytes[index] === value) } diff --git a/packages/core/src/session/prompt.ts b/packages/core/src/session/prompt.ts index a44ccaace33..6d3e591f2fa 100644 --- a/packages/core/src/session/prompt.ts +++ b/packages/core/src/session/prompt.ts @@ -19,6 +19,9 @@ import { SessionBlob } from "./blob.js" import { AttachmentError, SkillNotFoundError } from "./error.js" import { SessionStored } from "./stored.js" +// Image types the resizer can decode; others such as HEIC and AVIF go to the provider unchanged. +const RESIZABLE_IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp", "image/bmp"]) + export type Input = { text: string files?: PromptInput.Prompt["files"] @@ -146,7 +149,7 @@ const normalizeImageAttachment = Effect.fn("SessionPrompt.normalizeImageAttachme data: string, mime: string, ) { - if (!mime.startsWith("image/")) return { data: Base64.make(data), mime } + if (!RESIZABLE_IMAGE_MIMES.has(mime)) return { data: Base64.make(data), mime } const image = yield* Image.Service const label = attachmentLabel(input) const content = { uri: label, content: data, encoding: "base64" as const, mime } diff --git a/packages/core/test/mime.test.ts b/packages/core/test/mime.test.ts new file mode 100644 index 00000000000..a63f1900512 --- /dev/null +++ b/packages/core/test/mime.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, test } from "bun:test" +import { Mime } from "@opencode/core/mime" + +const ftyp = (hex: string) => Uint8Array.from(Buffer.from(hex, "hex")) + +describe("Mime.detect", () => { + test("tells ISO-BMFF brands apart", () => { + expect(Mime.detect(ftyp("000000206674797061766966000000004d695072617669666d6961666d696631"))).toBe("image/avif") + expect(Mime.detect(ftyp("000000186674797061766973000000006d73663161766973"))).toBe("image/avif") + expect(Mime.detect(ftyp("00000018667479706d69663100000000617669666d69663100000000"))).toBe("image/avif") + expect(Mime.detect(ftyp("000000246674797068656963000000006d6966314d6950726d6961664d694842"))).toBe("image/heic") + expect(Mime.detect(ftyp("00000018667479706d69663100000000686569636d69663100000000"))).toBe("image/heic") + expect(Mime.detect(ftyp("000000206674797069736f6d0000020069736f6d69736f32617663316d703431"))).toBe("video/mp4") + expect(Mime.detect(ftyp("00000014667479707174202000000200717420200000000877696465"))).toBe("video/quicktime") + expect(Mime.detect(ftyp("0000001c667479704d344120000002004d34412069736f6d69736f3200000008"))).toBe("audio/mp4") + }) +}) diff --git a/packages/core/test/session-prompt.test.ts b/packages/core/test/session-prompt.test.ts index 631e5e5cbda..533d2fa19d9 100644 --- a/packages/core/test/session-prompt.test.ts +++ b/packages/core/test/session-prompt.test.ts @@ -94,8 +94,10 @@ const locations = makeGlobalNode({ replacements: [Bus.node.replace(Layer.succeed(Bus.Service, bus))], }), Layer.mock(Image.Service, { - normalize: (_resource, content) => - Effect.succeed(content.content.length > 5 * 1024 * 1024 ? { ...content, content: "AA==" } : content), + normalize: (resource, content) => + content.mime === "image/heic" + ? Effect.fail(new Image.DecodeError({ resource })) + : Effect.succeed(content.content.length > 5 * 1024 * 1024 ? { ...content, content: "AA==" } : content), }), Layer.mock(Snapshot.Service, { capture: () => Effect.undefined, @@ -520,6 +522,23 @@ describe("Session.prompt", () => { }), ) + it.effect("passes images the resizer cannot decode through unchanged", () => + Effect.gen(function* () { + yield* setup + const session = yield* Session.Service + const data = Buffer.from("000000186674797068656963000000006d69663168656963", "hex").toString("base64") + + const message = yield* session.prompt({ + sessionID, + text: "Inspect this photo", + files: [{ uri: `data:image/heic;base64,${data}` }], + resume: false, + }) + + expect(message.payload.files).toEqual([{ data, mime: "image/heic", source: { type: "inline" } }]) + }), + ) + it.effect("sniffs data URL content instead of trusting its declared MIME", () => Effect.gen(function* () { yield* setup