fix(core): tell HEIC, QuickTime and M4A apart from MP4 (#58887)

This commit is contained in:
Aiden Cline
2026-10-26 00:04:36 -05:00
committed by GitHub
Unverified
parent 421e44ec8c
commit c54677bece
4 changed files with 66 additions and 8 deletions
+24 -5
View File
@@ -8,11 +8,26 @@ export function detect(bytes: Uint8Array) {
if (startsWith(bytes, [0x25, 0x50, 0x44, 0x46, 0x2d])) return "application/pdf"
if (startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes.subarray(8), [0x57, 0x45, 0x42, 0x50]))
return "image/webp"
if (
startsWith(bytes.subarray(4), [0x66, 0x74, 0x79, 0x70]) &&
(startsWith(bytes.subarray(8), [0x61, 0x76, 0x69, 0x66]) || startsWith(bytes.subarray(8), [0x61, 0x76, 0x69, 0x73]))
)
return "image/avif"
if (bytes.length >= 12 && ascii(bytes, 4, 8) === "ftyp") {
// AVIF, HEIC, QuickTime and M4A share MP4's container; the major brand tells them apart, and generic HEIF brands
// list `avif`/`avis` among the compatible brands that follow the minor version.
const brand = ascii(bytes, 8, 12)
const boxEnd = Math.min(new DataView(bytes.buffer, bytes.byteOffset).getUint32(0), bytes.length)
const compatible = Array.from({ length: Math.floor((boxEnd - 16) / 4) }, (_, index) =>
ascii(bytes, 16 + index * 4, 20 + index * 4),
)
const generic = brand === "mif1" || brand === "msf1"
if (
brand === "avif" ||
brand === "avis" ||
(generic && (compatible.includes("avif") || compatible.includes("avis")))
)
return "image/avif"
if (generic || ["heic", "heix", "hevc", "hevx"].includes(brand)) return "image/heic"
if (brand === "qt ") return "video/quicktime"
if (brand === "M4A ") return "audio/mp4"
return "video/mp4"
}
return isText(bytes) ? "text/plain" : "application/octet-stream"
}
@@ -26,6 +41,10 @@ export const isTextual = (mime: string) =>
mime === "application/javascript" ||
mime === "application/x-javascript"
function ascii(bytes: Uint8Array, start: number, end: number) {
return String.fromCharCode(...bytes.subarray(start, end))
}
function startsWith(bytes: Uint8Array, prefix: number[]) {
return prefix.every((value, index) => bytes[index] === value)
}
+4 -1
View File
@@ -19,6 +19,9 @@ import { SessionBlob } from "./blob.js"
import { AttachmentError, SkillNotFoundError } from "./error.js"
import { SessionStored } from "./stored.js"
// Image types the resizer can decode; others such as HEIC and AVIF go to the provider unchanged.
const RESIZABLE_IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp", "image/bmp"])
export type Input = {
text: string
files?: PromptInput.Prompt["files"]
@@ -146,7 +149,7 @@ const normalizeImageAttachment = Effect.fn("SessionPrompt.normalizeImageAttachme
data: string,
mime: string,
) {
if (!mime.startsWith("image/")) return { data: Base64.make(data), mime }
if (!RESIZABLE_IMAGE_MIMES.has(mime)) return { data: Base64.make(data), mime }
const image = yield* Image.Service
const label = attachmentLabel(input)
const content = { uri: label, content: data, encoding: "base64" as const, mime }
+17
View File
@@ -0,0 +1,17 @@
import { describe, expect, test } from "bun:test"
import { Mime } from "@opencode/core/mime"
const ftyp = (hex: string) => Uint8Array.from(Buffer.from(hex, "hex"))
describe("Mime.detect", () => {
test("tells ISO-BMFF brands apart", () => {
expect(Mime.detect(ftyp("000000206674797061766966000000004d695072617669666d6961666d696631"))).toBe("image/avif")
expect(Mime.detect(ftyp("000000186674797061766973000000006d73663161766973"))).toBe("image/avif")
expect(Mime.detect(ftyp("00000018667479706d69663100000000617669666d69663100000000"))).toBe("image/avif")
expect(Mime.detect(ftyp("000000246674797068656963000000006d6966314d6950726d6961664d694842"))).toBe("image/heic")
expect(Mime.detect(ftyp("00000018667479706d69663100000000686569636d69663100000000"))).toBe("image/heic")
expect(Mime.detect(ftyp("000000206674797069736f6d0000020069736f6d69736f32617663316d703431"))).toBe("video/mp4")
expect(Mime.detect(ftyp("00000014667479707174202000000200717420200000000877696465"))).toBe("video/quicktime")
expect(Mime.detect(ftyp("0000001c667479704d344120000002004d34412069736f6d69736f3200000008"))).toBe("audio/mp4")
})
})
+21 -2
View File
@@ -94,8 +94,10 @@ const locations = makeGlobalNode({
replacements: [Bus.node.replace(Layer.succeed(Bus.Service, bus))],
}),
Layer.mock(Image.Service, {
normalize: (_resource, content) =>
Effect.succeed(content.content.length > 5 * 1024 * 1024 ? { ...content, content: "AA==" } : content),
normalize: (resource, content) =>
content.mime === "image/heic"
? Effect.fail(new Image.DecodeError({ resource }))
: Effect.succeed(content.content.length > 5 * 1024 * 1024 ? { ...content, content: "AA==" } : content),
}),
Layer.mock(Snapshot.Service, {
capture: () => Effect.undefined,
@@ -520,6 +522,23 @@ describe("Session.prompt", () => {
}),
)
it.effect("passes images the resizer cannot decode through unchanged", () =>
Effect.gen(function* () {
yield* setup
const session = yield* Session.Service
const data = Buffer.from("000000186674797068656963000000006d69663168656963", "hex").toString("base64")
const message = yield* session.prompt({
sessionID,
text: "Inspect this photo",
files: [{ uri: `data:image/heic;base64,${data}` }],
resume: false,
})
expect(message.payload.files).toEqual([{ data, mime: "image/heic", source: { type: "inline" } }])
}),
)
it.effect("sniffs data URL content instead of trusting its declared MIME", () =>
Effect.gen(function* () {
yield* setup