fix(codemode): decode UTF-8 across chunk boundaries (#63600)

This commit is contained in:
Aiden Cline
2026-11-12 14:30:06 -06:00
committed by GitHub
Unverified
parent 8921769577
commit a3d6e54b3d
5 changed files with 169 additions and 13 deletions
+3 -2
View File
@@ -596,14 +596,15 @@ with a hint to encode as text first (`TextDecoder`, `toBase64`, `toHex`).
frozen. Functions, symbols, promises, iterators, URL, Headers, and tool references cannot be cloned; without a
`DOMException`, the failure is a `TypeError` whose message starts with `DataCloneError:`.
- [x] `TextEncoder` and `TextDecoder` for UTF-8 only: any other label is a `RangeError`. `TextDecoder` accepts the
`fatal` and `ignoreBOM` options; `decode` takes a Uint8Array or nothing.
`fatal` and `ignoreBOM` options. `decode(chunk, { stream: true })` accepts incremental Uint8Array input;
`decode()` flushes the stream.
- [x] `new Headers()` from records, synchronous iterables of pairs, and Headers, wrapping the host's `Headers`: names
fold to lowercase, values are normalized and combined, and invalid names or values throw a `TypeError`.
- [x] Headers `append`, `delete`, `get`, `getSetCookie`, `has`, `set`, `forEach`, `keys`, `values`, `entries`, and
`[Symbol.iterator]`; iteration is live and sorted by name, with `set-cookie` values kept apart.
- [x] Headers serialize to a `{ name: value }` object in JSON, in results, and in tool arguments.
- [ ] `Request`, `Response`, and `Blob`.
- [ ] `crypto.subtle` and `TextDecoder` streaming or non-UTF-8 encodings.
- [ ] `crypto.subtle` and non-UTF-8 `TextDecoder` encodings.
## Extensions
+7 -1
View File
@@ -225,7 +225,13 @@ const builtinSpecs = (): ReadonlyArray<ClassSpec> => [
{ name: "HeadersObj", ctor: HeadersObj, fields: ["headers"], obj: true, constants: { tag: "Headers" } },
{ name: "URLObj", ctor: URLObj, fields: ["searchParams", "url"], obj: true, constants: { tag: "URL" } },
{ name: "Bytes", ctor: Bytes, fields: ["bytes"], obj: true, constants: { tag: "Uint8Array" } },
{ name: "TextDecoderObj", ctor: TextDecoderObj, fields: ["decoder"], obj: true, constants: { tag: "Object" } },
{
name: "TextDecoderObj",
ctor: TextDecoderObj,
fields: ["decoder", "ignoreBOM", "queued", "seen"],
obj: true,
constants: { tag: "Object" },
},
{
name: "AsyncFromSyncIterator",
ctor: AsyncFromSyncIterator,
+71 -10
View File
@@ -196,14 +196,74 @@ export const textEncoderGlobal = <R>(ctx: Interpreter<R>) => {
})
}
/** A `TextDecoder` holding the host decoder its label and options configured. */
// Replacement mode only needs to identify an incomplete suffix (at most three bytes).
// Fatal mode also needs the queue position after the first error, which the host doesn't expose.
const streamBoundary = (bytes: Uint8Array, fatal: boolean) => {
let next = fatal ? 0 : Math.max(0, bytes.length - 3)
while (next < bytes.length) {
const start = next
const lead = bytes[next++]!
if (lead < 0x80) continue
if (lead < 0xc2 || lead > 0xf4) {
if (fatal) return { end: next, error: true }
continue
}
const width = lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4
let lower = lead === 0xe0 ? 0xa0 : lead === 0xf0 ? 0x90 : 0x80
let upper = lead === 0xed ? 0x9f : lead === 0xf4 ? 0x8f : 0xbf
while (next < start + width) {
if (next === bytes.length) return { end: start, error: false }
const byte = bytes[next]!
// A bad continuation is reprocessed as a leading byte, even after a fatal error.
if (byte < lower || byte > upper) {
if (fatal) return { end: next, error: true }
break
}
next++
lower = 0x80
upper = 0xbf
}
}
return { end: next, error: false }
}
const empty = new Uint8Array()
export class TextDecoderObj extends Obj {
queued: Uint8Array = empty
seen = false
constructor(
proto: Obj,
readonly decoder: TextDecoder,
readonly ignoreBOM: boolean,
) {
super(proto)
}
decode(bytes: Uint8Array, stream: boolean) {
if (this.queued.length && bytes.length) {
checkArrayLength(this.queued.length + bytes.length)
const joined = new Uint8Array(this.queued.length + bytes.length)
joined.set(this.queued)
joined.set(bytes, this.queued.length)
bytes = joined
}
if (!bytes.length) bytes = this.queued
const { end, error } = stream ? streamBoundary(bytes, this.decoder.fatal) : { end: bytes.length, error: false }
this.queued = end < bytes.length ? bytes.slice(end) : empty
if (error) throw typeError("TextDecoder.decode: the input is not valid utf-8.")
const stripBOM = !this.ignoreBOM && !this.seen
if (!stream) this.seen = false
try {
// Keep the host decoder stateless: snapshots own the pending bytes and BOM decision.
const text = this.decoder.decode(end === bytes.length ? bytes : bytes.subarray(0, end))
if (stream && text.length) this.seen = true
return stripBOM && text.charCodeAt(0) === 0xfeff ? text.slice(1) : text
} catch {
throw typeError("TextDecoder.decode: the input is not valid utf-8.")
}
}
}
// The WHATWG labels for UTF-8, the only encoding CodeMode decodes.
@@ -215,22 +275,22 @@ export const textDecoderGlobal = <R>(ctx: Interpreter<R>) => {
const self = (thisValue: Value, name: string) => receiver(TextDecoderObj, thisValue, `TextDecoder.prototype.${name}`)
defineAccessor(proto, "encoding", (thisValue) => self(thisValue, "encoding").decoder.encoding)
defineAccessor(proto, "fatal", (thisValue) => self(thisValue, "fatal").decoder.fatal)
defineAccessor(proto, "ignoreBOM", (thisValue) => self(thisValue, "ignoreBOM").decoder.ignoreBOM)
defineAccessor(proto, "ignoreBOM", (thisValue) => self(thisValue, "ignoreBOM").ignoreBOM)
methods(builtins, proto, [
[
"decode",
0,
(thisValue, args) => {
const decoder = self(thisValue, "decode").decoder
if (args[0] === undefined) return ""
if (!(args[0] instanceof Bytes)) {
const decoder = self(thisValue, "decode")
if (args[0] !== undefined && !(args[0] instanceof Bytes)) {
throw typeError(`TextDecoder.decode expects a Uint8Array, received ${describeValue(args[0])}.`)
}
try {
return decoder.decode(args[0].bytes)
} catch {
throw typeError(`TextDecoder.decode: the input is not valid ${decoder.encoding}.`)
const options = args[1]
if (options !== undefined && typeof options !== "object") {
throw typeError("TextDecoder.decode expects an options object.")
}
const stream = options instanceof Obj && Boolean(get(options, "stream"))
return decoder.decode(args[0]?.bytes ?? empty, stream)
},
],
])
@@ -244,7 +304,8 @@ export const textDecoderGlobal = <R>(ctx: Interpreter<R>) => {
const flag = (name: string) => options !== undefined && Boolean(get(options, name))
return new TextDecoderObj(
prototypeFrom(newTarget, proto),
new TextDecoder("utf-8", { fatal: flag("fatal"), ignoreBOM: flag("ignoreBOM") }),
new TextDecoder("utf-8", { fatal: flag("fatal"), ignoreBOM: true }),
flag("ignoreBOM"),
)
},
})
+8
View File
@@ -13,6 +13,14 @@ import { buildId } from "../src/snapshot.js"
// The VM's snapshots: a run saved and restored into a fresh runtime anywhere it slices goes on as if it never stopped.
const programs: Record<string, string> = {
"a UTF-8 decoder with unread bytes after a fatal stream error": `
const decoder = new TextDecoder('utf8', { fatal: true })
try { decoder.decode(new Uint8Array([255, 65, 66, 67, 68]), { stream: true }) }
catch (error) { return [error.name, decoder.decode()] }`,
"a UTF-8 decoder with pending bytes and a consumed BOM": `
const decoder = new TextDecoder('utf8', { fatal: true })
const start = decoder.decode(new Uint8Array([0xef, 0xbb, 0xbf, 0xef]), { stream: true })
return [start, decoder.decode(new Uint8Array([0xbb, 0xbf]))]`,
"closures over per-iteration bindings": `
const fns = []
for (let i = 0; i < 5; i++) fns.push(() => i * 2)
+80
View File
@@ -1218,6 +1218,86 @@ describe("TextEncoder and TextDecoder", () => {
])
})
test("streams an emoji one byte at a time and copies the pending bytes", async () => {
expect(
await value(`
const decoder = new TextDecoder()
const chunk = new Uint8Array([65, 0xf0])
const stream = { stream: true }
const start = decoder.decode(chunk, stream)
chunk[1] = 0
return [start, decoder.decode(undefined, stream), decoder.decode(new Uint8Array([0x9f]), stream),
decoder.decode(new Uint8Array([0x98]), stream), decoder.decode(new Uint8Array([0x80, 66])), decoder.decode()]
`),
).toEqual(["A", "", "", "", "😀B", ""])
})
test("only strips the first BOM of each stream, including a split BOM", async () => {
expect(
await value(`
return [false, true].map(ignoreBOM => {
const decoder = new TextDecoder('utf8', { ignoreBOM })
const bom = new Uint8Array([0xef, 0xbb, 0xbf])
return [decoder.decode(bom.slice(0, 1), { stream: true }), decoder.decode(bom.slice(1), { stream: true }),
decoder.decode(bom), decoder.decode(bom)]
})
`),
).toEqual([
["", "", "\ufeff", ""],
["", "\ufeff", "\ufeff", "\ufeff"],
])
})
test("reports malformed UTF-8 immediately and truncated UTF-8 on flush", async () => {
expect(
await value(`
return [false, true].map(fatal => {
return [[[0xe0], [0x80]], [[0xed, 0xa0]], [[0xf0, 0x80]], [[0xf4, 0x90]],
[[0xe2, 0x82], [65]], [[0xf0, 0x9f], undefined, undefined, [0xef, 0xbb, 0xbf, 66]]].flatMap(chunks => {
const decoder = new TextDecoder('utf8', { fatal })
return chunks.map(bytes => {
try { return bytes ? decoder.decode(new Uint8Array(bytes), { stream: true }) : decoder.decode() }
catch (error) { return error.name }
})
})
})
`),
).toEqual([
["", "��", "��", "��", "��", "", "�A", "", "�", "", "B"],
["", "TypeError", "TypeError", "TypeError", "TypeError", "", "TypeError", "", "TypeError", "", "B"],
])
})
test("keeps unread input in order after a fatal stream error", async () => {
expect(
await value(`
const decoder = new TextDecoder('utf8', { fatal: true })
const stream = { stream: true }
const fail = bytes => {
try { return decoder.decode(bytes, stream) } catch (error) { return error.name }
}
const chunk = new Uint8Array([0xff, 65])
const first = fail(chunk)
chunk[1] = 0
return [first, decoder.decode(), fail(new Uint8Array([0xe2, 0x82])),
fail(new Uint8Array([66, 67])), decoder.decode(new Uint8Array([68])), decoder.decode()]
`),
).toEqual(["TypeError", "A", "", "TypeError", "BCD", ""])
})
test("rejects primitive options before touching the partial character", async () => {
expect(
await value(`
const decoder = new TextDecoder()
decoder.decode(new Uint8Array([0xc2]), { stream: true })
const errors = [true, false, 0, '', Symbol.iterator].map(options => {
try { return decoder.decode(undefined, options) } catch (error) { return error.name }
})
return [...errors, decoder.decode(new Uint8Array([0xa2]), null)]
`),
).toEqual(["TypeError", "TypeError", "TypeError", "TypeError", "TypeError", "¢"])
})
test("only UTF-8 is supported; fatal decoding rejects malformed input", async () => {
expect((await error(`new TextDecoder("latin1")`)).message).toContain('The "latin1" encoding is not supported')
expect((await error(`new TextDecoder("utf-8", { fatal: true }).decode(new Uint8Array([0xff]))`)).message).toContain(