diff --git a/packages/codemode/interpreter-support.md b/packages/codemode/interpreter-support.md index 52ad4dc9c1e..099d615ff52 100644 --- a/packages/codemode/interpreter-support.md +++ b/packages/codemode/interpreter-support.md @@ -596,14 +596,15 @@ with a hint to encode as text first (`TextDecoder`, `toBase64`, `toHex`). frozen. Functions, symbols, promises, iterators, URL, Headers, and tool references cannot be cloned; without a `DOMException`, the failure is a `TypeError` whose message starts with `DataCloneError:`. - [x] `TextEncoder` and `TextDecoder` for UTF-8 only: any other label is a `RangeError`. `TextDecoder` accepts the - `fatal` and `ignoreBOM` options; `decode` takes a Uint8Array or nothing. + `fatal` and `ignoreBOM` options. `decode(chunk, { stream: true })` accepts incremental Uint8Array input; + `decode()` flushes the stream. - [x] `new Headers()` from records, synchronous iterables of pairs, and Headers, wrapping the host's `Headers`: names fold to lowercase, values are normalized and combined, and invalid names or values throw a `TypeError`. - [x] Headers `append`, `delete`, `get`, `getSetCookie`, `has`, `set`, `forEach`, `keys`, `values`, `entries`, and `[Symbol.iterator]`; iteration is live and sorted by name, with `set-cookie` values kept apart. - [x] Headers serialize to a `{ name: value }` object in JSON, in results, and in tool arguments. - [ ] `Request`, `Response`, and `Blob`. -- [ ] `crypto.subtle` and `TextDecoder` streaming or non-UTF-8 encodings. +- [ ] `crypto.subtle` and non-UTF-8 `TextDecoder` encodings. ## Extensions diff --git a/packages/codemode/src/snapshot/codec.ts b/packages/codemode/src/snapshot/codec.ts index 4345a199dfa..2f998e5da20 100644 --- a/packages/codemode/src/snapshot/codec.ts +++ b/packages/codemode/src/snapshot/codec.ts @@ -225,7 +225,13 @@ const builtinSpecs = (): ReadonlyArray => [ { name: "HeadersObj", ctor: HeadersObj, fields: ["headers"], obj: true, constants: { tag: "Headers" } }, { name: "URLObj", ctor: URLObj, fields: ["searchParams", "url"], obj: true, constants: { tag: "URL" } }, { name: "Bytes", ctor: Bytes, fields: ["bytes"], obj: true, constants: { tag: "Uint8Array" } }, - { name: "TextDecoderObj", ctor: TextDecoderObj, fields: ["decoder"], obj: true, constants: { tag: "Object" } }, + { + name: "TextDecoderObj", + ctor: TextDecoderObj, + fields: ["decoder", "ignoreBOM", "queued", "seen"], + obj: true, + constants: { tag: "Object" }, + }, { name: "AsyncFromSyncIterator", ctor: AsyncFromSyncIterator, diff --git a/packages/codemode/src/stdlib/bytes.ts b/packages/codemode/src/stdlib/bytes.ts index 815074471c4..c7b55b3415c 100644 --- a/packages/codemode/src/stdlib/bytes.ts +++ b/packages/codemode/src/stdlib/bytes.ts @@ -196,14 +196,74 @@ export const textEncoderGlobal = (ctx: Interpreter) => { }) } -/** A `TextDecoder` holding the host decoder its label and options configured. */ +// Replacement mode only needs to identify an incomplete suffix (at most three bytes). +// Fatal mode also needs the queue position after the first error, which the host doesn't expose. +const streamBoundary = (bytes: Uint8Array, fatal: boolean) => { + let next = fatal ? 0 : Math.max(0, bytes.length - 3) + while (next < bytes.length) { + const start = next + const lead = bytes[next++]! + if (lead < 0x80) continue + if (lead < 0xc2 || lead > 0xf4) { + if (fatal) return { end: next, error: true } + continue + } + const width = lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4 + let lower = lead === 0xe0 ? 0xa0 : lead === 0xf0 ? 0x90 : 0x80 + let upper = lead === 0xed ? 0x9f : lead === 0xf4 ? 0x8f : 0xbf + while (next < start + width) { + if (next === bytes.length) return { end: start, error: false } + const byte = bytes[next]! + // A bad continuation is reprocessed as a leading byte, even after a fatal error. + if (byte < lower || byte > upper) { + if (fatal) return { end: next, error: true } + break + } + next++ + lower = 0x80 + upper = 0xbf + } + } + return { end: next, error: false } +} + +const empty = new Uint8Array() + export class TextDecoderObj extends Obj { + queued: Uint8Array = empty + seen = false + constructor( proto: Obj, readonly decoder: TextDecoder, + readonly ignoreBOM: boolean, ) { super(proto) } + + decode(bytes: Uint8Array, stream: boolean) { + if (this.queued.length && bytes.length) { + checkArrayLength(this.queued.length + bytes.length) + const joined = new Uint8Array(this.queued.length + bytes.length) + joined.set(this.queued) + joined.set(bytes, this.queued.length) + bytes = joined + } + if (!bytes.length) bytes = this.queued + const { end, error } = stream ? streamBoundary(bytes, this.decoder.fatal) : { end: bytes.length, error: false } + this.queued = end < bytes.length ? bytes.slice(end) : empty + if (error) throw typeError("TextDecoder.decode: the input is not valid utf-8.") + const stripBOM = !this.ignoreBOM && !this.seen + if (!stream) this.seen = false + try { + // Keep the host decoder stateless: snapshots own the pending bytes and BOM decision. + const text = this.decoder.decode(end === bytes.length ? bytes : bytes.subarray(0, end)) + if (stream && text.length) this.seen = true + return stripBOM && text.charCodeAt(0) === 0xfeff ? text.slice(1) : text + } catch { + throw typeError("TextDecoder.decode: the input is not valid utf-8.") + } + } } // The WHATWG labels for UTF-8, the only encoding CodeMode decodes. @@ -215,22 +275,22 @@ export const textDecoderGlobal = (ctx: Interpreter) => { const self = (thisValue: Value, name: string) => receiver(TextDecoderObj, thisValue, `TextDecoder.prototype.${name}`) defineAccessor(proto, "encoding", (thisValue) => self(thisValue, "encoding").decoder.encoding) defineAccessor(proto, "fatal", (thisValue) => self(thisValue, "fatal").decoder.fatal) - defineAccessor(proto, "ignoreBOM", (thisValue) => self(thisValue, "ignoreBOM").decoder.ignoreBOM) + defineAccessor(proto, "ignoreBOM", (thisValue) => self(thisValue, "ignoreBOM").ignoreBOM) methods(builtins, proto, [ [ "decode", 0, (thisValue, args) => { - const decoder = self(thisValue, "decode").decoder - if (args[0] === undefined) return "" - if (!(args[0] instanceof Bytes)) { + const decoder = self(thisValue, "decode") + if (args[0] !== undefined && !(args[0] instanceof Bytes)) { throw typeError(`TextDecoder.decode expects a Uint8Array, received ${describeValue(args[0])}.`) } - try { - return decoder.decode(args[0].bytes) - } catch { - throw typeError(`TextDecoder.decode: the input is not valid ${decoder.encoding}.`) + const options = args[1] + if (options !== undefined && typeof options !== "object") { + throw typeError("TextDecoder.decode expects an options object.") } + const stream = options instanceof Obj && Boolean(get(options, "stream")) + return decoder.decode(args[0]?.bytes ?? empty, stream) }, ], ]) @@ -244,7 +304,8 @@ export const textDecoderGlobal = (ctx: Interpreter) => { const flag = (name: string) => options !== undefined && Boolean(get(options, name)) return new TextDecoderObj( prototypeFrom(newTarget, proto), - new TextDecoder("utf-8", { fatal: flag("fatal"), ignoreBOM: flag("ignoreBOM") }), + new TextDecoder("utf-8", { fatal: flag("fatal"), ignoreBOM: true }), + flag("ignoreBOM"), ) }, }) diff --git a/packages/codemode/test/snapshot.test.ts b/packages/codemode/test/snapshot.test.ts index 509b9f47e2d..0587bdfa45a 100644 --- a/packages/codemode/test/snapshot.test.ts +++ b/packages/codemode/test/snapshot.test.ts @@ -13,6 +13,14 @@ import { buildId } from "../src/snapshot.js" // The VM's snapshots: a run saved and restored into a fresh runtime anywhere it slices goes on as if it never stopped. const programs: Record = { + "a UTF-8 decoder with unread bytes after a fatal stream error": ` + const decoder = new TextDecoder('utf8', { fatal: true }) + try { decoder.decode(new Uint8Array([255, 65, 66, 67, 68]), { stream: true }) } + catch (error) { return [error.name, decoder.decode()] }`, + "a UTF-8 decoder with pending bytes and a consumed BOM": ` + const decoder = new TextDecoder('utf8', { fatal: true }) + const start = decoder.decode(new Uint8Array([0xef, 0xbb, 0xbf, 0xef]), { stream: true }) + return [start, decoder.decode(new Uint8Array([0xbb, 0xbf]))]`, "closures over per-iteration bindings": ` const fns = [] for (let i = 0; i < 5; i++) fns.push(() => i * 2) diff --git a/packages/codemode/test/stdlib.test.ts b/packages/codemode/test/stdlib.test.ts index 9c0d01672cc..ef09e2d3303 100644 --- a/packages/codemode/test/stdlib.test.ts +++ b/packages/codemode/test/stdlib.test.ts @@ -1218,6 +1218,86 @@ describe("TextEncoder and TextDecoder", () => { ]) }) + test("streams an emoji one byte at a time and copies the pending bytes", async () => { + expect( + await value(` + const decoder = new TextDecoder() + const chunk = new Uint8Array([65, 0xf0]) + const stream = { stream: true } + const start = decoder.decode(chunk, stream) + chunk[1] = 0 + return [start, decoder.decode(undefined, stream), decoder.decode(new Uint8Array([0x9f]), stream), + decoder.decode(new Uint8Array([0x98]), stream), decoder.decode(new Uint8Array([0x80, 66])), decoder.decode()] + `), + ).toEqual(["A", "", "", "", "😀B", ""]) + }) + + test("only strips the first BOM of each stream, including a split BOM", async () => { + expect( + await value(` + return [false, true].map(ignoreBOM => { + const decoder = new TextDecoder('utf8', { ignoreBOM }) + const bom = new Uint8Array([0xef, 0xbb, 0xbf]) + return [decoder.decode(bom.slice(0, 1), { stream: true }), decoder.decode(bom.slice(1), { stream: true }), + decoder.decode(bom), decoder.decode(bom)] + }) + `), + ).toEqual([ + ["", "", "\ufeff", ""], + ["", "\ufeff", "\ufeff", "\ufeff"], + ]) + }) + + test("reports malformed UTF-8 immediately and truncated UTF-8 on flush", async () => { + expect( + await value(` + return [false, true].map(fatal => { + return [[[0xe0], [0x80]], [[0xed, 0xa0]], [[0xf0, 0x80]], [[0xf4, 0x90]], + [[0xe2, 0x82], [65]], [[0xf0, 0x9f], undefined, undefined, [0xef, 0xbb, 0xbf, 66]]].flatMap(chunks => { + const decoder = new TextDecoder('utf8', { fatal }) + return chunks.map(bytes => { + try { return bytes ? decoder.decode(new Uint8Array(bytes), { stream: true }) : decoder.decode() } + catch (error) { return error.name } + }) + }) + }) + `), + ).toEqual([ + ["", "��", "��", "��", "��", "", "�A", "", "�", "", "B"], + ["", "TypeError", "TypeError", "TypeError", "TypeError", "", "TypeError", "", "TypeError", "", "B"], + ]) + }) + + test("keeps unread input in order after a fatal stream error", async () => { + expect( + await value(` + const decoder = new TextDecoder('utf8', { fatal: true }) + const stream = { stream: true } + const fail = bytes => { + try { return decoder.decode(bytes, stream) } catch (error) { return error.name } + } + const chunk = new Uint8Array([0xff, 65]) + const first = fail(chunk) + chunk[1] = 0 + return [first, decoder.decode(), fail(new Uint8Array([0xe2, 0x82])), + fail(new Uint8Array([66, 67])), decoder.decode(new Uint8Array([68])), decoder.decode()] + `), + ).toEqual(["TypeError", "A", "", "TypeError", "BCD", ""]) + }) + + test("rejects primitive options before touching the partial character", async () => { + expect( + await value(` + const decoder = new TextDecoder() + decoder.decode(new Uint8Array([0xc2]), { stream: true }) + const errors = [true, false, 0, '', Symbol.iterator].map(options => { + try { return decoder.decode(undefined, options) } catch (error) { return error.name } + }) + return [...errors, decoder.decode(new Uint8Array([0xa2]), null)] + `), + ).toEqual(["TypeError", "TypeError", "TypeError", "TypeError", "TypeError", "¢"]) + }) + test("only UTF-8 is supported; fatal decoding rejects malformed input", async () => { expect((await error(`new TextDecoder("latin1")`)).message).toContain('The "latin1" encoding is not supported') expect((await error(`new TextDecoder("utf-8", { fatal: true }).decode(new Uint8Array([0xff]))`)).message).toContain(