mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-20 18:57:17 +03:00
fix(codemode): decode UTF-8 across chunk boundaries (#63600)
This commit is contained in:
committed by
GitHub
Unverified
parent
8921769577
commit
a3d6e54b3d
@@ -596,14 +596,15 @@ with a hint to encode as text first (`TextDecoder`, `toBase64`, `toHex`).
|
||||
frozen. Functions, symbols, promises, iterators, URL, Headers, and tool references cannot be cloned; without a
|
||||
`DOMException`, the failure is a `TypeError` whose message starts with `DataCloneError:`.
|
||||
- [x] `TextEncoder` and `TextDecoder` for UTF-8 only: any other label is a `RangeError`. `TextDecoder` accepts the
|
||||
`fatal` and `ignoreBOM` options; `decode` takes a Uint8Array or nothing.
|
||||
`fatal` and `ignoreBOM` options. `decode(chunk, { stream: true })` accepts incremental Uint8Array input;
|
||||
`decode()` flushes the stream.
|
||||
- [x] `new Headers()` from records, synchronous iterables of pairs, and Headers, wrapping the host's `Headers`: names
|
||||
fold to lowercase, values are normalized and combined, and invalid names or values throw a `TypeError`.
|
||||
- [x] Headers `append`, `delete`, `get`, `getSetCookie`, `has`, `set`, `forEach`, `keys`, `values`, `entries`, and
|
||||
`[Symbol.iterator]`; iteration is live and sorted by name, with `set-cookie` values kept apart.
|
||||
- [x] Headers serialize to a `{ name: value }` object in JSON, in results, and in tool arguments.
|
||||
- [ ] `Request`, `Response`, and `Blob`.
|
||||
- [ ] `crypto.subtle` and `TextDecoder` streaming or non-UTF-8 encodings.
|
||||
- [ ] `crypto.subtle` and non-UTF-8 `TextDecoder` encodings.
|
||||
|
||||
## Extensions
|
||||
|
||||
|
||||
@@ -225,7 +225,13 @@ const builtinSpecs = (): ReadonlyArray<ClassSpec> => [
|
||||
{ name: "HeadersObj", ctor: HeadersObj, fields: ["headers"], obj: true, constants: { tag: "Headers" } },
|
||||
{ name: "URLObj", ctor: URLObj, fields: ["searchParams", "url"], obj: true, constants: { tag: "URL" } },
|
||||
{ name: "Bytes", ctor: Bytes, fields: ["bytes"], obj: true, constants: { tag: "Uint8Array" } },
|
||||
{ name: "TextDecoderObj", ctor: TextDecoderObj, fields: ["decoder"], obj: true, constants: { tag: "Object" } },
|
||||
{
|
||||
name: "TextDecoderObj",
|
||||
ctor: TextDecoderObj,
|
||||
fields: ["decoder", "ignoreBOM", "queued", "seen"],
|
||||
obj: true,
|
||||
constants: { tag: "Object" },
|
||||
},
|
||||
{
|
||||
name: "AsyncFromSyncIterator",
|
||||
ctor: AsyncFromSyncIterator,
|
||||
|
||||
@@ -196,14 +196,74 @@ export const textEncoderGlobal = <R>(ctx: Interpreter<R>) => {
|
||||
})
|
||||
}
|
||||
|
||||
/** A `TextDecoder` holding the host decoder its label and options configured. */
|
||||
// Replacement mode only needs to identify an incomplete suffix (at most three bytes).
|
||||
// Fatal mode also needs the queue position after the first error, which the host doesn't expose.
|
||||
const streamBoundary = (bytes: Uint8Array, fatal: boolean) => {
|
||||
let next = fatal ? 0 : Math.max(0, bytes.length - 3)
|
||||
while (next < bytes.length) {
|
||||
const start = next
|
||||
const lead = bytes[next++]!
|
||||
if (lead < 0x80) continue
|
||||
if (lead < 0xc2 || lead > 0xf4) {
|
||||
if (fatal) return { end: next, error: true }
|
||||
continue
|
||||
}
|
||||
const width = lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4
|
||||
let lower = lead === 0xe0 ? 0xa0 : lead === 0xf0 ? 0x90 : 0x80
|
||||
let upper = lead === 0xed ? 0x9f : lead === 0xf4 ? 0x8f : 0xbf
|
||||
while (next < start + width) {
|
||||
if (next === bytes.length) return { end: start, error: false }
|
||||
const byte = bytes[next]!
|
||||
// A bad continuation is reprocessed as a leading byte, even after a fatal error.
|
||||
if (byte < lower || byte > upper) {
|
||||
if (fatal) return { end: next, error: true }
|
||||
break
|
||||
}
|
||||
next++
|
||||
lower = 0x80
|
||||
upper = 0xbf
|
||||
}
|
||||
}
|
||||
return { end: next, error: false }
|
||||
}
|
||||
|
||||
const empty = new Uint8Array()
|
||||
|
||||
export class TextDecoderObj extends Obj {
|
||||
queued: Uint8Array = empty
|
||||
seen = false
|
||||
|
||||
constructor(
|
||||
proto: Obj,
|
||||
readonly decoder: TextDecoder,
|
||||
readonly ignoreBOM: boolean,
|
||||
) {
|
||||
super(proto)
|
||||
}
|
||||
|
||||
decode(bytes: Uint8Array, stream: boolean) {
|
||||
if (this.queued.length && bytes.length) {
|
||||
checkArrayLength(this.queued.length + bytes.length)
|
||||
const joined = new Uint8Array(this.queued.length + bytes.length)
|
||||
joined.set(this.queued)
|
||||
joined.set(bytes, this.queued.length)
|
||||
bytes = joined
|
||||
}
|
||||
if (!bytes.length) bytes = this.queued
|
||||
const { end, error } = stream ? streamBoundary(bytes, this.decoder.fatal) : { end: bytes.length, error: false }
|
||||
this.queued = end < bytes.length ? bytes.slice(end) : empty
|
||||
if (error) throw typeError("TextDecoder.decode: the input is not valid utf-8.")
|
||||
const stripBOM = !this.ignoreBOM && !this.seen
|
||||
if (!stream) this.seen = false
|
||||
try {
|
||||
// Keep the host decoder stateless: snapshots own the pending bytes and BOM decision.
|
||||
const text = this.decoder.decode(end === bytes.length ? bytes : bytes.subarray(0, end))
|
||||
if (stream && text.length) this.seen = true
|
||||
return stripBOM && text.charCodeAt(0) === 0xfeff ? text.slice(1) : text
|
||||
} catch {
|
||||
throw typeError("TextDecoder.decode: the input is not valid utf-8.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The WHATWG labels for UTF-8, the only encoding CodeMode decodes.
|
||||
@@ -215,22 +275,22 @@ export const textDecoderGlobal = <R>(ctx: Interpreter<R>) => {
|
||||
const self = (thisValue: Value, name: string) => receiver(TextDecoderObj, thisValue, `TextDecoder.prototype.${name}`)
|
||||
defineAccessor(proto, "encoding", (thisValue) => self(thisValue, "encoding").decoder.encoding)
|
||||
defineAccessor(proto, "fatal", (thisValue) => self(thisValue, "fatal").decoder.fatal)
|
||||
defineAccessor(proto, "ignoreBOM", (thisValue) => self(thisValue, "ignoreBOM").decoder.ignoreBOM)
|
||||
defineAccessor(proto, "ignoreBOM", (thisValue) => self(thisValue, "ignoreBOM").ignoreBOM)
|
||||
methods(builtins, proto, [
|
||||
[
|
||||
"decode",
|
||||
0,
|
||||
(thisValue, args) => {
|
||||
const decoder = self(thisValue, "decode").decoder
|
||||
if (args[0] === undefined) return ""
|
||||
if (!(args[0] instanceof Bytes)) {
|
||||
const decoder = self(thisValue, "decode")
|
||||
if (args[0] !== undefined && !(args[0] instanceof Bytes)) {
|
||||
throw typeError(`TextDecoder.decode expects a Uint8Array, received ${describeValue(args[0])}.`)
|
||||
}
|
||||
try {
|
||||
return decoder.decode(args[0].bytes)
|
||||
} catch {
|
||||
throw typeError(`TextDecoder.decode: the input is not valid ${decoder.encoding}.`)
|
||||
const options = args[1]
|
||||
if (options !== undefined && typeof options !== "object") {
|
||||
throw typeError("TextDecoder.decode expects an options object.")
|
||||
}
|
||||
const stream = options instanceof Obj && Boolean(get(options, "stream"))
|
||||
return decoder.decode(args[0]?.bytes ?? empty, stream)
|
||||
},
|
||||
],
|
||||
])
|
||||
@@ -244,7 +304,8 @@ export const textDecoderGlobal = <R>(ctx: Interpreter<R>) => {
|
||||
const flag = (name: string) => options !== undefined && Boolean(get(options, name))
|
||||
return new TextDecoderObj(
|
||||
prototypeFrom(newTarget, proto),
|
||||
new TextDecoder("utf-8", { fatal: flag("fatal"), ignoreBOM: flag("ignoreBOM") }),
|
||||
new TextDecoder("utf-8", { fatal: flag("fatal"), ignoreBOM: true }),
|
||||
flag("ignoreBOM"),
|
||||
)
|
||||
},
|
||||
})
|
||||
|
||||
@@ -13,6 +13,14 @@ import { buildId } from "../src/snapshot.js"
|
||||
// The VM's snapshots: a run saved and restored into a fresh runtime anywhere it slices goes on as if it never stopped.
|
||||
|
||||
const programs: Record<string, string> = {
|
||||
"a UTF-8 decoder with unread bytes after a fatal stream error": `
|
||||
const decoder = new TextDecoder('utf8', { fatal: true })
|
||||
try { decoder.decode(new Uint8Array([255, 65, 66, 67, 68]), { stream: true }) }
|
||||
catch (error) { return [error.name, decoder.decode()] }`,
|
||||
"a UTF-8 decoder with pending bytes and a consumed BOM": `
|
||||
const decoder = new TextDecoder('utf8', { fatal: true })
|
||||
const start = decoder.decode(new Uint8Array([0xef, 0xbb, 0xbf, 0xef]), { stream: true })
|
||||
return [start, decoder.decode(new Uint8Array([0xbb, 0xbf]))]`,
|
||||
"closures over per-iteration bindings": `
|
||||
const fns = []
|
||||
for (let i = 0; i < 5; i++) fns.push(() => i * 2)
|
||||
|
||||
@@ -1218,6 +1218,86 @@ describe("TextEncoder and TextDecoder", () => {
|
||||
])
|
||||
})
|
||||
|
||||
test("streams an emoji one byte at a time and copies the pending bytes", async () => {
|
||||
expect(
|
||||
await value(`
|
||||
const decoder = new TextDecoder()
|
||||
const chunk = new Uint8Array([65, 0xf0])
|
||||
const stream = { stream: true }
|
||||
const start = decoder.decode(chunk, stream)
|
||||
chunk[1] = 0
|
||||
return [start, decoder.decode(undefined, stream), decoder.decode(new Uint8Array([0x9f]), stream),
|
||||
decoder.decode(new Uint8Array([0x98]), stream), decoder.decode(new Uint8Array([0x80, 66])), decoder.decode()]
|
||||
`),
|
||||
).toEqual(["A", "", "", "", "😀B", ""])
|
||||
})
|
||||
|
||||
test("only strips the first BOM of each stream, including a split BOM", async () => {
|
||||
expect(
|
||||
await value(`
|
||||
return [false, true].map(ignoreBOM => {
|
||||
const decoder = new TextDecoder('utf8', { ignoreBOM })
|
||||
const bom = new Uint8Array([0xef, 0xbb, 0xbf])
|
||||
return [decoder.decode(bom.slice(0, 1), { stream: true }), decoder.decode(bom.slice(1), { stream: true }),
|
||||
decoder.decode(bom), decoder.decode(bom)]
|
||||
})
|
||||
`),
|
||||
).toEqual([
|
||||
["", "", "\ufeff", ""],
|
||||
["", "\ufeff", "\ufeff", "\ufeff"],
|
||||
])
|
||||
})
|
||||
|
||||
test("reports malformed UTF-8 immediately and truncated UTF-8 on flush", async () => {
|
||||
expect(
|
||||
await value(`
|
||||
return [false, true].map(fatal => {
|
||||
return [[[0xe0], [0x80]], [[0xed, 0xa0]], [[0xf0, 0x80]], [[0xf4, 0x90]],
|
||||
[[0xe2, 0x82], [65]], [[0xf0, 0x9f], undefined, undefined, [0xef, 0xbb, 0xbf, 66]]].flatMap(chunks => {
|
||||
const decoder = new TextDecoder('utf8', { fatal })
|
||||
return chunks.map(bytes => {
|
||||
try { return bytes ? decoder.decode(new Uint8Array(bytes), { stream: true }) : decoder.decode() }
|
||||
catch (error) { return error.name }
|
||||
})
|
||||
})
|
||||
})
|
||||
`),
|
||||
).toEqual([
|
||||
["", "��", "��", "��", "��", "", "�A", "", "�", "", "B"],
|
||||
["", "TypeError", "TypeError", "TypeError", "TypeError", "", "TypeError", "", "TypeError", "", "B"],
|
||||
])
|
||||
})
|
||||
|
||||
test("keeps unread input in order after a fatal stream error", async () => {
|
||||
expect(
|
||||
await value(`
|
||||
const decoder = new TextDecoder('utf8', { fatal: true })
|
||||
const stream = { stream: true }
|
||||
const fail = bytes => {
|
||||
try { return decoder.decode(bytes, stream) } catch (error) { return error.name }
|
||||
}
|
||||
const chunk = new Uint8Array([0xff, 65])
|
||||
const first = fail(chunk)
|
||||
chunk[1] = 0
|
||||
return [first, decoder.decode(), fail(new Uint8Array([0xe2, 0x82])),
|
||||
fail(new Uint8Array([66, 67])), decoder.decode(new Uint8Array([68])), decoder.decode()]
|
||||
`),
|
||||
).toEqual(["TypeError", "A", "", "TypeError", "BCD", ""])
|
||||
})
|
||||
|
||||
test("rejects primitive options before touching the partial character", async () => {
|
||||
expect(
|
||||
await value(`
|
||||
const decoder = new TextDecoder()
|
||||
decoder.decode(new Uint8Array([0xc2]), { stream: true })
|
||||
const errors = [true, false, 0, '', Symbol.iterator].map(options => {
|
||||
try { return decoder.decode(undefined, options) } catch (error) { return error.name }
|
||||
})
|
||||
return [...errors, decoder.decode(new Uint8Array([0xa2]), null)]
|
||||
`),
|
||||
).toEqual(["TypeError", "TypeError", "TypeError", "TypeError", "TypeError", "¢"])
|
||||
})
|
||||
|
||||
test("only UTF-8 is supported; fatal decoding rejects malformed input", async () => {
|
||||
expect((await error(`new TextDecoder("latin1")`)).message).toContain('The "latin1" encoding is not supported')
|
||||
expect((await error(`new TextDecoder("utf-8", { fatal: true }).decode(new Uint8Array([0xff]))`)).message).toContain(
|
||||
|
||||
Reference in New Issue
Block a user