diff --git a/packages/codemode/package.json b/packages/codemode/package.json index 6736f2415a7..34a41d3b3b3 100644 --- a/packages/codemode/package.json +++ b/packages/codemode/package.json @@ -24,8 +24,7 @@ "pack": "bun run script/pack.ts", "typecheck": "bun check --no-script", "typecheck:tsc": "tsc --noEmit", - "test": "bun test && bun run test:machine", - "test:machine": "CODEMODE_EVALUATOR=machine bun test" + "test": "bun test" }, "dependencies": { "acorn": "8.15.0", diff --git a/packages/codemode/src/data.ts b/packages/codemode/src/data.ts index b4aa3fe2ff8..6a09f748f17 100644 --- a/packages/codemode/src/data.ts +++ b/packages/codemode/src/data.ts @@ -21,7 +21,7 @@ import { HeadersObj, } from "./interpreter/objects.js" import { typeofValue } from "./interpreter/references.js" -import { type Eval, Slice } from "./interpreter/runner.js" +import { type Eval, Slice } from "./interpreter/signals.js" export type Json = Schema.Json @@ -50,7 +50,7 @@ const walk = ( const stack = new Set() const visit = function* (holder: Obj, key: string, depth: number): Eval { // Shared references expand into a tree, so the work is not bounded by the value's size or depth. - if (ctx.runner.tick()) yield Slice + if (ctx.machine.tick()) yield Slice if (depth > MAX_VALUE_DEPTH) throw rangeError(`Value exceeds the maximum depth of ${MAX_VALUE_DEPTH}.`) const raw = get(holder, key) if (raw instanceof PromiseObj && !boundary) { diff --git a/packages/codemode/src/interpreter/bridge.ts b/packages/codemode/src/interpreter/bridge.ts new file mode 100644 index 00000000000..b632734ffcf --- /dev/null +++ b/packages/codemode/src/interpreter/bridge.ts @@ -0,0 +1,378 @@ +import type { Expression } from "acorn" +import { ToolReference } from "../tool-runtime.js" +import { locate } from "./errors.js" +import type { Interpreter } from "./interpreter.js" +import type { Builtins } from "./intrinsics.js" +import { type AstNode, AsyncIteratorSymbol, GeneratorReturn, IteratorSymbol, Throw, typeError } from "./model.js" +import { native } from "./native.js" +import { + Builtin, + type Cursor, + define, + Fn, + get, + hidden, + IteratorObj, + iteratorResult, + type NativeCall, + type NativeResult, + Obj, + type PromiseObj, + type Step, + type Value, +} from "./objects.js" +import { + builtinCursor, + type CustomIterator, + notAFunction, + requireIteratorMethod, + requireIteratorObject, +} from "./operations.js" +import { capability } from "./promises.js" +import { describeValue } from "./references.js" +import { + Await, + type Eval, + type GeneratorRequest, + Interruption, + Invoke, + isEval, + normal, + Slice, + Yield, +} from "./signals.js" + +/** + * Interpreter work for the built-ins that still run as host generators in native frames: calls into the program, + * iterating, `yield*`, and the async-from-sync iterator adapter. A program function it calls is an `Invoke` the + * machine runs on its own stack. + * + * Phase 2c moves `yield*` and the async-from-sync adapter onto machine frames, and phase 3 replaces the built-ins that + * call program code with the prelude, after which this goes away. + */ +export class Bridge { + constructor(private readonly ctx: Interpreter) {} + + /** Await: suspends the innermost async body until `value` settles. */ + *awaitValue(value: Value, node?: AstNode): Eval { + return (yield new Await(value, node)) as Value + } + + *call(callable: Value, thisValue: Value, args: Array, node?: AstNode, callee?: Expression): Eval { + const result = this.dispatch(callable, thisValue, args, node, callee) + if (isEval(result)) return yield* result + return result instanceof Invoke ? ((yield result) as Value) : result + } + + // The single dispatch for every call a built-in makes. A program function is an `Invoke` for the machine to run on + // its stack; a built-in that returns a value needs no generator. + private dispatch( + callable: Value, + thisValue: Value, + args: Array, + node?: AstNode, + callee?: Expression, + ): Value | Eval | Invoke { + if (callable instanceof Fn) return new Invoke(callable, thisValue, args, node) + if (callable instanceof Builtin) { + const ctx = this.ctx + // A loop made only of built-in calls has no program call or loop back-edge to count, so count the call. + if (ctx.machine.tick()) + return afterSlice(() => this.native(() => callable.invoke(ctx, thisValue, args, node), node)) + return this.native(() => callable.invoke(ctx, thisValue, args, node), node) + } + if (callable instanceof ToolReference) { + if (callable.path.length === 0) { + throw typeError("The tools root is not callable.", callee ?? node) + } + return this.ctx.toolCall(callable.path, args, node) + } + throw notAFunction(callee, node) + } + + // Built-ins throw without a location, synchronously or inside their work; the call site supplies it. + private native(body: () => NativeResult, node?: AstNode): Value | Eval { + let result: NativeResult + try { + result = body() + } catch (error) { + throw locate(error, node) + } + return isEval(result) ? located(result, node) : result + } + + // CreateAsyncFromSyncIterator: an async iterator over a synchronous one. Each `next()` promise settles a job after + // the value does, and a rejected value closes the synchronous iterator first; `return()` calls the synchronous + // iterator's own `return`, or settles at once when it has none. + asyncFromSync( + builtin: Cursor | undefined, + custom: CustomIterator | undefined, + node: AstNode | undefined, + ): CustomIterator { + const ctx = this.ctx + const machine = ctx.machine + const builtins = ctx.builtins + const result = (value: Value, done: boolean) => iteratorResult(builtins.Object, value, done) + const self = this + // IteratorNext passes the value `next` received, when it received one. + const syncNext = function* (args: Array): Eval { + if (!custom) return yield* builtin!.next() + const object = requireIteratorObject( + yield* self.call(custom.next, custom.iterator, args, node), + "Iterator next() result", + node, + ) + return { done: Boolean(get(object, "done")), value: get(object, "value") } + } + // The synchronous iterator's `return`, looked up when it is needed; built-in collection iterators have none. + const syncReturn = (): Value => (custom ? get(custom.iterator, "return") : undefined) + const settle = (step: Step, closeOnRejection: boolean, made: ReturnType) => { + const rejected = (reason: unknown) => { + if (step.done || !closeOnRejection || reason instanceof Interruption) return made.reject(reason) + const body = function* (): Eval { + try { + if (custom) yield* self.closeIterator(custom, node) + else yield* builtin!.close() + } catch { + // The rejection wins over a failing close. + } + made.reject(reason) + } + return body() + } + let wrapper: PromiseObj + try { + wrapper = machine.promiseResolve(step.value) + } catch (error) { + const body = rejected(error) + if (body !== undefined) + machine.enqueue({ kind: "resume", frames: [machine.job(body)], input: normal(undefined) }) + return + } + machine.then(wrapper, { + kind: "callback", + run: (settled) => + settled.ok ? made.resolve(result(settled.value as Value, step.done)) : rejected(settled.error), + }) + } + const next = asyncFromSyncMethod(builtins, "next", function* (_, args) { + const made = capability(ctx, node) + try { + settle(yield* syncNext(args), true, made) + } catch (error) { + made.reject(error) + } + return made.promise + }) + const close = asyncFromSyncMethod(builtins, "return", function* (_, args) { + const made = capability(ctx, node) + try { + const method = syncReturn() + if (method === undefined || method === null) { + if (builtin !== undefined) yield* builtin.close() + made.resolve(result(args[0], true)) + return made.promise + } + const returned = requireIteratorObject( + yield* self.call(requireIteratorMethod(method, "Iterator return", node), custom!.iterator, args, node), + "Iterator return() result", + node, + ) + settle({ done: Boolean(get(returned, "done")), value: get(returned, "value") }, false, made) + } catch (error) { + made.reject(error) + } + return made.promise + }) + const raise = asyncFromSyncMethod(builtins, "throw", function* (_, args) { + const made = capability(ctx, node) + try { + const method = custom ? get(custom.iterator, "throw") : undefined + if (method === undefined || method === null) { + if (custom) yield* self.closeIterator(custom, node) + else yield* builtin!.close() + made.reject(typeError("The delegated iterator does not provide a throw() method.", node)) + return made.promise + } + const thrown = requireIteratorObject( + yield* self.call(requireIteratorMethod(method, "Iterator throw", node), custom!.iterator, args, node), + "Iterator throw() result", + node, + ) + settle({ done: Boolean(get(thrown, "done")), value: get(thrown, "value") }, true, made) + } catch (error) { + made.reject(error) + } + return made.promise + }) + const adapter = new Obj(builtins.Object) + define(adapter, "next", next, hidden) + define(adapter, "return", close, hidden) + define(adapter, "throw", raise, hidden) + return { iterator: adapter, next, asynchronous: true } + } + + *iterate(value: Value, node?: AstNode): Eval { + const cursor = builtinCursor(this.ctx.builtins, value) + if (cursor !== undefined) return cursor + const iterator = yield* this.customIterator(value, node, false) + return iterator === undefined ? undefined : this.customCursor(iterator, node) + } + + /** GetIteratorDirect: drive an iterator by its own `next`, without asking for `[Symbol.iterator]`. */ + iterateDirect(value: Value, node?: AstNode): Cursor { + if (value instanceof IteratorObj) return value.cursor + if (!(value instanceof Obj)) { + throw typeError(`An iterator must be an object, received ${describeValue(value)}.`, node) + } + return this.customCursor( + { + iterator: value, + next: requireIteratorMethod(get(value, "next"), "Iterator next", node), + asynchronous: false, + }, + node, + ) + } + + private customCursor(iterator: CustomIterator, node: AstNode | undefined): Cursor { + return { + next: () => this.nextIteratorResult(iterator, node), + close: () => this.closeIterator(iterator, node), + } + } + + private *customIterator( + value: Value, + node: AstNode | undefined, + allowAsync = true, + ): Eval { + if (!(value instanceof Obj)) return undefined + const asyncMethod = allowAsync ? get(value, AsyncIteratorSymbol) : undefined + const method = asyncMethod ?? get(value, IteratorSymbol) + if (method === undefined || method === null) return undefined + const iterator = yield* this.call(requireIteratorMethod(method, "Iterator method", node), value, [], node) + const object = requireIteratorObject(iterator, "Iterator method result", node) + return { + iterator: object, + next: requireIteratorMethod(get(object, "next"), "Iterator next", node), + asynchronous: asyncMethod !== undefined && asyncMethod !== null, + } + } + + private *nextIteratorResult(iterator: CustomIterator, node: AstNode | undefined): Eval { + const called = yield* this.call(iterator.next, iterator.iterator, [], node) + const object = requireIteratorObject( + iterator.asynchronous ? yield* this.awaitValue(called, node) : called, + "Iterator next() result", + node, + ) + return { done: Boolean(get(object, "done")), value: get(object, "value") } + } + + private *closeIterator(iterator: CustomIterator, node: AstNode | undefined): Eval { + const close = get(iterator.iterator, "return") + if (close === undefined || close === null) return + const method = requireIteratorMethod(close, "Iterator return", node) + const called = yield* this.call(method, iterator.iterator, [], node) + requireIteratorObject( + iterator.asynchronous ? yield* this.awaitValue(called, node) : called, + "Iterator return() result", + node, + ) + } + + // Yield `value` and continue with the request that resumes the body: `next` supplies the value of the `yield`, + // `throw` throws there, and `return` unwinds through `finally` blocks (an async generator awaits the value first). + private *suspendGenerator(value: Value, asynchronous: boolean): Eval { + const request = (yield new Yield(value)) as GeneratorRequest + if (request.kind === "next") return request.value + if (request.kind === "throw") throw new Throw(request.value) + throw new GeneratorReturn(asynchronous ? yield* this.awaitValue(request.value) : request.value) + } + + *delegateYield(value: Value, node: AstNode, asynchronous: boolean): Eval { + const builtin = builtinCursor(this.ctx.builtins, value) + if (builtin !== undefined && !asynchronous) { + while (true) { + const step = yield* builtin.next() + if (step.done) return undefined + try { + yield* this.suspendGenerator(step.value, asynchronous) + } catch (error) { + if (error instanceof GeneratorReturn) { + yield* builtin.close() + throw error + } + if (error instanceof Throw) { + yield* builtin.close() + throw typeError("The delegated iterator does not provide a throw() method.", node) + } + throw error + } + } + } + + const custom = builtin === undefined ? yield* this.customIterator(value, node, asynchronous) : undefined + if (builtin === undefined && !custom) throw typeError("yield* requires a compatible iterable value.", node) + // An async generator delegates to a synchronous iterator through CreateAsyncFromSyncIterator, as JS does. + const iterator = asynchronous && !custom?.asynchronous ? this.asyncFromSync(builtin, custom, node) : custom! + let kind: GeneratorRequest["kind"] = "next" + let input: Value = undefined + while (true) { + const method = kind === "next" ? iterator.next : get(iterator.iterator, kind) + if (method === undefined || method === null) { + if (kind === "return") throw new GeneratorReturn(asynchronous ? yield* this.awaitValue(input, node) : input) + yield* this.closeIterator(iterator, node) + throw typeError("The delegated iterator does not provide a throw() method.", node) + } + const called = yield* this.call( + requireIteratorMethod(method, `Iterator ${kind}`, node), + iterator.iterator, + [input], + node, + ) + const result = requireIteratorObject( + iterator.asynchronous ? yield* this.awaitValue(called, node) : called, + `Iterator ${kind}() result`, + node, + ) + const done = Boolean(get(result, "done")) + const resultValue = get(result, "value") + if (done) { + if (kind === "return") throw new GeneratorReturn(resultValue) + return resultValue + } + + try { + // Delegated values are yielded as they are; an async generator's inner promise already settled them. + input = yield* this.suspendGenerator(resultValue, asynchronous) + kind = "next" + } catch (error) { + if (!(error instanceof GeneratorReturn) && !(error instanceof Throw)) throw error + kind = error instanceof GeneratorReturn ? "return" : "throw" + input = error.value + } + } + } +} + +// The async-from-sync adapter still keeps its iterator in host closures, so its methods are made per iteration. +const asyncFromSyncMethod = (builtins: Builtins, name: string, call: NativeCall) => + native(builtins, { name, length: 1, call, runtime: "async-from-sync" }) + +// Finishes a built-in's work, locating a failure at the call that started it. +function* located(work: Eval, node: AstNode | undefined): Eval { + try { + return yield* work + } catch (error) { + throw locate(error, node) + } +} + +// Lets the host run first, then starts the work. +function* afterSlice(start: () => Value | Eval): Eval { + yield Slice + const result = start() + return isEval(result) ? yield* result : result +} diff --git a/packages/codemode/src/interpreter/callback.ts b/packages/codemode/src/interpreter/callback.ts index c5319c22967..4de68426071 100644 --- a/packages/codemode/src/interpreter/callback.ts +++ b/packages/codemode/src/interpreter/callback.ts @@ -14,7 +14,7 @@ import { type Value, } from "./objects.js" import { isOpaque, typeofValue } from "./references.js" -import { type Eval, Interruption, isEval } from "./runner.js" +import { type Eval, Interruption, isEval } from "./signals.js" /** IteratorClose: a consumer failure closes the iterator and wins over any close failure, except that a generator's * return() is a return completion, so a failing close wins over it, as after `break`. */ diff --git a/packages/codemode/src/interpreter/driver.ts b/packages/codemode/src/interpreter/driver.ts new file mode 100644 index 00000000000..074d4571033 --- /dev/null +++ b/packages/codemode/src/interpreter/driver.ts @@ -0,0 +1,128 @@ +import { Cause, type Context, Effect, Exit, Fiber } from "effect" +import type { Machine } from "../machine/machine.js" +import type { HostCall } from "../machine/scheduler.js" +import type { ToolRuntime } from "../tool-runtime.js" +import { locate } from "./errors.js" +import { PendingThrow } from "./model.js" +import { abrupt, type Completion, Interruption, normal } from "./signals.js" + +/** + * Runs a machine in Effect: the only place program execution meets the host. It calls `machine.run` until the program + * ends, and does what each stop asks: yields to the Effect scheduler for a slice, and starts the host calls the machine + * queued, as fibers with the caller's services, reporting each outcome back with `machine.complete`. Fibers live only + * here, by host call id, and never become part of the machine's state. + */ +export class Driver { + private readonly fibers = new Map>() + private wake: (() => void) | undefined + private stopping = false + + constructor( + private readonly machine: Machine, + private readonly tools: ToolRuntime, + /** The services host work runs with: the caller's, so tools see its context, tracing, and clock. */ + private readonly services: Context.Context, + ) {} + + /** + * Runs the machine until the program's body completes, then returns how it ended. Host work it started may still be + * running; `interrupt` stops it. Host work interrupted on its own that reaches the program's body, or blocking host + * work that is interrupted, interrupts the execution. + */ + run(): Effect.Effect { + const self = this + const machine = this.machine + return Effect.gen(function* () { + while (true) { + const stop = machine.run() + if (stop.kind === "slice") { + yield* Effect.yieldNow + continue + } + if (stop.kind === "host") { + if (stop.calls.length === 0) { + yield* self.settled + continue + } + for (const call of stop.calls) { + if (!call.blocking) { + self.start(call) + continue + } + const exit = yield* Effect.exit(self.work(call)) + if (Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) return yield* Effect.failCause(exit.cause) + machine.complete(call.id, Exit.isSuccess(exit) ? normal(exit.value) : abrupt(Cause.squash(exit.cause))) + } + continue + } + if (stop.kind === "stuck") { + return abrupt( + new PendingThrow( + "Error", + "The program is waiting on a promise that can never settle: nothing else is running, so nothing can resolve it. Code Mode has no timers or sleep, so don't wait or poll inside a program.", + stop.node, + ), + ) + } + const outcome = stop.completion + if (!outcome.ok && outcome.error instanceof Interruption) return yield* Effect.failCause(outcome.error.cause) + return outcome + } + }) + } + + /** Stops host work still running, waiting for its cleanup. */ + interrupt(): Effect.Effect { + const self = this + return Effect.gen(function* () { + self.stopping = true + while (self.fibers.size > 0) yield* Fiber.interruptAll([...self.fibers.values()]) + }) + } + + private work(call: HostCall): Effect.Effect { + const request = call.request + if (request.kind === "tool") return this.tools.execute(request.path, request.args) + if (request.kind === "search") return this.tools.search(request.args) + // An extension's work, made by the extension bridge with the program's services in its requirements. + return request.effect as Effect.Effect + } + + // Starts concurrent host work now; its outcome goes back to the machine's inbox, in completion order. + private start(call: HostCall): void { + // Starts synchronously, so admission and hooks happen before the machine runs another job. + const fiber = Effect.runForkWith(this.services)(this.work(call)) + this.fibers.set(call.id, fiber) + fiber.addObserver((exit) => { + this.fibers.delete(call.id) + // Interrupted by the program ending: nothing is left to settle. + if (this.stopping) return + this.machine.complete( + call.id, + Exit.isSuccess(exit) + ? normal(exit.value) + : abrupt( + Cause.hasInterruptsOnly(exit.cause) + ? new Interruption(exit.cause) + : Cause.hasDies(exit.cause) + ? locate(Cause.squash(exit.cause), call.site) + : Cause.squash(exit.cause), + ), + ) + this.wake?.() + }) + } + + private get settled(): Effect.Effect { + return Effect.callback((resume) => { + if (this.machine.ready) return resume(Effect.void) + this.wake = () => { + this.wake = undefined + resume(Effect.void) + } + return Effect.sync(() => { + this.wake = undefined + }) + }) + } +} diff --git a/packages/codemode/src/interpreter/errors.ts b/packages/codemode/src/interpreter/errors.ts index 34d9eb5b332..2896b1ed213 100644 --- a/packages/codemode/src/interpreter/errors.ts +++ b/packages/codemode/src/interpreter/errors.ts @@ -19,7 +19,7 @@ import { type Value, } from "./objects.js" import type { Interpreter } from "./interpreter.js" -import type { Eval } from "./runner.js" +import type { Eval } from "./signals.js" import { toPrimitiveString } from "./callback.js" import { formatValue } from "../stdlib/console.js" diff --git a/packages/codemode/src/interpreter/execute.ts b/packages/codemode/src/interpreter/execute.ts index f2969d85641..becfb04c56b 100644 --- a/packages/codemode/src/interpreter/execute.ts +++ b/packages/codemode/src/interpreter/execute.ts @@ -7,9 +7,9 @@ import { createBuiltins } from "./intrinsics.js" import { Interpreter } from "./interpreter.js" import type { Value } from "./objects.js" import { formatLocation, sourceLocation } from "./model.js" -import type { Runner } from "./runner.js" +import { Driver } from "./driver.js" +import type { Machine } from "../machine/machine.js" import { Source } from "./source.js" -import type { Evaluator } from "../machine/machine.js" export const executeProgram = ( code: string, @@ -33,24 +33,25 @@ export const executeProgram = ( const logs: Array = [] const logged = () => (logs.length > 0 ? { logs: [...logs] } : {}) // Set only after copy-out so timeouts cannot report invalid values as completed. - let returned: { value: DataValue; runner: Runner } | undefined - let runner: Runner | undefined + let returned: { value: DataValue; machine: Machine } | undefined + let driver: Driver | undefined const base = Effect.gen(function* () { const parsed = parseProgram(code) if (!parsed.ok) return { ok: false, error: parsed.error, toolCalls: [] } satisfies Result const services = yield* Effect.context() - const ctx = new Interpreter({ tools, services, builtins, logs, globals, evaluator: evaluator() }) - runner = ctx.runner - const outcome = yield* runner.run(ctx.main(parsed.source)) + const ctx = new Interpreter({ tools, builtins, logs, globals }) + ctx.start(parsed.source) + driver = new Driver(ctx.machine, tools, services) + const outcome = yield* driver.run() if (!outcome.ok) { return { ok: false, error: normalizeError(outcome.error), ...logged(), toolCalls: tools.calls } satisfies Result } const result = (outcome.value as DataValue | undefined) ?? null - returned = { value: result, runner } + returned = { value: result, machine: ctx.machine } // Host work the program left running is interrupted, not awaited. - yield* runner.interrupt() - const warnings = runner.diagnostics() + yield* driver.interrupt() + const warnings = ctx.machine.diagnostics() return { ok: true, value: result, @@ -58,7 +59,7 @@ export const executeProgram = ( ...logged(), toolCalls: tools.calls, } satisfies Result - }).pipe(Effect.ensuring(Effect.suspend(() => runner?.interrupt() ?? Effect.void))) + }).pipe(Effect.ensuring(Effect.suspend(() => driver?.interrupt() ?? Effect.void))) const timeoutMs = limits.timeoutMs const operation = timeoutMs === undefined @@ -85,7 +86,7 @@ export const executeProgram = ( kind: "TimeoutExceeded", message: `The program returned, but background work was still running at the ${timeoutMs}ms timeout and was interrupted. Await all started promises.`, }, - ...returned.runner.diagnostics(), + ...returned.machine.diagnostics(), ], ...logged(), toolCalls: tools.calls, @@ -112,9 +113,6 @@ export const executeProgram = ( }) } -// Internal while the machine evaluator is ported: `CODEMODE_EVALUATOR=machine` runs programs on it. -const evaluator = (): Evaluator => (process.env.CODEMODE_EVALUATOR === "machine" ? "machine" : "legacy") - export const parseProgram = ( code: string, ): { ok: true; program: Program; source: Source } | { ok: false; error: Diagnostic } => { diff --git a/packages/codemode/src/interpreter/extensions.ts b/packages/codemode/src/interpreter/extensions.ts index 22218606b14..7ac50eadd58 100644 --- a/packages/codemode/src/interpreter/extensions.ts +++ b/packages/codemode/src/interpreter/extensions.ts @@ -28,6 +28,7 @@ import { type Value, } from "./objects.js" import { describeValue, isOpaque } from "./references.js" +import { Host } from "./signals.js" /** * The global bindings of one run's extensions. Everything crossing the boundary is converted: plain data and built-in @@ -184,18 +185,18 @@ export const extensionGlobals = ( label, site, ) - if (before) yield ctx.runner.blocking(before) + if (before) yield new Host({ kind: "effect", effect: before }) const host = callHost(value, args) if (host.async) return pending(settle(host.effect), label, site) - return (yield ctx.runner.blocking(settle(host.effect))) as Value + return (yield new Host({ kind: "effect", effect: settle(host.effect) })) as Value }, }) } // An asynchronous host call: the program holds a promise that settles when the host's work does. const pending = (work: Effect.Effect, label: string, site: AstNode | undefined) => { - const resolving = ctx.runner.resolving(ctx.runner.newPromise(), site) - ctx.runner.host(work, { label, result: "value", resolving, site }) + const resolving = ctx.machine.resolving(ctx.machine.newPromise(), site) + ctx.machine.host({ kind: "effect", effect: work }, { label, result: "value", resolving, site }) return resolving.promise } diff --git a/packages/codemode/src/interpreter/generators.ts b/packages/codemode/src/interpreter/generators.ts index aee63489aa0..f13d6067dd6 100644 --- a/packages/codemode/src/interpreter/generators.ts +++ b/packages/codemode/src/interpreter/generators.ts @@ -5,13 +5,14 @@ import type { Interpreter } from "./interpreter.js" import { abrupt, type Boundary, + Call, type Completion, Enter, - type Entry, type Eval, type GeneratorRequest, normal, -} from "./runner.js" +} from "./signals.js" +import type { Frame } from "../machine/frame.js" /** `next`/`return`/`throw` on the generator prototypes; async generators answer with promises. */ export const generatorGlobals = (ctx: Interpreter): void => { @@ -49,17 +50,15 @@ export const generatorGlobals = (ctx: Interpreter): void => { const result = (ctx: Interpreter, value: Value, done: boolean) => iteratorResult(ctx.builtins.Object, value, done) -// Puts the suspended body back on the stack and continues it with `request`; the boundary answers the caller when -// the body next yields or ends. -const resume = (ctx: Interpreter, generator: GeneratorObj, request: GeneratorRequest): Enter => { +// Starts the body behind its boundary, or puts the suspended body's frames back on the stack and continues it with +// `request`; the boundary answers the caller when the body next yields or ends. +const resume = (ctx: Interpreter, generator: GeneratorObj, request: GeneratorRequest): Call | Enter => { const started = generator.state !== "suspendedStart" generator.state = "executing" - if (!started) { - return new Enter([{ body: generator.body, boundary: new GeneratorBoundary(ctx, generator) }], normal(undefined)) - } - const entries = generator.entries! - generator.entries = undefined - return new Enter(entries, normal(request)) + if (!started) return new Call(generator.body, new GeneratorBoundary(ctx, generator)) + const frames = generator.frames! + generator.frames = undefined + return new Enter(frames, normal(request)) } // A synchronous generator step: the body runs on the caller's stack until it yields, returns, or throws. @@ -76,9 +75,9 @@ function* step(ctx: Interpreter, generator: GeneratorObj, kind: GeneratorR // AsyncGenerator.prototype.next/return/throw: queue the request with its promise, and run the body now if it is // waiting for one. A request that arrives while the body runs is answered when the body reaches it. function* request(ctx: Interpreter, thisValue: Value, kind: GeneratorRequestKind, value: Value): Eval { - const runner = ctx.runner - const promise = runner.newPromise() - const resolving = runner.resolving(promise) + const machine = ctx.machine + const promise = machine.newPromise() + const resolving = machine.resolving(promise) if (!(thisValue instanceof GeneratorObj) || !thisValue.asynchronous) { resolving.reject(typeError(`AsyncGenerator.prototype.${kind} called on incompatible receiver.`)) return promise @@ -138,9 +137,9 @@ const drainQueue = (ctx: Interpreter, generator: GeneratorObj) => { // AsyncGeneratorAwaitReturn: a `return(value)` on an ended or unstarted generator settles once `value` does. const awaitReturn = (ctx: Interpreter, generator: GeneratorObj) => { - const runner = ctx.runner + const machine = ctx.machine try { - runner.then(runner.promiseResolve(generator.queue[0]!.request.value), { kind: "awaitReturn", generator }) + machine.then(machine.promiseResolve(generator.queue[0]!.request.value), { kind: "awaitReturn", generator }) } catch (error) { finishAwaitReturn(ctx, generator, abrupt(error)) } @@ -179,10 +178,10 @@ class GeneratorBoundary implements Boundary { return normal(undefined) } - yielded(value: Value, split: () => Array): Completion { + yielded(value: Value, split: () => Array): Completion { const generator = this.target if (!generator.asynchronous) { - generator.entries = split() + generator.frames = split() generator.state = "suspendedYield" return normal(result(this.ctx, value, false)) } @@ -190,7 +189,7 @@ class GeneratorBoundary implements Boundary { completeStep(this.ctx, generator, normal(value), false) const next = generator.queue[0] if (next !== undefined) return normal(next.request) - generator.entries = split() + generator.frames = split() generator.state = "suspendedYield" return normal(undefined) } diff --git a/packages/codemode/src/interpreter/interpreter.ts b/packages/codemode/src/interpreter/interpreter.ts index 60028d2190e..53a9d5c9b4e 100644 --- a/packages/codemode/src/interpreter/interpreter.ts +++ b/packages/codemode/src/interpreter/interpreter.ts @@ -1,216 +1,56 @@ -import type { - ArrayExpression, - ArrayPattern, - AssignmentPattern, - AssignmentExpression, - AssignmentProperty, - AwaitExpression, - BinaryExpression, - BlockStatement, - BreakStatement, - CallExpression, - ChainExpression, - ConditionalExpression, - ContinueStatement, - DoWhileStatement, - Expression, - ForInStatement, - ForOfStatement, - ForStatement, - Identifier, - IfStatement, - ImportExpression, - LabeledStatement, - LogicalExpression, - MemberExpression, - ModuleDeclaration, - NewExpression, - ObjectExpression, - Pattern, - Program, - Property, - ReturnStatement, - SequenceExpression, - SpreadElement, - Statement, - SwitchStatement, - TaggedTemplateExpression, - TemplateLiteral, - ThrowStatement, - TryStatement, - UnaryExpression, - UpdateExpression, - VariableDeclaration, - WhileStatement, - YieldExpression, -} from "acorn" -import type { Context } from "effect" +import type { TaggedTemplateExpression } from "acorn" import { fromJson, type Json, toBoundary } from "../data.js" -import { ToolReference, type ToolRuntime } from "../tool-runtime.js" -import { - type AstNode, - AsyncIteratorSymbol, - type Binding, - GeneratorReturn, - IteratorSymbol, - OptionalShortCircuit, - invalidData, - PendingThrow, - syntaxError, - Throw, - rangeError, - type StatementResult, - typeError, - unsupportedSyntax, -} from "./model.js" -import { checkStringLength } from "./limits.js" -import { locate, materialize } from "./errors.js" -import type { Builtins } from "./intrinsics.js" +import type { ToolRuntime } from "../tool-runtime.js" +import { Machine } from "../machine/machine.js" +import { Bridge } from "./bridge.js" import { globals } from "./globals.js" -import { - assign, - define, - get, - IteratorObj, - type Cursor, - has, - hidden, - Builtin, - nativeRule, - Arguments, - Arr, - Fn, - GeneratorObj, - Obj, - PromiseObj, - iteratorResult, - remove, - coerceToNumber, - coerceToString, - entries, - type NativeCall, - type NativeResult, - type Step, - type Value, -} from "./objects.js" -import { type Hint, preserveConsumerError, toPrimitive } from "./callback.js" -import { - AsyncBoundary, - Await, - type Body, - Call, - type Eval, - generatorBody, - type GeneratorRequest, - Interruption, - Invoke, - isEval, - MAX_CALL_DEPTH, - Runner, - Slice, - Task, - Yield, - normal, -} from "./runner.js" -import { moduleName, resolveModule } from "./modules.js" -import { describeValue, isOpaque, typeofValue } from "./references.js" -import { native } from "./native.js" -import { capability, promiseLike } from "./promises.js" -import { ScopeStack } from "./scope.js" -import { type Evaluator, Machine } from "../machine/machine.js" +import type { Builtins } from "./intrinsics.js" import { IntrinsicNames } from "./intrinsic-names.js" +import { type AstNode, type Binding, PendingThrow } from "./model.js" +import { moduleName, resolveModule } from "./modules.js" +import { type Arr, type Cursor, define, entries, nativeRule, Obj, type Value } from "./objects.js" +import type { CustomIterator } from "./operations.js" +import { promiseLike } from "./promises.js" +import { ScopeStack } from "./scope.js" +import { AsyncBoundary, Call, type Eval, Host } from "./signals.js" import type { NodeId, Source } from "./source.js" -import { constructRegExp } from "../stdlib/regexp.js" -import { enumerableSource } from "../stdlib/object.js" -import { - applyBinaryOperator, - assignToReference, - bindImport, - builtinCursor, - collectPatternNames, - createFunction, - createsClosure, - type CustomIterator, - declaresLexically, - destructuredProperty, - enumerableKeys, - hoistFunctions, - hoistVars, - instanceofValue, - isAbrupt, - isAnonymousFunction, - isSimple, - isThenable, - literal, - loopDeclaration, - loopExit, - type MemberReference, - none, - notAFunction, - notConstructible, - operatorConversion, - predeclareLexical, - predeclarePattern, - propertyKey, - readReference, - requireIteratorMethod, - requireIteratorObject, - requireMemberBase, - resolveProperty, - templateObject, - usesArguments, - writableReference, -} from "./operations.js" -import { compoundOperators } from "../stdlib/value.js" -/** Marks an operand a fast path has not evaluated yet, since `undefined` is a value. */ -const unevaluated: unique symbol = Symbol("codemode.unevaluated") -type Reference = MemberReference | ToolReference | { value: Value } | typeof OptionalShortCircuit - -/** One program execution: the tool bridge, job queue, captured logs, and the global scope built once. */ +/** One program execution: the tool bridge, the machine that runs it, captured logs, and the global scope built once. */ export class Interpreter { readonly tools: ToolRuntime - readonly runner: Runner readonly builtins: Builtins readonly logs: Array + /** Runs program code, and keeps the job queue, promises, and host calls the program waits on. */ + readonly machine: Machine /** Names for this runtime's intrinsics, taken before any program code can change a built-in. */ readonly intrinsics: IntrinsicNames /** Template objects by site: a tag sees the same `strings` array every time its literal is evaluated, as in JS. */ readonly templates = new WeakMap() /** The parsed sources this execution runs, by id, so a `NodeId` resolves to its node. */ readonly sources = new Map() - /** The machine evaluator, when this execution runs on it instead of the generator evaluator. */ - readonly machine: Machine | undefined - private readonly root: Frame + private readonly bridge: Bridge private readonly globalScope: Map private readonly modules = new Map() private readonly namespaces = new Map() constructor(options: { readonly tools: ToolRuntime - /** The caller's services, which host work runs with. */ - readonly services: Context.Context readonly builtins: Builtins readonly logs?: Array readonly globals?: (ctx: Interpreter) => ReadonlyArray - /** Which evaluator runs program code; the generator evaluator unless the machine is asked for. */ - readonly evaluator?: Evaluator }) { this.tools = options.tools this.builtins = options.builtins this.logs = options.logs ?? [] - this.runner = new Runner(this, options.services) - // The machine never runs host work with the caller's services itself; it asks the runner, which is typed by them. - this.machine = options.evaluator === "machine" ? new Machine(this as Interpreter) : undefined + // The machine never runs host work itself, so it does not need the services that work runs with. + this.machine = new Machine(this as Interpreter) + this.bridge = new Bridge(this) // Program code has no receiver: top-level `this` is undefined, as in a module. - const globalScope = new Map([["this", { mutable: false, value: undefined }]]) - // Calling back into the program never reads frame state, so any frame serves; the root is always alive. - this.root = new Frame(this, new ScopeStack([globalScope])) - this.globalScope = globalScope + this.globalScope = new Map([["this", { mutable: false, value: undefined }]]) const intrinsics = globals(this) this.intrinsics = new IntrinsicNames(intrinsics, this.builtins) for (const [name, value] of [...intrinsics, ...(options.globals?.(this) ?? [])]) { - globalScope.set(name, { mutable: false, value }) + this.globalScope.set(name, { mutable: false, value }) } nativeRule.sealed.add(this.builtins.Function) } @@ -250,14 +90,10 @@ export class Interpreter { return namespace } - /** The program's body, as the first job: its completion value, copied out for the host. */ - main(source: Source): Body { + /** Queues the program's body as the first job; its completion value is copied out for the host. */ + start(source: Source): void { this.sources.set(source.id, source) - return this.machine === undefined ? this.legacyMain(source) : this.machine.program(source) - } - - private *legacyMain(source: Source): Eval { - return yield* toBoundary(this, yield* this.root.run(source.program)) + this.machine.start(source) } /** A new chain over the global scope, where the program's own scope is pushed. */ @@ -265,47 +101,29 @@ export class Interpreter { return new ScopeStack([this.globalScope]) } - /** Starts a program call the runner reached outside the machine as its own activation. */ - enter(invoke: Invoke): Call { - if (this.machine === undefined) throw new TypeError("Invoke reached the runner without the machine evaluator.") - return this.machine.enter(invoke) - } - // The call returns its promise at once. Arguments cross on the caller's stack, since `toJSON` is program code, and a - // promise argument is awaited first; the host call starts once they have crossed, so admission and hooks happen at - // the call site, and runs concurrently with the program. + // promise argument is awaited first; the host call is queued once they have crossed, and the driver starts it, so + // admission and hooks happen before any other job runs, and it runs concurrently with the program. *toolCall(path: ReadonlyArray, args: Array, site: AstNode | undefined): Eval { const ctx = this - const runner = ctx.runner - const promise = runner.newPromise() - const resolving = runner.resolving(promise) + const machine = ctx.machine + const resolving = machine.resolving(machine.newPromise()) const body = function* (): Eval { const json: Array = [] for (const arg of args) json.push(yield* toBoundary(ctx, arg)) - runner.host(ctx.tools.execute(path, json), { - label: path.join("."), - args: json, - result: "json", - resolving, - site, - }) + machine.host({ kind: "tool", path, args: json }, { label: path.join("."), result: "json", resolving, site }) } - return (yield new Call( - body(), - new AsyncBoundary((completion) => { - if (!completion.ok) resolving.reject(completion.error) - }, promise), - )) as Value + return (yield new Call(body(), new AsyncBoundary(resolving))) as Value } - /** `yield*` in a generator body, as interpreter work; the machine evaluator runs it as a native frame for now. */ + /** `yield*` in a generator body, as interpreter work; the machine runs it as a native frame for now. */ delegateYield(value: Value, node: AstNode, asynchronous: boolean): Eval { - return this.root.delegateYield(value, node, asynchronous) + return this.bridge.delegateYield(value, node, asynchronous) } /** CreateAsyncFromSyncIterator, for `for await` over a synchronous iterable. */ asyncFromSync(builtin: Cursor | undefined, custom: CustomIterator | undefined, node: AstNode | undefined) { - return this.root.asyncFromSync(builtin, custom, node) + return this.bridge.asyncFromSync(builtin, custom, node) } /** The node a `NodeId` names, in one of this execution's sources. */ @@ -315,1744 +133,26 @@ export class Interpreter { /** Calls into the program; a built-in passes its own call site so failures inside are located there. */ call(callable: Value, thisValue: Value, args: Array, site?: AstNode): Eval { - return this.root.call(callable, thisValue, args, site) + return this.bridge.call(callable, thisValue, args, site) } /** Await: suspends the innermost async body until `value` settles. */ - *awaitValue(value: Value, node?: AstNode): Eval { - return (yield new Await(value, node)) as Value + awaitValue(value: Value, node?: AstNode): Eval { + return this.bridge.awaitValue(value, node) } iterate(value: Value) { - return this.root.iterate(value) + return this.bridge.iterate(value) } iterateDirect(value: Value) { - return this.root.iterateDirect(value) - } - - /** Runs one host call to completion before any other job: arguments cross as JSON, the result comes back as values. */ - *tool(run: ToolRuntime["search"], args: Array) { - const json: Array = [] - for (const arg of args) json.push(yield* toBoundary(this, arg)) - return fromJson(this, (yield this.runner.blocking(run(json))) as Json | undefined) + return this.bridge.iterateDirect(value) } /** The built-in search: its result is usable directly, and can also be awaited or chained like a Promise. */ *search(args: Array) { - return promiseLike(this, yield* this.tool(this.tools.search, args)) + const json: Array = [] + for (const arg of args) json.push(yield* toBoundary(this, arg)) + return promiseLike(this, fromJson(this, (yield new Host({ kind: "search", args: json })) as Json | undefined)) } } - -/** One activation: the top-level program or a single function call, evaluating against its own scope chain. */ -class Frame { - /** The body of a generator function, where `yield` suspends. */ - private generator = false - private generatorAsync = false - - constructor( - private readonly ctx: Interpreter, - private scopes: ScopeStack, - ) {} - - *run(program: Program): Eval { - // Keep top-level declarations separate so they can shadow builtins. - this.scopes.push() - try { - // Imports load before any code runs, as in a module. - for (const statement of program.body) - if (statement.type === "ImportDeclaration") bindImport(this.ctx, this.scopes, statement) - predeclareLexical(this.scopes, program.body) - hoistFunctions(this.ctx.builtins, this.scopes, program.body) - hoistVars(this.scopes, program.body) - let value: Value = undefined - for (const [index, statement] of program.body.entries()) { - if (statement.type === "ImportDeclaration") continue - if (index === program.body.length - 1 && statement.type === "ExpressionStatement") { - value = yield* this.evaluateExpression(statement.expression) - break - } - const result = yield* this.evaluateStatement(statement) - - if (result.kind === "return") { - value = result.value - break - } - - if (result.kind === "break" || result.kind === "continue") { - throw typeError(`Unexpected '${result.kind}' outside of a loop.`, statement) - } - } - - // The implicit async body adopts returned promises before copy-out. - return isThenable(value) ? yield* this.awaitValue(value) : value - } finally { - this.scopes.pop() - } - } - - awaitValue(value: Value, node?: AstNode): Eval { - return this.ctx.awaitValue(value, node) - } - - private *evaluateStatement(node: Statement | ModuleDeclaration): Eval { - const result = this.execute(node) - return isEval(result) ? yield* result : result - } - - // Like `evaluate`: statements that cannot run program code complete without a generator. - private execute(node: Statement | ModuleDeclaration): StatementResult | Eval { - switch (node.type) { - case "ExpressionStatement": { - const result = this.evaluate(node.expression) - return isEval(result) ? this.finishExpressionStatement(result) : none - } - case "VariableDeclaration": - return this.evaluateVariableStatement(node) - case "ReturnStatement": - return this.evaluateReturnStatement(node) - case "BlockStatement": - return this.evaluateBlock(node) - case "IfStatement": - return this.evaluateIfStatement(node) - case "SwitchStatement": - return this.evaluateSwitchStatement(node) - case "LabeledStatement": - return this.evaluateLabeledStatement(node) - case "WhileStatement": - return this.evaluateWhileStatement(node) - case "DoWhileStatement": - return this.evaluateDoWhileStatement(node) - case "ForStatement": - return this.evaluateForStatement(node) - case "ForOfStatement": - return this.evaluateForOfStatement(node) - case "ForInStatement": - return this.evaluateForInStatement(node) - case "BreakStatement": - return this.evaluateBreakStatement(node) - case "ContinueStatement": - return this.evaluateContinueStatement(node) - case "ThrowStatement": - return this.evaluateThrowStatement(node) - case "TryStatement": - return this.evaluateTryStatement(node) - case "EmptyStatement": - return none - case "FunctionDeclaration": - return none - case "ImportDeclaration": - throw syntaxError("An import declaration may only appear at the top level of the program.", node) - default: - throw unsupportedSyntax(node.type, node) - } - } - - private *finishExpressionStatement(work: Eval): Eval { - yield* work - return none - } - - private *evaluateVariableStatement(node: VariableDeclaration): Eval { - yield* this.evaluateVariableDeclaration(node) - return none - } - - private *evaluateReturnStatement(node: ReturnStatement): Eval { - const argumentNode = node.argument - if (!argumentNode) return { kind: "return", value: undefined } - const evaluated = this.evaluate(argumentNode) - const value = isEval(evaluated) ? yield* evaluated : evaluated - // An async generator awaits what it returns, as it awaits what it yields. - return { kind: "return", value: this.generatorAsync ? yield* this.awaitValue(value, node) : value } - } - - private *evaluateBlock(node: BlockStatement): Eval { - const body = node.body - const scoped = declaresLexically(body) - if (scoped) { - this.scopes.push() - predeclareLexical(this.scopes, body) - hoistFunctions(this.ctx.builtins, this.scopes, body) - } - try { - for (const statement of body) { - const executed = this.execute(statement) - const result = isEval(executed) ? yield* executed : executed - if (result.kind !== "none") return result - } - - return none - } finally { - if (scoped) this.scopes.pop() - } - } - - // NamedEvaluation: an anonymous function definition takes the name of what it is assigned to. - private *evaluateNamed(node: Expression, name: string): Eval { - if (isAnonymousFunction(node)) { - return createFunction(this.ctx.builtins, this.scopes, node, name) - } - return yield* this.evaluateExpression(node) - } - - private *evaluateIfStatement(node: IfStatement): Eval { - const test = this.evaluate(node.test) - const branch = (isEval(test) ? yield* test : test) ? node.consequent : node.alternate - if (!branch) return none - const result = this.execute(branch) - return isEval(result) ? yield* result : result - } - - private *evaluateSwitchStatement(node: SwitchStatement): Eval { - const discriminant = yield* this.evaluateExpression(node.discriminant) - this.scopes.push() - try { - const cases = node.cases - const statements = cases.flatMap((branch) => branch.consequent) - predeclareLexical(this.scopes, statements) - hoistFunctions(this.ctx.builtins, this.scopes, statements) - let defaultIndex: number | undefined - let selected: number | undefined - for (const [index, branch] of cases.entries()) { - const test = branch.test - if (!test) { - defaultIndex = index - continue - } - if ((yield* this.evaluateExpression(test)) === discriminant) { - selected = index - break - } - } - const start = selected ?? defaultIndex - if (start === undefined) return none - for (let index = start; index < cases.length; index += 1) { - for (const statement of cases[index]!.consequent) { - const result = yield* this.evaluateStatement(statement) - if (result.kind === "break") { - if (result.label === undefined) return none - return result - } - if (result.kind === "return" || result.kind === "continue") return result - } - } - return none - } finally { - this.scopes.pop() - } - } - - private *evaluateWhileStatement(node: WhileStatement, labels?: ReadonlySet): Eval { - const runner = this.ctx.runner - while (true) { - const test = this.evaluate(node.test) - if (!(isEval(test) ? yield* test : test)) break - const executed = this.execute(node.body) - const exit = loopExit(isEval(executed) ? yield* executed : executed, labels) - if (exit !== undefined) return exit - if (runner.tick()) yield Slice - } - - return none - } - - private *evaluateDoWhileStatement(node: DoWhileStatement, labels?: ReadonlySet): Eval { - const runner = this.ctx.runner - do { - const exit = loopExit(yield* this.evaluateStatement(node.body), labels) - if (exit !== undefined) return exit - if (runner.tick()) yield Slice - } while (yield* this.evaluateExpression(node.test)) - - return none - } - - private *evaluateForStatement(node: ForStatement, labels?: ReadonlySet): Eval { - const runner = this.ctx.runner - this.scopes.push() - try { - const initNode = node.init - const testNode = node.test - const updateNode = node.update - - if (initNode?.type === "VariableDeclaration" && initNode.kind !== "var") { - predeclareLexical(this.scopes, [initNode]) - } - - if (initNode) { - if (initNode.type === "VariableDeclaration") { - yield* this.evaluateVariableDeclaration(initNode) - } else { - yield* this.evaluateExpression(initNode) - } - } - - // Each iteration gets fresh copies of the loop's `let` bindings, but only closures can tell them apart. - const perIterationBindings = - initNode?.type === "VariableDeclaration" && - initNode.kind !== "var" && - [node.init, node.test, node.update, node.body].some((part) => part && createsClosure(part)) - ? Array.from(this.scopes.current().keys()) - : [] - - const nextIteration = () => { - if (perIterationBindings.length === 0) return - const current = this.scopes.current() - this.scopes.pop() - this.scopes.push( - new Map(perIterationBindings.map((name): [string, Binding] => [name, { ...current.get(name)! }])), - ) - } - nextIteration() - - while (true) { - if (testNode) { - const test = this.evaluate(testNode) - if (!(isEval(test) ? yield* test : test)) break - } - const executed = this.execute(node.body) - const exit = loopExit(isEval(executed) ? yield* executed : executed, labels) - if (exit !== undefined) return exit - - nextIteration() - if (updateNode) { - const update = this.evaluate(updateNode) - if (isEval(update)) yield* update - } - if (runner.tick()) yield Slice - } - - return none - } finally { - this.scopes.pop() - } - } - - private *evaluateForOfStatement(node: ForOfStatement, labels?: ReadonlySet): Eval { - const runner = this.ctx.runner - const awaiting = node.await - const left = node.left - const declared = loopDeclaration(left, "for...of") - if (declared?.lexical) this.scopes.push() - - try { - if (declared?.lexical) predeclarePattern(this.scopes, declared.pattern, declared.mutable, left) - const right = yield* this.evaluateExpression(node.right) - - const builtin = builtinCursor(this.ctx.builtins, right) - const custom = builtin === undefined ? yield* this.customIterator(right, node, awaiting) : undefined - if (custom === undefined && builtin === undefined) { - throw invalidData( - `${awaiting ? "for await...of" : "for...of"} requires an iterable value, received ${describeValue(right)}.`, - node, - ) - } - // `for await` over a synchronous iterator drives it through CreateAsyncFromSyncIterator, as JS does. - const iterator = awaiting && !custom?.asynchronous ? this.asyncFromSync(builtin, custom, node) : custom - const cursor = iterator === undefined ? builtin : undefined - const self = this - const close = function* (): Eval { - if (iterator) return yield* self.closeIterator(iterator, node) - yield* cursor!.close() - } - - if (left.type === "RestElement" || left.type === "AssignmentPattern") { - throw typeError("Unsupported for...of binding.", left) - } - const assignment = left.type === "VariableDeclaration" ? undefined : left - // Each iteration gets a fresh binding, but only a closure in the body can tell; without one, a single - // `for (const x of ...)` binding is reused. - const reused = - declared?.lexical && declared.pattern.type === "Identifier" && !createsClosure(node.body) - ? { mutable: declared.mutable, value: undefined as Value, initialized: false } - : undefined - if (reused !== undefined) this.scopes.push(new Map([[(declared!.pattern as Identifier).name, reused]])) - - try { - while (true) { - const step: Step = iterator ? yield* this.nextIteratorResult(iterator, node) : yield* cursor!.next() - if (step.done) return none - let result: StatementResult - try { - if (reused !== undefined) { - reused.value = step.value - reused.initialized = true - const executed = this.execute(node.body) - result = isEval(executed) ? yield* executed : executed - } else if (declared?.lexical) { - this.scopes.push() - try { - predeclarePattern(this.scopes, declared.pattern, declared.mutable, left) - yield* this.declarePattern(declared.pattern, step.value, declared.mutable, left, true) - result = yield* this.evaluateStatement(node.body) - } finally { - this.scopes.pop() - } - } else { - if (declared) yield* this.assignPattern(declared.pattern, step.value, left) - else if (assignment) yield* this.assignPattern(assignment, step.value, left) - result = yield* this.evaluateStatement(node.body) - } - } catch (error) { - return yield* preserveConsumerError(close, thrower(error)) - } - const exit = loopExit(result, labels) - if (exit !== undefined) { - yield* close() - return exit - } - if (runner.tick()) yield Slice - } - } finally { - if (reused !== undefined) this.scopes.pop() - } - } finally { - if (declared?.lexical) this.scopes.pop() - } - } - - // CreateAsyncFromSyncIterator: an async iterator over a synchronous one. Each `next()` promise settles a job after - // the value does, and a rejected value closes the synchronous iterator first; `return()` calls the synchronous - // iterator's own `return`, or settles at once when it has none. - asyncFromSync( - builtin: Cursor | undefined, - custom: CustomIterator | undefined, - node: AstNode | undefined, - ): CustomIterator { - const ctx = this.ctx - const runner = ctx.runner - const builtins = ctx.builtins - const result = (value: Value, done: boolean) => iteratorResult(builtins.Object, value, done) - const self = this - // IteratorNext passes the value `next` received, when it received one. - const syncNext = function* (args: Array): Eval { - if (!custom) return yield* builtin!.next() - const object = requireIteratorObject( - yield* self.call(custom.next, custom.iterator, args, node), - "Iterator next() result", - node, - ) - return { done: Boolean(get(object, "done")), value: get(object, "value") } - } - // The synchronous iterator's `return`, looked up when it is needed; built-in collection iterators have none. - const syncReturn = (): Value => (custom ? get(custom.iterator, "return") : undefined) - const settle = (step: Step, closeOnRejection: boolean, made: ReturnType) => { - const rejected = (reason: unknown) => { - if (step.done || !closeOnRejection || reason instanceof Interruption) return made.reject(reason) - const body = function* (): Eval { - try { - if (custom) yield* self.closeIterator(custom, node) - else yield* builtin!.close() - } catch { - // The rejection wins over a failing close. - } - made.reject(reason) - } - return { task: new Task([{ body: body(), boundary: undefined }]), input: normal(undefined) } - } - let wrapper: PromiseObj - try { - wrapper = runner.promiseResolve(step.value) - } catch (error) { - const start = rejected(error) - if (start !== undefined) runner.enqueue({ kind: "resume", task: start.task, input: start.input }) - return - } - runner.then(wrapper, { - kind: "callback", - run: (settled) => - settled.ok ? made.resolve(result(settled.value as Value, step.done)) : rejected(settled.error), - }) - } - const next = asyncFromSyncMethod(builtins, "next", function* (_, args) { - const made = capability(ctx, node) - try { - settle(yield* syncNext(args), true, made) - } catch (error) { - made.reject(error) - } - return made.promise - }) - const close = asyncFromSyncMethod(builtins, "return", function* (_, args) { - const made = capability(ctx, node) - try { - const method = syncReturn() - if (method === undefined || method === null) { - if (builtin !== undefined) yield* builtin.close() - made.resolve(result(args[0], true)) - return made.promise - } - const returned = requireIteratorObject( - yield* self.call(requireIteratorMethod(method, "Iterator return", node), custom!.iterator, args, node), - "Iterator return() result", - node, - ) - settle({ done: Boolean(get(returned, "done")), value: get(returned, "value") }, false, made) - } catch (error) { - made.reject(error) - } - return made.promise - }) - const raise = asyncFromSyncMethod(builtins, "throw", function* (_, args) { - const made = capability(ctx, node) - try { - const method = custom ? get(custom.iterator, "throw") : undefined - if (method === undefined || method === null) { - if (custom) yield* self.closeIterator(custom, node) - else yield* builtin!.close() - made.reject(typeError("The delegated iterator does not provide a throw() method.", node)) - return made.promise - } - const thrown = requireIteratorObject( - yield* self.call(requireIteratorMethod(method, "Iterator throw", node), custom!.iterator, args, node), - "Iterator throw() result", - node, - ) - settle({ done: Boolean(get(thrown, "done")), value: get(thrown, "value") }, true, made) - } catch (error) { - made.reject(error) - } - return made.promise - }) - const adapter = new Obj(builtins.Object) - define(adapter, "next", next, hidden) - define(adapter, "return", close, hidden) - define(adapter, "throw", raise, hidden) - return { iterator: adapter, next, asynchronous: true } - } - - *iterate(value: Value, node?: AstNode): Eval { - const cursor = builtinCursor(this.ctx.builtins, value) - if (cursor !== undefined) return cursor - const iterator = yield* this.customIterator(value, node, false) - return iterator === undefined ? undefined : this.customCursor(iterator, node) - } - - /** GetIteratorDirect: drive an iterator by its own `next`, without asking for `[Symbol.iterator]`. */ - iterateDirect(value: Value, node?: AstNode): Cursor { - if (value instanceof IteratorObj) return value.cursor - if (!(value instanceof Obj)) { - throw typeError(`An iterator must be an object, received ${describeValue(value)}.`, node) - } - return this.customCursor( - { - iterator: value, - next: requireIteratorMethod(get(value, "next"), "Iterator next", node), - asynchronous: false, - }, - node, - ) - } - - private customCursor(iterator: CustomIterator, node: AstNode | undefined): Cursor { - return { - next: () => this.nextIteratorResult(iterator, node), - close: () => this.closeIterator(iterator, node), - } - } - - private *customIterator( - value: Value, - node: AstNode | undefined, - allowAsync = true, - ): Eval { - if (!(value instanceof Obj)) return undefined - const asyncMethod = allowAsync ? get(value, AsyncIteratorSymbol) : undefined - const method = asyncMethod ?? get(value, IteratorSymbol) - if (method === undefined || method === null) return undefined - const iterator = yield* this.call(requireIteratorMethod(method, "Iterator method", node), value, [], node) - const object = requireIteratorObject(iterator, "Iterator method result", node) - return { - iterator: object, - next: requireIteratorMethod(get(object, "next"), "Iterator next", node), - asynchronous: asyncMethod !== undefined && asyncMethod !== null, - } - } - - private *nextIteratorResult(iterator: CustomIterator, node: AstNode | undefined): Eval { - const called = yield* this.call(iterator.next, iterator.iterator, [], node) - const object = requireIteratorObject( - iterator.asynchronous ? yield* this.awaitValue(called, node) : called, - "Iterator next() result", - node, - ) - return { done: Boolean(get(object, "done")), value: get(object, "value") } - } - - private *closeIterator(iterator: CustomIterator, node: AstNode | undefined): Eval { - const close = get(iterator.iterator, "return") - if (close === undefined || close === null) return - const method = requireIteratorMethod(close, "Iterator return", node) - const called = yield* this.call(method, iterator.iterator, [], node) - requireIteratorObject( - iterator.asynchronous ? yield* this.awaitValue(called, node) : called, - "Iterator return() result", - node, - ) - } - - private *evaluateForInStatement(node: ForInStatement, labels?: ReadonlySet): Eval { - const runner = this.ctx.runner - const left = node.left - const declared = loopDeclaration(left, "for...in") - if (declared?.lexical) this.scopes.push() - - try { - if (declared?.lexical) predeclarePattern(this.scopes, declared.pattern, declared.mutable, left) - const right = yield* this.evaluateExpression(node.right) - - const keys = enumerableKeys(this.ctx, right, node.right) - - if (left.type === "RestElement" || left.type === "AssignmentPattern") { - throw typeError("Unsupported for...in binding.", left) - } - const assignment = left.type === "VariableDeclaration" ? undefined : left - - for (const key of keys) { - let result: StatementResult - if (declared?.lexical) { - this.scopes.push() - try { - predeclarePattern(this.scopes, declared.pattern, declared.mutable, left) - yield* this.declarePattern(declared.pattern, key, declared.mutable, left, true) - result = yield* this.evaluateStatement(node.body) - } finally { - this.scopes.pop() - } - } else { - if (declared) yield* this.assignPattern(declared.pattern, key, left) - else if (assignment) yield* this.assignPattern(assignment, key, left) - result = yield* this.evaluateStatement(node.body) - } - - const exit = loopExit(result, labels) - if (exit !== undefined) return exit - if (runner.tick()) yield Slice - } - - return none - } finally { - if (declared?.lexical) this.scopes.pop() - } - } - - private evaluateBreakStatement(node: BreakStatement): StatementResult { - return node.label ? { kind: "break", label: node.label.name } : { kind: "break" } - } - - private evaluateContinueStatement(node: ContinueStatement): StatementResult { - return node.label ? { kind: "continue", label: node.label.name } : { kind: "continue" } - } - - private *evaluateLabeledStatement(node: LabeledStatement): Eval { - const labels = new Set() - let body: Statement = node - while (body.type === "LabeledStatement") { - labels.add(body.label.name) - body = body.body - } - - const result = - body.type === "WhileStatement" - ? yield* this.evaluateWhileStatement(body, labels) - : body.type === "DoWhileStatement" - ? yield* this.evaluateDoWhileStatement(body, labels) - : body.type === "ForStatement" - ? yield* this.evaluateForStatement(body, labels) - : body.type === "ForOfStatement" - ? yield* this.evaluateForOfStatement(body, labels) - : body.type === "ForInStatement" - ? yield* this.evaluateForInStatement(body, labels) - : yield* this.evaluateStatement(body) - - return result.kind === "break" && result.label !== undefined && labels.has(result.label) ? none : result - } - - private *evaluateThrowStatement(node: ThrowStatement): Eval { - throw new Throw(yield* this.evaluateExpression(node.argument)) - } - - private *evaluateTryStatement(node: TryStatement): Eval { - const handler = node.handler - const finalizer = node.finalizer - let result: StatementResult = none - let failure: { readonly error: unknown } | undefined - - try { - result = yield* this.evaluateStatement(node.block) - } catch (error) { - if (error instanceof Interruption) throw error - // A generator's return() unwinds through `finally` but is not an exception `catch` sees. - if (error instanceof GeneratorReturn || !handler) { - if (!finalizer) throw error - failure = { error } - } else { - const caught = materialize(this.ctx, error) - const parameter = handler.param - this.scopes.push() - try { - if (parameter) yield* this.declarePattern(parameter, caught, true, handler) - result = yield* this.evaluateStatement(handler.body) - } catch (error) { - if (!finalizer || error instanceof Interruption) throw error - failure = { error } - } finally { - this.scopes.pop() - } - } - } - - if (!finalizer) return result - const final = yield* this.evaluateStatement(finalizer) - if (isAbrupt(final)) return final - if (failure) throw failure.error - return result - } - - private *evaluateVariableDeclaration(node: VariableDeclaration): Eval { - const kind = node.kind - for (const declaration of node.declarations) { - if (declaration.type !== "VariableDeclarator") { - throw typeError("Unsupported variable declaration shape.", declaration) - } - - const init = declaration.init - // `var x` alone is a no-op: the binding was hoisted on function entry. - const id = declaration.id - if (kind === "var") { - if (init) { - const value = - id.type === "Identifier" ? yield* this.evaluateNamed(init, id.name) : yield* this.evaluateExpression(init) - yield* this.assignPattern(id, value, declaration) - } - continue - } - const value = !init - ? undefined - : id.type === "Identifier" - ? yield* this.evaluateNamed(init, id.name) - : yield* this.evaluateExpression(init) - yield* this.declarePattern(declaration.id, value, kind !== "const", declaration, true) - } - } - - private *declarePattern( - pattern: Pattern, - value: Value, - mutable: boolean, - node: AstNode, - initialize = false, - ): Eval { - if (pattern.type === "Identifier") { - const name = pattern.name - if (initialize) this.scopes.initialize(name, value, node) - else this.scopes.declare(name, value, mutable, node) - return - } - - if (pattern.type === "AssignmentPattern") { - const resolved = value === undefined ? yield* this.evaluateDefault(pattern) : value - yield* this.declarePattern(pattern.left, resolved, mutable, node, initialize) - return - } - - if (pattern.type === "ObjectPattern") { - if (value === null || value === undefined) { - throw typeError(`Cannot destructure ${describeValue(value)} as it is ${value}.`, pattern) - } - - const consumed = new Set() - for (const property of pattern.properties) { - if (property.type === "RestElement") { - const rest = new Obj(this.ctx.builtins.Object) - assign(rest, enumerableSource(this.ctx, "Object destructuring", value, pattern), consumed) - yield* this.declarePattern(property.argument, rest, mutable, property, initialize) - continue - } - - const key = yield* this.destructuringPropertyKey(property) - consumed.add(typeof key === "symbol" ? key : String(key)) - yield* this.declarePattern( - property.value, - destructuredProperty(this.ctx.builtins, value, key, property), - mutable, - property, - initialize, - ) - } - return - } - - if (pattern.type === "ArrayPattern") { - return yield* this.destructureArrayPattern(pattern, value, (target, item, context) => - this.declarePattern(target, item, mutable, context, initialize), - ) - } - - throw typeError(`Unsupported binding pattern '${pattern.type}'.`, pattern) - } - - private *assignPattern(pattern: Pattern, value: Value, node: AstNode): Eval { - if (pattern.type === "Identifier") { - this.scopes.set(pattern.name, value, pattern) - return - } - - if (pattern.type === "MemberExpression") { - yield* this.writeMember(pattern, value) - return - } - - if (pattern.type === "AssignmentPattern") { - const resolved = value === undefined ? yield* this.evaluateDefault(pattern) : value - yield* this.assignPattern(pattern.left, resolved, node) - return - } - - if (pattern.type === "ObjectPattern") { - if (value === null || value === undefined) { - throw typeError(`Cannot destructure ${describeValue(value)} as it is ${value}.`, pattern) - } - - const consumed = new Set() - for (const property of pattern.properties) { - if (property.type === "RestElement") { - const rest = new Obj(this.ctx.builtins.Object) - assign(rest, enumerableSource(this.ctx, "Object destructuring", value, pattern), consumed) - yield* this.assignPattern(property.argument, rest, property) - continue - } - const key = yield* this.destructuringPropertyKey(property) - consumed.add(typeof key === "symbol" ? key : String(key)) - yield* this.assignPattern( - property.value, - destructuredProperty(this.ctx.builtins, value, key, property), - property, - ) - } - return - } - - if (pattern.type === "ArrayPattern") { - return yield* this.destructureArrayPattern(pattern, value, (target, item, context) => - this.assignPattern(target, item, context), - ) - } - - throw typeError(`Unsupported assignment pattern '${pattern.type}'.`, node) - } - - private evaluateDefault(pattern: AssignmentPattern): Eval { - return pattern.left.type === "Identifier" - ? this.evaluateNamed(pattern.right, pattern.left.name) - : this.evaluateExpression(pattern.right) - } - - private *destructureArrayPattern( - pattern: ArrayPattern, - value: Value, - consume: (target: Pattern, value: Value, context: AstNode) => Eval, - ): Eval { - const cursor = yield* this.iterate(value, pattern) - if (cursor === undefined) { - throw typeError("Array destructuring requires a supported iterable value.", pattern) - } - let done = false - for (const element of pattern.elements) { - if (done) { - if (element === null) continue - yield* consume( - element.type === "RestElement" ? element.argument : element, - element.type === "RestElement" ? new Arr(this.ctx.builtins.Array) : undefined, - element, - ) - if (element.type === "RestElement") return - continue - } - const step = yield* cursor.next() - done = step.done - if (element === null) continue - if (element.type === "RestElement") { - const rest: Array = [] - if (!step.done) rest.push(step.value) - while (!done) { - const next = yield* cursor.next() - done = next.done - if (!done) rest.push(next.value) - } - yield* consume(element.argument, new Arr(this.ctx.builtins.Array, rest), element) - return - } - const consumed = consume(element, step.done ? undefined : step.value, pattern) - yield* step.done ? consumed : preserveConsumerError(cursor.close, consumed) - } - if (!done) yield* cursor.close() - } - - private *destructuringPropertyKey(property: Property | AssignmentProperty): Eval { - if (property.type !== "Property" || property.kind !== "init") { - throw typeError("Unsupported object destructuring property.", property) - } - const keyNode = property.key - if (property.computed) return yield* this.toPropertyKey(yield* this.evaluateExpression(keyNode), keyNode) - if (keyNode.type === "Identifier") return keyNode.name - if (keyNode.type === "Literal") return String(keyNode.value) - throw unsupportedSyntax(keyNode.type, keyNode) - } - - private *evaluateExpression(node: Expression): Eval { - const result = this.evaluate(node) - return isEval(result) ? yield* result : result - } - - // Evaluates leaves directly and returns the work for anything that may run program code, so the common cases cost - // no generator; callers finish the work with `yield*`. - private evaluate(node: Expression): Value | Eval { - switch (node.type) { - case "Literal": { - const regex = node.regex - if (regex) return constructRegExp(this.ctx.builtins, [regex.pattern, regex.flags]) - return literal(node) - } - case "Identifier": - return this.scopes.get(node.name, node) - case "ThisExpression": - return this.scopes.get("this", node) - case "BinaryExpression": { - const operator = node.operator - if (node.left.type === "PrivateIdentifier") throw unsupportedSyntax(node.left.type, node.left) - const left = this.evaluate(node.left) - if (isEval(left)) return this.evaluateBinaryExpression(node, left) - const right = this.evaluate(node.right) - if (isEval(right)) return this.evaluateBinaryExpression(node, left, right) - if (operator === "instanceof") return instanceofValue(left, right, node) - if (left instanceof Obj || right instanceof Obj) return this.applyOperator(operator, left, right, node) - return applyBinaryOperator(operator, left, right, node) - } - case "LogicalExpression": - return this.evaluateLogicalExpression(node) - case "UnaryExpression": - return this.evaluateUnaryExpression(node) - case "AssignmentExpression": { - const left = node.left - if (left.type !== "Identifier") return this.evaluateAssignmentExpression(node) - if (node.operator === "=") { - const right = - node.right.type === "ArrowFunctionExpression" || - (node.right.type === "FunctionExpression" && !node.right.id) - ? createFunction(this.ctx.builtins, this.scopes, node.right, left.name) - : this.evaluate(node.right) - return isEval(right) ? this.assignAfter(left, right) : this.scopes.set(left.name, right, left) - } - if (!compoundOperators.has(node.operator)) return this.evaluateAssignmentExpression(node) - const current = this.scopes.get(left.name, left) - const right = this.evaluate(node.right) - if (isEval(right) || current instanceof Obj || right instanceof Obj) - return this.compoundAfter(node, current, right) - return this.scopes.set(left.name, applyBinaryOperator(node.operator.slice(0, -1), current, right, node), left) - } - case "SequenceExpression": - return this.evaluateSequenceExpression(node) - case "CallExpression": - return this.evaluateCallExpression(node) - case "ArrowFunctionExpression": - case "FunctionExpression": - return createFunction(this.ctx.builtins, this.scopes, node) - case "MemberExpression": { - const objectNode = node.object - const propertyNode = node.property - if (objectNode.type === "Super") throw unsupportedSyntax(objectNode.type, objectNode) - if (propertyNode.type === "PrivateIdentifier") throw unsupportedSyntax(propertyNode.type, propertyNode) - const object = this.evaluate(objectNode) - if (isEval(object)) return this.readMember(node, object) - if (object === OptionalShortCircuit) return OptionalShortCircuit - if ((object === null || object === undefined) && node.optional) return OptionalShortCircuit - const key = - !node.computed && propertyNode.type === "Identifier" ? propertyNode.name : this.evaluate(propertyNode) - if (isEval(key)) return this.readMember(node, object, key) - const reference = this.memberReference(node, object, key) - return isEval(reference) ? this.readMember(node, object, key) : this.readResolved(node, reference) - } - case "ChainExpression": - return this.evaluateChainExpression(node) - case "ObjectExpression": - return this.evaluateObjectExpression(node) - case "ArrayExpression": - return this.evaluateArrayExpression(node) - case "TemplateLiteral": - return this.evaluateTemplateLiteral(node) - case "TaggedTemplateExpression": - return this.evaluateTaggedTemplate(node) - case "ConditionalExpression": - return this.evaluateConditionalExpression(node) - case "UpdateExpression": { - const argument = node.argument - if (argument.type !== "Identifier") return this.evaluateUpdateExpression(node) - const current = this.scopes.get(argument.name, argument) - if (current instanceof Obj) return this.evaluateUpdateExpression(node) - return this.update(node, argument, current) - } - case "AwaitExpression": - return this.evaluateAwaitExpression(node) - case "YieldExpression": - return this.evaluateYieldExpression(node) - case "NewExpression": - return this.evaluateNewExpression(node) - case "ImportExpression": - return this.evaluateImportExpression(node) - default: - throw unsupportedSyntax(node.type, node) - } - } - - private *evaluateSequenceExpression(node: SequenceExpression): Eval { - let result: Value - for (const expression of node.expressions) { - const value = this.evaluate(expression) - result = isEval(value) ? yield* value : value - } - return result - } - - private *evaluateChainExpression(node: ChainExpression): Eval { - const value = yield* this.evaluateExpression(node.expression) - return value === OptionalShortCircuit ? undefined : value - } - - private *evaluateConditionalExpression(node: ConditionalExpression): Eval { - const test = this.evaluate(node.test) - const branch = this.evaluate((isEval(test) ? yield* test : test) ? node.consequent : node.alternate) - return isEval(branch) ? yield* branch : branch - } - - private *evaluateAwaitExpression(node: AwaitExpression): Eval { - // Await always suspends, including for plain values. - return yield* this.awaitValue(yield* this.evaluateExpression(node.argument), node) - } - - private *evaluateNewExpression(node: NewExpression): Eval { - const callee = yield* this.evaluateExpression(node.callee) - const construct = callee instanceof Builtin ? callee.construct : undefined - if (construct === undefined) throw notConstructible(callee, node) - const args = yield* this.evaluateCallArguments(node.arguments) - const result = this.native(() => construct(args, callee as Builtin, node), node) - return isEval(result) ? yield* result : result - } - - // Finishes a binary expression whose left operand (and maybe right) `evaluate` already started. - private *evaluateBinaryExpression( - node: BinaryExpression, - leftValue: Value | Eval, - started: Value | Eval | typeof unevaluated = unevaluated, - ): Eval { - const operator = node.operator - const lhs = isEval(leftValue) ? yield* leftValue : leftValue - const rightValue = started === unevaluated ? this.evaluate(node.right) : started - const rhs = isEval(rightValue) ? yield* rightValue : rightValue - if (operator === "instanceof") return instanceofValue(lhs, rhs, node) - if (lhs instanceof Obj || rhs instanceof Obj) return yield* this.applyOperator(operator, lhs, rhs, node) - return applyBinaryOperator(operator, lhs, rhs, node) - } - - /** ToPrimitive for an operand: data objects run their own methods; opaque values stay for the data gates below. */ - private *toPrimitive(value: Value, hint: Hint, node: AstNode): Eval { - const result = this.native(() => toPrimitive(this.ctx, value, hint), node) - return isEval(result) ? yield* result : result - } - - // Arithmetic, relational, and bitwise operators convert both operands first, left then right, so a `valueOf` - // runs (and throws) in spec order; `+` asks for the default hint and the rest for a number. - private *applyOperator(operator: string, lhs: Value, rhs: Value, node: AstNode): Eval { - if (!(lhs instanceof Obj || rhs instanceof Obj)) return applyBinaryOperator(operator, lhs, rhs, node) - const conversion = operatorConversion(operator, lhs, rhs) - if (conversion === undefined) return applyBinaryOperator(operator, lhs, rhs, node) - if (conversion.kind === "key") return has(rhs as Obj, yield* this.toPropertyKey(lhs, node)) - const l = yield* this.toPrimitive(lhs, conversion.hint, node) - const r = yield* this.toPrimitive(rhs, conversion.hint, node) - return applyBinaryOperator(operator, l, r, node) - } - - private *evaluateLogicalExpression(node: LogicalExpression): Eval { - const operator = node.operator - const leftValue = this.evaluate(node.left) - const left = isEval(leftValue) ? yield* leftValue : leftValue - if (operator === "&&") return left ? yield* this.evaluateExpression(node.right) : left - if (operator === "||") return left ? left : yield* this.evaluateExpression(node.right) - if (operator === "??") - return left !== null && left !== undefined ? left : yield* this.evaluateExpression(node.right) - throw typeError(`Unsupported logical operator '${operator}'.`, node) - } - - private *evaluateUnaryExpression(node: UnaryExpression): Eval { - const operator = node.operator - const argument = node.argument - if (operator === "delete") return yield* this.evaluateDeleteExpression(argument) - // Undeclared names short-circuit, but declared TDZ bindings must still throw. - if (operator === "typeof" && argument.type === "Identifier" && !this.scopes.resolve(argument.name)) { - return "undefined" - } - const value = yield* this.evaluateExpression(argument) - if (operator === "typeof") return typeofValue(value) - if (operator === "!") return !value - if (operator === "void") return undefined - const operand = yield* this.toPrimitive(value, "number", node) - if (isOpaque(operand)) { - throw invalidData("Unary operators require data values.", node) - } - switch (operator) { - case "+": - return +(operand as number) - case "-": - return -(operand as number) - case "~": - return ~(operand as number) - default: - throw typeError(`Unsupported unary operator '${operator}'.`, node) - } - } - - private *evaluateAssignmentExpression(node: AssignmentExpression): Eval { - const left = node.left - const operator = node.operator - // The binary operator a compound assignment applies: `+=` is `+`. - const binary = operator.slice(0, -1) - if (operator === "??=" || operator === "||=" || operator === "&&=") { - return yield* this.evaluateLogicalAssignment(node, left, operator) - } - if (operator !== "=" && !compoundOperators.has(operator)) { - throw typeError(`Unsupported assignment operator '${operator}'.`, node) - } - if (operator === "=" && (left.type === "ObjectPattern" || left.type === "ArrayPattern")) { - const rightValue = yield* this.evaluateExpression(node.right) - yield* this.assignPattern(left, rightValue, node) - return rightValue - } - if (left.type === "Identifier") { - const name = left.name - if (operator !== "=") { - const current = this.scopes.get(name, left) - const evaluated = this.evaluate(node.right) - const rightValue = isEval(evaluated) ? yield* evaluated : evaluated - const next = - current instanceof Obj || rightValue instanceof Obj - ? yield* this.applyOperator(binary, current, rightValue, node) - : applyBinaryOperator(binary, current, rightValue, node) - return this.scopes.set(name, next, left) - } - const rightValue = yield* this.evaluateNamed(node.right, name) - return this.scopes.set(name, rightValue, left) - } - if (left.type === "MemberExpression") { - const self = this - return yield* this.modifyMember(left, function* (current) { - const rightValue = yield* self.evaluateExpression(node.right) - if (operator === "=") return { write: true, next: rightValue, result: rightValue } - const next = yield* self.applyOperator(binary, current, rightValue, node) - return { write: true, next, result: next } - }) - } - throw typeError("Assignment target must be an Identifier or MemberExpression.", left) - } - - private *assignAfter(left: Identifier, right: Eval): Eval { - return this.scopes.set(left.name, yield* right, left) - } - - private *compoundAfter(node: AssignmentExpression, current: Value, started: Value | Eval): Eval { - const left = node.left as Identifier - const binary = node.operator.slice(0, -1) - const right = isEval(started) ? yield* started : started - const next = - current instanceof Obj || right instanceof Obj - ? yield* this.applyOperator(binary, current, right, node) - : applyBinaryOperator(binary, current, right, node) - return this.scopes.set(left.name, next, left) - } - - // `++`/`--` on a name whose value is already primitive. - private update(node: UpdateExpression, argument: Identifier, current: Value): Value { - if (isOpaque(current)) throw invalidData(`'${node.operator}' requires a data value.`, argument) - const before = coerceToNumber(current) - const next = node.operator === "++" ? before + 1 : before - 1 - this.scopes.set(argument.name, next, argument) - return node.prefix ? next : before - } - - private *evaluateLogicalAssignment(node: AssignmentExpression, left: Pattern, operator: string): Eval { - const shouldAssign = (current: Value): boolean => - operator === "??=" ? current === null || current === undefined : operator === "||=" ? !current : Boolean(current) - if (left.type === "Identifier") { - const name = left.name - const current = this.scopes.get(name, left) - if (!shouldAssign(current)) return current - const rightValue = yield* this.evaluateNamed(node.right, name) - return this.scopes.set(name, rightValue, left) - } - if (left.type === "MemberExpression") { - const self = this - return yield* this.modifyMember(left, function* (current) { - if (!shouldAssign(current)) return { write: false, next: current, result: current } - const rightValue = yield* self.evaluateExpression(node.right) - return { write: true, next: rightValue, result: rightValue } - }) - } - throw typeError("Assignment target must be an Identifier or MemberExpression.", left) - } - - private *evaluateUpdateExpression(node: UpdateExpression): Eval { - const operator = node.operator - const argument = node.argument - const prefix = node.prefix - - const increment = operator === "++" ? 1 : operator === "--" ? -1 : undefined - - if (increment === undefined) { - throw typeError(`Unsupported update operator '${operator}'.`, node) - } - - // CodeMode numeric coercion, not host Number(), so opaque runtime references reject clearly. - const operand = (current: Value): number => { - if (isOpaque(current)) { - throw invalidData(`'${operator}' requires a data value.`, argument) - } - return coerceToNumber(current) - } - - if (argument.type === "Identifier") { - const name = argument.name - const current = this.scopes.get(name, argument) - const before = operand(current instanceof Obj ? yield* this.toPrimitive(current, "number", argument) : current) - const next = before + increment - this.scopes.set(name, next, argument) - return prefix ? next : before - } - - if (argument.type === "MemberExpression") { - const self = this - return yield* this.modifyMember(argument, function* (current) { - const value = operand(yield* self.toPrimitive(current, "number", argument)) - const next = value + increment - return { write: true, next, result: prefix ? next : value } - }) - } - - throw typeError("Update target must be an Identifier or MemberExpression.", argument) - } - - // A module that fails to load rejects the promise, as a failed dynamic import does. - private *evaluateImportExpression(node: ImportExpression): Eval { - const specifier = coerceToString(yield* this.evaluateExpression(node.source)) - if (node.options) yield* this.evaluateExpression(node.options) - const promise = capability(this.ctx, node) - try { - promise.resolve(this.ctx.namespace(specifier, node)) - } catch (error) { - if (!(error instanceof PendingThrow)) throw error - promise.reject(error) - } - return promise.promise - } - - // EvaluateCall: a member callee supplies its base object as `this`; anything else calls with undefined. - private *evaluateCallExpression(node: CallExpression): Eval { - const callee = node.callee - if (callee.type === "Super") throw unsupportedSyntax(callee.type, callee) - const { callable, thisValue } = - callee.type === "MemberExpression" - ? yield* this.readMethod(callee) - : { callable: yield* this.evaluateExpression(callee), thisValue: undefined } - if (callable === OptionalShortCircuit) return OptionalShortCircuit - if ((callable === null || callable === undefined) && node.optional) return OptionalShortCircuit - - const args = yield* this.evaluateCallArguments(node.arguments) - const result = this.dispatch(callable, thisValue, args, node, callee) - if (isEval(result)) return yield* result - return result instanceof Call || result instanceof Invoke ? ((yield result) as Value) : result - } - - private *readMethod(node: MemberExpression): Eval<{ callable: Value; thisValue: Value }> { - const reference = yield* this.getMemberReference(node) - if (reference === OptionalShortCircuit) return { callable: OptionalShortCircuit, thisValue: undefined } - if (reference instanceof ToolReference) return { callable: reference, thisValue: undefined } - if ("value" in reference) return { callable: reference.value, thisValue: undefined } - return { callable: readReference(reference, node), thisValue: reference.receiver } - } - - *call(callable: Value, thisValue: Value, args: Array, node?: AstNode, callee?: Expression): Eval { - const result = this.dispatch(callable, thisValue, args, node, callee) - if (isEval(result)) return yield* result - return result instanceof Call || result instanceof Invoke ? ((yield result) as Value) : result - } - - // The single dispatch for every invocation: call expressions and callbacks share it. A program function is a `Call` - // for the caller to yield; a built-in that returns a value needs no generator. - private dispatch( - callable: Value, - thisValue: Value, - args: Array, - node?: AstNode, - callee?: Expression, - ): Value | Eval | Call | Invoke { - if (callable instanceof Fn) { - if (this.ctx.machine !== undefined) return new Invoke(callable, thisValue, args, node) - return this.invokeFunction(callable, thisValue, args, node) - } - if (callable instanceof Builtin) { - const ctx = this.ctx - // A loop made only of built-in calls has no program call or loop back-edge to count, so count the call. - if (ctx.runner.tick()) - return afterSlice(() => this.native(() => callable.invoke(ctx, thisValue, args, node), node)) - return this.native(() => callable.invoke(ctx, thisValue, args, node), node) - } - if (callable instanceof ToolReference) { - if (callable.path.length === 0) { - throw typeError("The tools root is not callable.", callee ?? node) - } - return this.ctx.toolCall(callable.path, args, node) - } - throw notAFunction(callee, node) - } - - // Built-ins throw without a location, synchronously or inside their work; the call site supplies it. - private native(body: () => NativeResult, node?: AstNode): Value | Eval { - let result: NativeResult - try { - result = body() - } catch (error) { - throw locate(error, node) - } - return isEval(result) ? located(result, node) : result - } - - private *evaluateCallArguments(argNodes: ReadonlyArray): Eval> { - const args: Array = [] - for (const argNode of argNodes) { - if (argNode.type === "SpreadElement") { - const spread = yield* this.evaluateExpression(argNode.argument) - const cursor = yield* this.iterate(spread, argNode) - if (cursor === undefined) throw typeError("Spread arguments require a synchronous iterable.", argNode) - while (true) { - const step = yield* cursor.next() - if (step.done) break - args.push(step.value) - } - } else { - const value = this.evaluate(argNode) - args.push(isEval(value) ? yield* value : value) - } - } - return args - } - - // A program call runs as its own activation on the explicit stack, so it is a `Call` for the caller to yield. An - // async call returns its promise when the body first suspends or ends; a generator call binds its parameters and - // returns the generator without running the body. - invokeFunction(fn: Fn, thisValue: Value, args: Array, node?: AstNode): Call | Value | Eval { - const runner = this.ctx.runner - // The program's own activation is at the bottom of the stack. - if (runner.depth > MAX_CALL_DEPTH) throw rangeError("Maximum call stack size exceeded", node) - const invocation = new Frame(this.ctx, new ScopeStack([...fn.capturedScopes, new Map()])) - const paramScope = invocation.scopes.current() - // `this` and `arguments` are scope bindings so arrows resolve them lexically; a parameter named - // `arguments` shadows the object, as in JS. - if (!fn.arrow) paramScope.set("this", { mutable: false, value: thisValue, initialized: true }) - if (!fn.arrow && usesArguments(fn)) { - paramScope.set("arguments", { - mutable: true, - value: new Arguments(this.ctx.builtins.Object, args), - initialized: true, - }) - } - const simple = isSimple(fn.parameters) - if (simple) { - // A repeated name takes the last argument for it, as in JS. - for (const [index, parameter] of fn.parameters.entries()) { - paramScope.set((parameter as Identifier).name, { mutable: true, value: args[index], initialized: true }) - } - } else { - // Seed all parameters first so defaults cannot fall through to same-named outer bindings. - for (const parameter of fn.parameters) { - for (const name of collectPatternNames(parameter)) { - paramScope.set(name, { mutable: true, value: undefined, initialized: false }) - } - } - } - if (fn.generator) { - invocation.generator = true - invocation.generatorAsync = fn.async - // The prototype is read after the parameters bind, since a default may replace it. - const create = () => { - const proto = get(fn, "prototype") - return new GeneratorObj( - proto instanceof Obj ? proto : fn.async ? this.ctx.builtins.AsyncGenerator : this.ctx.builtins.Generator, - fn.async, - generatorBody(invocation.functionBody(fn, paramScope)), - ) - } - if (simple) return create() - return (function* () { - yield* invocation.bindParameters(fn, args) - return create() - })() - } - const body = simple ? invocation.functionBody(fn, paramScope) : invocation.enter(fn, args, paramScope) - if (!fn.async) return new Call(body) - const promise = runner.newPromise() - const resolving = runner.resolving(promise) - return new Call( - body, - new AsyncBoundary( - (completion) => - completion.ok ? resolving.resolve(completion.value as Value) : resolving.reject(completion.error), - promise, - ), - ) - } - - private *enter(fn: Fn, args: Array, paramScope: Map): Eval { - yield* this.bindParameters(fn, args) - return yield* this.functionBody(fn, paramScope) - } - - private *bindParameters(fn: Fn, args: Array): Eval { - const parameters = fn.parameters - for (const [index, parameter] of parameters.entries()) { - if (parameter.type === "RestElement") { - yield* this.declarePattern( - parameter.argument, - new Arr(this.ctx.builtins.Array, args.slice(index)), - true, - parameter, - true, - ) - break - } - if (parameter.type === "Identifier") { - // A sloppy simple parameter list may repeat a name; the last occurrence wins, as in JS. - if (parameters.findLastIndex((other) => other.type === "Identifier" && other.name === parameter.name) !== index) - continue - this.scopes.initialize(parameter.name, args[index], parameter) - continue - } - yield* this.declarePattern(parameter, args[index], true, parameter, true) - } - } - - private *functionBody(fn: Fn, paramScope: Map): Eval { - if (fn.body.type === "BlockStatement") { - this.scopes.push() - hoistVars(this.scopes, fn.body.body, paramScope) - const result = yield* this.evaluateBlock(fn.body) - return result.kind === "return" ? result.value : undefined - } - const result = this.evaluate(fn.body) - return isEval(result) ? yield* result : result - } - - private *evaluateYieldExpression(node: YieldExpression): Eval { - const argument = node.argument - if (!this.generator) throw typeError("yield is only valid inside a generator.", node) - const value = argument ? yield* this.evaluateExpression(argument) : undefined - if (node.delegate) return yield* this.delegateYield(value, node) - return yield* this.suspendGenerator(this.generatorAsync ? yield* this.awaitValue(value, node) : value) - } - - // Yield `value` and continue with the request that resumes the body: `next` supplies the value of the `yield`, - // `throw` throws there, and `return` unwinds through `finally` blocks (an async generator awaits the value first). - private *suspendGenerator(value: Value, asynchronous = this.generatorAsync): Eval { - const request = (yield new Yield(value)) as GeneratorRequest - if (request.kind === "next") return request.value - if (request.kind === "throw") throw new Throw(request.value) - throw new GeneratorReturn(asynchronous ? yield* this.awaitValue(request.value) : request.value) - } - - *delegateYield(value: Value, node: AstNode, asynchronous = this.generatorAsync): Eval { - const builtin = builtinCursor(this.ctx.builtins, value) - if (builtin !== undefined && !asynchronous) { - while (true) { - const step = yield* builtin.next() - if (step.done) return undefined - try { - yield* this.suspendGenerator(step.value, asynchronous) - } catch (error) { - if (error instanceof GeneratorReturn) { - yield* builtin.close() - throw error - } - if (error instanceof Throw) { - yield* builtin.close() - throw typeError("The delegated iterator does not provide a throw() method.", node) - } - throw error - } - } - } - - const custom = builtin === undefined ? yield* this.customIterator(value, node, asynchronous) : undefined - if (builtin === undefined && !custom) throw typeError("yield* requires a compatible iterable value.", node) - // An async generator delegates to a synchronous iterator through CreateAsyncFromSyncIterator, as JS does. - const iterator = asynchronous && !custom?.asynchronous ? this.asyncFromSync(builtin, custom, node) : custom! - let kind: GeneratorRequest["kind"] = "next" - let input: Value = undefined - while (true) { - const method = kind === "next" ? iterator.next : get(iterator.iterator, kind) - if (method === undefined || method === null) { - if (kind === "return") throw new GeneratorReturn(asynchronous ? yield* this.awaitValue(input, node) : input) - yield* this.closeIterator(iterator, node) - throw typeError("The delegated iterator does not provide a throw() method.", node) - } - const called = yield* this.call( - requireIteratorMethod(method, `Iterator ${kind}`, node), - iterator.iterator, - [input], - node, - ) - const result = requireIteratorObject( - iterator.asynchronous ? yield* this.awaitValue(called, node) : called, - `Iterator ${kind}() result`, - node, - ) - const done = Boolean(get(result, "done")) - const resultValue = get(result, "value") - if (done) { - if (kind === "return") throw new GeneratorReturn(resultValue) - return resultValue - } - - try { - // Delegated values are yielded as they are; an async generator's inner promise already settled them. - input = yield* this.suspendGenerator(resultValue, asynchronous) - kind = "next" - } catch (error) { - if (!(error instanceof GeneratorReturn) && !(error instanceof Throw)) throw error - kind = error instanceof GeneratorReturn ? "return" : "throw" - input = error.value - } - } - } - - private *evaluateObjectExpression(node: ObjectExpression): Eval { - const objectValue = new Obj(this.ctx.builtins.Object) - for (const property of node.properties) { - if (property.type === "SpreadElement") { - const spread = yield* this.evaluateExpression(property.argument) - if (spread === null || spread === undefined) continue - assign(objectValue, enumerableSource(this.ctx, "Object spread", spread, property)) - continue - } - - if (property.kind !== "init") { - throw typeError("Only init object properties are supported.", property) - } - - const keyNode = property.key - - let key: PropertyKey - - if (property.computed) { - key = yield* this.toPropertyKey(yield* this.evaluateExpression(keyNode), keyNode) - } else if (keyNode.type === "Identifier") { - key = keyNode.name - } else if (keyNode.type === "Literal") { - key = propertyKey(literal(keyNode)) - } else { - throw typeError("Unsupported object property key shape.", keyNode) - } - - const name = - key === IteratorSymbol - ? "[Symbol.iterator]" - : key === AsyncIteratorSymbol - ? "[Symbol.asyncIterator]" - : String(key) - define(objectValue, key, yield* this.evaluateNamed(property.value, name)) - } - - return objectValue - } - - private *evaluateArrayExpression(node: ArrayExpression): Eval { - const values: Array = [] - - for (const element of node.elements) { - if (element === null) { - // A literal elision is a real hole, like JS: extend length without an own index. - values.length += 1 - continue - } - if (element.type === "SpreadElement") { - const spread = yield* this.evaluateExpression(element.argument) - const cursor = yield* this.iterate(spread, element) - if (cursor === undefined) throw typeError("Array spread requires a synchronous iterable.", element) - while (true) { - const step = yield* cursor.next() - if (step.done) break - values.push(step.value) - } - } else { - values.push(yield* this.evaluateExpression(element)) - } - } - return new Arr(this.ctx.builtins.Array, values) - } - - private *evaluateTemplateLiteral(node: TemplateLiteral): Eval { - const quasis = node.quasis - const expressions = node.expressions - - let output = "" - - for (let index = 0; index < quasis.length; index += 1) { - const quasi = quasis[index]! - // acorn only omits `cooked` for invalid escapes, which are a parse error outside a tagged template. - if (typeof quasi.value.cooked !== "string") { - throw typeError("Invalid template literal quasi.", quasi) - } - output += quasi.value.cooked - checkStringLength(output.length) - - if (index < expressions.length) { - const raw = yield* this.evaluateExpression(expressions[index]) - output += coerceToString(yield* this.toPrimitive(raw, "string", expressions[index])) - checkStringLength(output.length) - } - } - - return output - } - - // `tag\`a${x}b\`` is `tag(strings, x)`: the tag is read like a callee (a member keeps its receiver), the - // substitutions are evaluated in order, and `strings` carries the escaped source as `strings.raw`. - private *evaluateTaggedTemplate(node: TaggedTemplateExpression): Eval { - const { callable, thisValue } = - node.tag.type === "MemberExpression" - ? yield* this.readMethod(node.tag) - : { callable: yield* this.evaluateExpression(node.tag), thisValue: undefined } - const strings = templateObject(this.ctx, node) - const values: Array = [] - for (const expression of node.quasi.expressions) values.push(yield* this.evaluateExpression(expression)) - return yield* this.call(callable, thisValue, [strings, ...values], node, node.tag) - } - - private *getMemberReference( - node: MemberExpression, - object: Value | Eval | typeof unevaluated = unevaluated, - key: Value | Eval | typeof unevaluated = unevaluated, - ): Eval { - const objectNode = node.object - const propertyNode = node.property - if (objectNode.type === "Super") throw unsupportedSyntax(objectNode.type, objectNode) - if (propertyNode.type === "PrivateIdentifier") throw unsupportedSyntax(propertyNode.type, propertyNode) - const evaluated = object === unevaluated ? this.evaluate(objectNode) : object - const objectValue = isEval(evaluated) ? yield* evaluated : evaluated - if (objectValue === OptionalShortCircuit) return OptionalShortCircuit - if ((objectValue === null || objectValue === undefined) && node.optional) return OptionalShortCircuit - - const computed = - key !== unevaluated - ? key - : !node.computed && propertyNode.type === "Identifier" - ? propertyNode.name - : this.evaluate(propertyNode) - const keyValue = isEval(computed) ? yield* computed : computed - const reference = this.memberReference(node, objectValue, keyValue) - return isEval(reference) ? yield* reference : reference - } - - // The reference once the object and key are evaluated: ToPropertyKey on an object key may run program code. - // GetValue applies ToObject to the base before ToPropertyKey, so a nullish base throws before the key's own - // toString runs. - private memberReference(node: MemberExpression, objectValue: Value, keyValue: Value): Reference | Eval { - requireMemberBase(node, objectValue, keyValue) - if (keyValue instanceof Obj) return this.resolveAfterKey(node, objectValue, keyValue) - return resolveProperty(this.ctx.builtins, objectValue, propertyKey(keyValue), node.object, node.property) - } - - private *resolveAfterKey(node: MemberExpression, objectValue: Value, keyValue: Obj): Eval { - const key = yield* this.toPropertyKey(keyValue, node.property) - return resolveProperty(this.ctx.builtins, objectValue, key, node.object, node.property) - } - - private readResolved(node: MemberExpression, reference: Reference): Value { - if (reference === OptionalShortCircuit) return OptionalShortCircuit - if (reference instanceof ToolReference) return reference - if ("value" in reference) return reference.value - return readReference(reference, node) - } - - private *readMember( - node: MemberExpression, - object: Value | Eval | typeof unevaluated = unevaluated, - key: Value | Eval | typeof unevaluated = unevaluated, - ): Eval { - return this.readResolved(node, yield* this.getMemberReference(node, object, key)) - } - - private writeMember(node: MemberExpression, value: Value): Eval { - return this.modifyMember(node, function* () { - return { write: true, next: value, result: value } - }) - } - - private *evaluateDeleteExpression(argument: Expression): Eval { - const target = argument.type === "ChainExpression" ? argument.expression : argument - if (target.type === "Identifier") throw typeError("Only data fields may be deleted.", argument) - // `delete ` evaluates the operand and is true, as in JS. - if (target.type !== "MemberExpression") { - yield* this.evaluateExpression(target) - return true - } - const reference = yield* this.getMemberReference(target) - if (reference === OptionalShortCircuit) return true - if (reference instanceof ToolReference || "value" in reference || reference.receiver !== reference.target) { - throw invalidData("Only data fields may be deleted.", target) - } - if (remove(reference.target, reference.key)) return true - throw typeError(`Cannot delete property '${String(reference.key)}'.`, target) - } - - // Resolve side-effecting object and key expressions exactly once. - private *modifyMember( - node: MemberExpression, - compute: (current: Value) => Eval<{ write: boolean; next: Value; result: Value }>, - ): Eval { - const reference = writableReference(yield* this.getMemberReference(node), node) - const key = reference.key - const { write, next, result } = yield* compute(readReference(reference, node)) - if (write) assignToReference(reference.target, key, next, node) - return result - } - - // ToPropertyKey: a data object converts through its own `toString`/`valueOf` first; anything else becomes its - // string form synchronously, so `counts[row.category]` works when the field is null. - private *toPropertyKey(value: Value, node: AstNode): Eval { - if (!(value instanceof Obj)) return propertyKey(value) - return propertyKey(yield* this.toPrimitive(value, "string", node)) - } -} - -// The async-from-sync adapter still keeps its iterator in host closures, so its methods are made per iteration. -const asyncFromSyncMethod = (builtins: Builtins, name: string, call: NativeCall) => - native(builtins, { name, length: 1, call, runtime: "async-from-sync" }) - -// Rethrows inside interpreter work, so a failure already caught can pass through `preserveConsumerError`. -function* thrower(error: unknown): Eval { - throw error -} - -// Finishes a built-in's work, locating a failure at the call that started it. -function* located(work: Eval, node: AstNode | undefined): Eval { - try { - return yield* work - } catch (error) { - throw locate(error, node) - } -} - -// Lets the host run first, then starts the work. -function* afterSlice(start: () => Value | Eval): Eval { - yield Slice - const result = start() - return isEval(result) ? yield* result : result -} diff --git a/packages/codemode/src/interpreter/objects.ts b/packages/codemode/src/interpreter/objects.ts index 575452b7b36..096bda0f586 100644 --- a/packages/codemode/src/interpreter/objects.ts +++ b/packages/codemode/src/interpreter/objects.ts @@ -12,7 +12,9 @@ import { Throw, typeError, } from "./model.js" -import type { Entry, Eval, GeneratorRequest, Reaction, Resolving } from "./runner.js" +import type { Eval, GeneratorRequest } from "./signals.js" +import type { Frame } from "../machine/frame.js" +import type { Reaction, Resolving } from "../machine/scheduler.js" import { OrderedTable, type TableCursor } from "./table.js" /** Property attributes, as in a JS property descriptor. */ @@ -347,8 +349,8 @@ export type GeneratorState = export class GeneratorObj extends Opaque { override readonly tag = "Generator" state: GeneratorState = "suspendedStart" - /** The suspended body's activations, kept while it waits for the next request. */ - entries: Array | undefined + /** The suspended body's frames, kept while it waits for the next request. */ + frames: Array | undefined /** An async generator's requests, oldest first, each with the functions that settle its promise. */ readonly queue: Array<{ readonly request: GeneratorRequest; readonly resolving: Resolving }> = [] constructor( diff --git a/packages/codemode/src/interpreter/promises.ts b/packages/codemode/src/interpreter/promises.ts index 54804dfed53..e3dce972436 100644 --- a/packages/codemode/src/interpreter/promises.ts +++ b/packages/codemode/src/interpreter/promises.ts @@ -5,17 +5,8 @@ import { createAggregateErrorValue, materialize } from "./errors.js" import { describeValue, typeofValue } from "./references.js" import { isSupportedCallback } from "./callback.js" import type { Interpreter } from "./interpreter.js" -import { - type Combination, - type Completion, - type Eval, - Interruption, - normal, - type Reaction, - type Resolving, - type Start, - Task, -} from "./runner.js" +import { type Completion, type Eval, Interruption } from "./signals.js" +import type { Combination, Reaction, Resolving } from "../machine/scheduler.js" import { finishAwaitReturn } from "./generators.js" // A `resolve`/`reject` handed to an executor or thenable: calling it settles the capability. @@ -29,18 +20,18 @@ const resolvingFunctions = (ctx: Interpreter, resolving: Resolving) => ({ * it is located at `site`, the call that created it. */ export const capability = (ctx: Interpreter, site: AstNode | undefined): Resolving => - ctx.runner.resolving(ctx.runner.newPromise(), site) + ctx.machine.resolving(ctx.machine.newPromise(), site) /** Settles `resolving` with a completion: a fulfillment value is resolved, so a thenable is adopted. */ const settleWith = (resolving: Resolving, completion: Completion) => completion.ok ? resolving.resolve(completion.value as Value) : resolving.reject(completion.error) -/** Runs a settled promise's reaction as a job: it finishes at once, or returns the task that calls program code. */ +/** Runs a settled promise's reaction as a job: it finishes at once, or returns the work that calls program code. */ export const react = ( ctx: Interpreter, reaction: Exclude, settled: Completion, -): Start | void => { +): Eval | void => { switch (reaction.kind) { case "settle": return settleWith(reaction.target, settled) @@ -61,8 +52,33 @@ export const react = ( } } -/** A task that runs `body` as its own job. */ -const job = (body: Eval): Start => ({ task: new Task([{ body, boundary: undefined }]), input: normal(undefined) }) +/** + * NewPromiseResolveThenableJob: the work that calls the thenable's `then`. Adopting a native promise through the + * intrinsic `then` takes the same jobs as calling it, and keeps the rejection as thrown so its diagnostic survives. + */ +export const adoptThenable = ( + ctx: Interpreter, + promise: PromiseObj, + thenable: Obj, + then: Value, +): Eval | void => { + const machine = ctx.machine + const resolving = machine.resolving(promise) + if (then === machine.intrinsicThen && thenable instanceof PromiseObj) { + machine.then(thenable, { kind: "settle", target: resolving }) + return + } + const proto = ctx.builtins.Function + const resolve = new Resolver(proto, "", resolving, "resolve") + const reject = new Resolver(proto, "", resolving, "reject") + return (function* (): Eval { + try { + yield* ctx.call(then, thenable, [resolve, reject]) + } catch (error) { + resolving.reject(error) + } + })() +} // A `then`/`catch` reaction calls program code: the handler runs as a job, and its result or failure settles the // derived promise. A missing handler passes the outcome through. @@ -70,7 +86,7 @@ const handle = ( ctx: Interpreter, reaction: Extract, settled: Completion, -): Start | void => { +): Eval | void => { const handler = settled.ok ? reaction.onFulfilled : reaction.onRejected const derived = reaction.derived if (handler === undefined || (!settled.ok && settled.error instanceof Interruption)) { @@ -87,7 +103,7 @@ const handle = ( } derived.resolve(result) } - return job(body()) + return body() } const promiseStatics = ["all", "allSettled", "race", "any", "resolve", "reject", "withResolvers", "try"] as const @@ -98,8 +114,8 @@ function* invokePromiseMethod( args: Array, site: AstNode | undefined, ): Eval { - const runner = ctx.runner - if (name === "resolve") return runner.promiseResolve(args[0]) + const machine = ctx.machine + if (name === "resolve") return machine.promiseResolve(args[0]) if (name === "reject") { const made = capability(ctx, site) made.reject(new Throw(args[0])) @@ -134,7 +150,7 @@ function* combinator( iterable: Value, site: AstNode | undefined, ): Eval { - const runner = ctx.runner + const machine = ctx.machine const made = capability(ctx, site) try { const cursor = yield* ctx.iterate(iterable) @@ -146,13 +162,13 @@ function* combinator( while (true) { const step = yield* cursor.next() if (step.done) break - const item = runner.promiseResolve(step.value) + const item = machine.promiseResolve(step.value) const reaction: Reaction = { kind: "combinator", combination, index: index++, called: false } combination.values.push(undefined) combination.reasons.push(undefined) combination.remaining++ combination.attached.push([item, reaction]) - runner.then(item, reaction) + machine.then(item, reaction) } if (name === "race" && index === 0) { throw typeError("Promise.race([]) would never settle; provide at least one promise or value.") @@ -203,7 +219,7 @@ const combine = ( } const detachAll = (ctx: Interpreter, combination: Combination) => { - for (const [item, reaction] of combination.attached) ctx.runner.detach(item, reaction) + for (const [item, reaction] of combination.attached) ctx.machine.detach(item, reaction) } const finishCombination = (ctx: Interpreter, combination: Combination) => { @@ -230,7 +246,7 @@ const instanceMethod = ( const onFulfilled = name === "then" ? reactionHandler(args[0], method) : undefined const onRejected = reactionHandler(name === "then" ? args[1] : args[0], method) const made = capability(ctx, site) - ctx.runner.then(promise, { kind: "handler", onFulfilled, onRejected, derived: made }) + ctx.machine.then(promise, { kind: "handler", onFulfilled, onRejected, derived: made }) return made.promise } @@ -243,7 +259,7 @@ const chainFinally = ( site: AstNode | undefined, ): PromiseObj => { const made = capability(ctx, site) - ctx.runner.then( + ctx.machine.then( source, cleanup === undefined ? { kind: "settle", target: made } : { kind: "finally", cleanup, target: made, site }, ) @@ -255,7 +271,7 @@ const cleanUp = ( reaction: Extract, settled: Completion, ) => { - const runner = ctx.runner + const machine = ctx.machine const made = reaction.target if (!settled.ok && settled.error instanceof Interruption) return made.reject(settled.error) const cleanup = reaction.cleanup @@ -269,17 +285,17 @@ const cleanUp = ( } let after: PromiseObj try { - after = runner.promiseResolve(result) + after = machine.promiseResolve(result) } catch (error) { made.reject(error) return } // The spec chains `then(valueThunk)` onto the cleanup's promise: one more job before the outcome passes. const derived = capability(ctx, reaction.site) - runner.then(after, { kind: "finallyCleaned", target: derived }) - runner.then(derived.promise, { kind: "finallyOutcome", target: made, outcome: settled }) + machine.then(after, { kind: "finallyCleaned", target: derived }) + machine.then(derived.promise, { kind: "finallyOutcome", target: made, outcome: settled }) } - return job(body()) + return body() } const reactionHandler = (value: Value, method: string): Callable | undefined => { @@ -321,7 +337,7 @@ const promiseLikeMethods = (ctx: Interpreter) => { const source = receiver(Obj, thisValue, name) const copy = new Obj(ctx.builtins.Object) for (const [key, slot] of source.props) copy.props.set(key, { ...slot }) - return ctx.runner.promiseResolve(copy) + return ctx.machine.promiseResolve(copy) } methods( ctx.builtins, @@ -363,7 +379,7 @@ export const promiseGlobal = (ctx: Interpreter) => { ["finally", 1, (thisValue, args, site) => instanceMethod(ctx, "finally", thisValue, args, site)], ]) const then = get(proto, "then") - if (then instanceof Callable) ctx.runner.intrinsicThen = then + if (then instanceof Callable) ctx.machine.intrinsicThen = then promiseLikeMethods(ctx) return promise } diff --git a/packages/codemode/src/interpreter/runner.ts b/packages/codemode/src/interpreter/runner.ts deleted file mode 100644 index 22c6efc01b2..00000000000 --- a/packages/codemode/src/interpreter/runner.ts +++ /dev/null @@ -1,590 +0,0 @@ -import { Cause, type Context, Effect, Exit, Fiber } from "effect" -import type { Diagnostic } from "../codemode.js" -import { locate, normalizeError } from "./errors.js" -import type { Interpreter } from "./interpreter.js" -import { MAX_PENDING_PROMISES } from "./limits.js" -import { type AstNode, PendingThrow, rangeError, typeError } from "./model.js" -import { type Callable, type GeneratorObj, get, Obj, PromiseObj, Resolver, type Value } from "./objects.js" -import { react } from "./promises.js" -import { fromJson, type Json } from "../data.js" -import { typeofValue } from "./references.js" - -export { - abrupt, - AsyncBoundary, - type Body, - type Boundary, - Call, - type Completion, - Enter, - type Entry, - type Eval, - Await, - generatorBody, - type GeneratorRequest, - Host, - Interruption, - Invoke, - isEval, - MAX_CALL_DEPTH, - normal, - type Signal, - Slice, - Task, - Yield, -} from "./signals.js" -import { - abrupt, - AsyncBoundary, - Await, - type Body, - Call, - type Completion, - Enter, - type Eval, - Host, - Interruption, - Invoke, - normal, - type Signal, - Task, - Yield, -} from "./signals.js" - -export type Start = { readonly task: Task; readonly input: Completion } - -/** What the member reactions of one `Promise.all`/`allSettled`/`race`/`any` share. */ -export type Combination = { - readonly name: "all" | "allSettled" | "race" | "any" - /** Settles the combinator's result. */ - readonly made: Resolving - readonly values: Array - readonly reasons: Array - /** Members still to settle, plus one until iteration ends. */ - remaining: number - /** Every member's reaction, detached once the result is decided. */ - readonly attached: Array -} - -/** - * A settled promise's reaction, as data. It runs as a job (see `react` in promises.ts) and either finishes at once or - * starts a task. The kinds: - * - `await`: resume the async body parked on the promise. - * - `settle`: pass the outcome on to `target`. - * - `handler`: `then`/`catch` handlers; the handler's result settles `derived`. - * - `combinator`: one member of a `Promise` combinator. - * - `finally`, `finallyCleaned`, `finallyOutcome`: the steps of `Promise.prototype.finally` with a cleanup. - * - `awaitReturn`: an async generator's `return(value)` once `value` settles. - * - `callback`: a host closure, still used by the async-from-sync iterator adapter. - */ -export type Reaction = - | { readonly kind: "await"; readonly task: Task } - | { readonly kind: "settle"; readonly target: Resolving } - | { - readonly kind: "handler" - readonly onFulfilled: Callable | undefined - readonly onRejected: Callable | undefined - readonly derived: Resolving - } - | { readonly kind: "combinator"; readonly combination: Combination; readonly index: number; called: boolean } - | { - readonly kind: "finally" - readonly cleanup: Callable - readonly target: Resolving - readonly site: AstNode | undefined - } - | { readonly kind: "finallyCleaned"; readonly target: Resolving } - | { readonly kind: "finallyOutcome"; readonly target: Resolving; readonly outcome: Completion } - | { readonly kind: "awaitReturn"; readonly generator: GeneratorObj } - | { readonly kind: "callback"; readonly run: (settled: Completion) => Start | void } - -/** One queued job, as data: run a settled promise's reaction, resume a task, or adopt a thenable. */ -export type Job = - | { readonly kind: "reaction"; readonly reaction: Reaction; readonly settled: Completion } - | { readonly kind: "resume"; readonly task: Task; readonly input: Completion } - | { readonly kind: "thenable"; readonly promise: PromiseObj; readonly thenable: Obj; readonly then: Value } - -/** - * Host work the program waits on: a tool or extension call, running as a fiber the runner keeps by `id`. Its outcome - * arrives as an inbox record and settles the call's promise in a later job. - */ -export class HostCall { - status: "inFlight" | "completed" = "inFlight" - constructor( - readonly id: number, - /** What runs: a tool path, or an extension function's label. */ - readonly label: string, - /** A tool's arguments, as they crossed. */ - readonly args: ReadonlyArray | undefined, - /** How the result becomes a program value: a tool's crosses as JSON, an extension's is already one. */ - readonly result: "json" | "value", - readonly resolving: Resolving, - /** The call that started the work, where a defect it raises is located. */ - readonly site: AstNode | undefined, - ) {} -} - -/** A host call's outcome, waiting to be delivered. */ -type Delivery = { readonly call: HostCall; readonly completion: Completion } - -/** - * CreateResolvingFunctions as data: settles one promise at most once, however many `Resolver`s share it. `reject` takes - * the failure as thrown; one without a location is located at `site`, the call that created the promise. - */ -export class Resolving { - done = false - constructor( - private readonly runner: Pick, "resolveWith" | "reject">, - readonly promise: PromiseObj, - readonly site: AstNode | undefined, - ) {} - - resolve(resolution: Value): void { - if (this.done) return - this.done = true - this.runner.resolveWith(this.promise, resolution, this.site) - } - - reject(reason: unknown): void { - if (this.done) return - this.done = true - this.runner.reject(this.promise, locate(reason, this.site)) - } -} - -type Pause = - | { readonly kind: "slice"; readonly input: Completion } - | { readonly kind: "host"; readonly effect: Effect.Effect } - -/** Interpreter steps between clock checks, and how long one slice may run before the host gets a turn. */ -const FUEL = 64 -const SLICE_MS = 1 - -/** - * One execution's job queue. Program code runs only inside a job, one job at a time and each to completion, as in a - * JavaScript engine: promise reactions and resumed awaits queue in order, host work runs concurrently and settles its - * promise as a later job, and the program is stuck exactly when no job is queued and no host work is running. - */ -export class Runner { - /** The intrinsic `Promise.prototype.then`, so adopting a native promise skips exposing its rejection first. */ - intrinsicThen: Callable | undefined - private readonly queue: Array = [] - private head = 0 - private current: Task | undefined - private fuel = FUEL - private sliceStart = 0 - private pending = 0 - private nextID = 0 - private nextHostID = 0 - /** Rejections nothing handled yet, described when they happened so reporting them at the end is cheap. */ - private readonly unhandled = new Map() - /** Bodies suspended at an `await` and the promise each waits on, so a stall can point at the one that is stuck. */ - private readonly parked = new Map() - /** Running host work by `HostCall` id; fibers stay here and never become part of the program's state. */ - private readonly fibers = new Map>() - private readonly inbox: Array = [] - private wake: (() => void) | undefined - private stopping = false - - constructor( - private readonly ctx: Interpreter, - /** The services host work runs with: the caller's, so tools see its context, tracing, and clock. */ - private readonly services: Context.Context, - ) {} - - /** Activations on the running task's stack. */ - get depth(): number { - return this.current?.stack.length ?? 0 - } - - /** - * Runs the program as the first job until its body completes, then returns how it ended. Host work it started may - * still be running; `interrupt` stops it. - */ - run(main: Body): Effect.Effect { - const self = this - return Effect.gen(function* () { - let outcome: Completion | undefined - let start: Start | undefined = { - task: new Task([ - { - body: main, - boundary: new AsyncBoundary((completion) => { - outcome = completion - }, undefined), - }, - ]), - input: normal(undefined), - } - self.sliceStart = performance.now() - while (outcome === undefined) { - if (start === undefined) { - const job = self.dequeue() - if (job !== undefined) { - start = self.runJob(job) ?? undefined - if (start === undefined && self.tick()) yield* self.yieldToHost - continue - } - const delivery = self.inbox.shift() - if (delivery !== undefined) { - self.deliver(delivery) - continue - } - if (self.fibers.size > 0) { - yield* self.hostSettled - continue - } - return abrupt( - new PendingThrow( - "Error", - "The program is waiting on a promise that can never settle: nothing else is running, so nothing can resolve it. Code Mode has no timers or sleep, so don't wait or poll inside a program.", - self.stuckAwait(), - ), - ) - } - const pause = self.drive(start.task, start.input) - if (pause === undefined) { - start = undefined - if (self.tick()) yield* self.yieldToHost - continue - } - if (pause.kind === "slice") { - start = { task: start.task, input: pause.input } - yield* self.yieldToHost - continue - } - const exit = yield* Effect.exit(pause.effect) - if (Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) return yield* Effect.failCause(exit.cause) - start = { - task: start.task, - input: Exit.isSuccess(exit) ? normal(exit.value) : abrupt(Cause.squash(exit.cause)), - } - } - if (!outcome.ok && outcome.error instanceof Interruption) return yield* Effect.failCause(outcome.error.cause) - return outcome - }) - } - - /** Stops host work still running, waiting for its cleanup. */ - interrupt(): Effect.Effect { - const self = this - return Effect.gen(function* () { - self.stopping = true - while (self.fibers.size > 0) yield* Fiber.interruptAll([...self.fibers.values()]) - }) - } - - /** - * Rejections no handler observed, in the order their promises were created. Host work that finished before the - * program did settles first, so its failures count even though no later job ran. - */ - diagnostics(): Array { - for (let delivery = this.inbox.shift(); delivery !== undefined; delivery = this.inbox.shift()) - this.deliver(delivery) - return [...this.unhandled].sort(([left], [right]) => left.id - right.id).map(([, diagnostic]) => diagnostic) - } - - /** Counts one interpreter step; true when the slice is over and the caller should yield `Slice`. */ - tick(): boolean { - if (--this.fuel > 0) return false - this.fuel = FUEL - return performance.now() - this.sliceStart >= SLICE_MS - } - - /** A blocking host call, with the caller's services provided so the runner can run it as is. */ - blocking(effect: Effect.Effect): Host { - return new Host(Effect.provideContext(effect, this.services)) - } - - enqueue(job: Job): void { - this.queue.push(job) - } - - /** - * Starts host work now, settling `resolving`'s promise in a later job, in completion order. A defect is located at - * `site`, the call that started the work. - */ - host( - effect: Effect.Effect, - call: { - readonly label: string - readonly args?: ReadonlyArray - readonly result: "json" | "value" - readonly resolving: Resolving - readonly site: AstNode | undefined - }, - ): HostCall { - const record = new HostCall(this.nextHostID++, call.label, call.args, call.result, call.resolving, call.site) - // Starts synchronously, so admission and hooks happen at the call site. - const fiber = Effect.runForkWith(this.services)(effect) - this.fibers.set(record.id, fiber) - fiber.addObserver((exit) => { - this.fibers.delete(record.id) - // Interrupted by the program ending: nothing is left to settle. - if (this.stopping) return - this.inbox.push({ - call: record, - completion: Exit.isSuccess(exit) - ? normal(exit.value) - : abrupt( - Cause.hasInterruptsOnly(exit.cause) - ? new Interruption(exit.cause) - : Cause.hasDies(exit.cause) - ? locate(Cause.squash(exit.cause), record.site) - : Cause.squash(exit.cause), - ), - }) - this.wake?.() - }) - return record - } - - private deliver(delivery: Delivery): void { - const call = delivery.call - const completion = delivery.completion - call.status = "completed" - if (!completion.ok) return call.resolving.reject(completion.error) - call.resolving.resolve( - call.result === "json" ? fromJson(this.ctx, completion.value as Json | undefined) : (completion.value as Value), - ) - } - - // --- Promise abstract operations --- - - newPromise(): PromiseObj { - if (this.pending >= MAX_PENDING_PROMISES) { - throw rangeError( - `Too many pending promises (limit ${MAX_PENDING_PROMISES}); await promises before creating more.`, - ) - } - this.pending++ - return new PromiseObj(this.ctx.builtins.Promise, this.nextID++) - } - - /** CreateResolvingFunctions for `promise`; a failure without a location is located at `site`. */ - resolving(promise: PromiseObj, site?: AstNode): Resolving { - return new Resolving(this, promise, site) - } - - /** PerformPromiseThen with one reaction for either outcome; marks the promise handled. */ - then(promise: PromiseObj, reaction: Reaction): void { - if (promise.state === "pending") promise.reactions.push(reaction) - else { - if (promise.state === "rejected") this.unhandled.delete(promise) - const settled = promise.state === "fulfilled" ? normal(promise.result) : abrupt(promise.result) - this.enqueue({ kind: "reaction", reaction, settled }) - } - promise.handled = true - } - - /** Removes a reaction an internal consumer no longer needs, so a long-pending promise does not keep it. */ - detach(promise: PromiseObj, reaction: Reaction): void { - const index = promise.reactions.indexOf(reaction) - if (index >= 0) promise.reactions.splice(index, 1) - } - - /** PromiseResolve(%Promise%, value). */ - promiseResolve(value: Value): PromiseObj { - if (value instanceof PromiseObj) return value - const promise = this.newPromise() - this.resolveWith(promise, value) - return promise - } - - /** Rejects `promise` with a failure as thrown. */ - reject(promise: PromiseObj, reason: unknown): void { - this.settle(promise, abrupt(reason)) - } - - private settle(promise: PromiseObj, completion: Completion): void { - if (promise.state !== "pending") return - this.pending-- - promise.state = completion.ok ? "fulfilled" : "rejected" - promise.result = completion.ok ? completion.value : completion.error - const reactions = promise.reactions - promise.reactions = [] - if (!completion.ok && !promise.handled && !(completion.error instanceof Interruption)) { - const failure = normalizeError(completion.error) - this.unhandled.set(promise, { - ...failure, - message: `Unhandled rejection from an un-awaited promise: ${failure.message}`, - }) - } - for (const reaction of reactions) this.enqueue({ kind: "reaction", reaction, settled: completion }) - } - - /** ResolvePromise: adopts a thenable, otherwise fulfills. */ - resolveWith(promise: PromiseObj, resolution: Value, site?: AstNode): void { - if (resolution === promise) { - const cycle = typeError("Chaining cycle detected: a promise cannot resolve with itself.", site) - return this.settle(promise, abrupt(cycle)) - } - if (!(resolution instanceof Obj)) return this.settle(promise, normal(resolution)) - const then = get(resolution, "then") - if (typeofValue(then) !== "function") return this.settle(promise, normal(resolution)) - this.enqueue({ kind: "thenable", promise, thenable: resolution, then }) - } - - // NewPromiseResolveThenableJob. Adopting a native promise through the intrinsic `then` takes the same jobs as - // calling it, and keeps the rejection as thrown so its diagnostic survives. - private thenableJob(promise: PromiseObj, thenable: Obj, then: Value): Start | void { - const resolving = this.resolving(promise) - if (then === this.intrinsicThen && thenable instanceof PromiseObj) { - this.then(thenable, { kind: "settle", target: resolving }) - return - } - const proto = this.ctx.builtins.Function - const resolve = new Resolver(proto, "", resolving, "resolve") - const reject = new Resolver(proto, "", resolving, "reject") - const ctx = this.ctx - const body = function* (): Eval { - try { - yield* ctx.call(then, thenable, [resolve, reject]) - } catch (error) { - resolving.reject(error) - } - } - return { task: new Task([{ body: body(), boundary: undefined }]), input: normal(undefined) } - } - - private runJob(job: Job): Start | void { - if (job.kind === "resume") return { task: job.task, input: job.input } - if (job.kind === "thenable") return this.thenableJob(job.promise, job.thenable, job.then) - const reaction = job.reaction - if (reaction.kind !== "await") return react(this.ctx, reaction, job.settled) - this.parked.delete(reaction.task) - return { task: reaction.task, input: job.settled } - } - - // --- Driver --- - - // Runs a task until its stack empties or it pauses for a slice or host work, which resumes it before any other job. - private drive(task: Task, input: Completion): Pause | undefined { - const stack = task.stack - this.current = task - let completion = input - while (stack.length > 0) { - const entry = stack[stack.length - 1]! - let step: IteratorResult - try { - step = completion.ok ? entry.body.next(completion.value) : entry.body.throw(completion.error) - } catch (error) { - stack.pop() - completion = entry.boundary === undefined ? abrupt(error) : entry.boundary.finish(abrupt(error)) - continue - } - if (step.done) { - stack.pop() - completion = entry.boundary === undefined ? normal(step.value) : entry.boundary.finish(normal(step.value)) - continue - } - const signal = step.value - if (signal instanceof Call) { - stack.push({ body: signal.body, boundary: signal.boundary }) - completion = normal(undefined) - if (this.tick()) return { kind: "slice", input: completion } - continue - } - if (signal instanceof Invoke) { - let call: Call - try { - call = this.ctx.enter(signal) - } catch (error) { - completion = abrupt(error) - continue - } - stack.push({ body: call.body, boundary: call.boundary }) - completion = normal(undefined) - if (this.tick()) return { kind: "slice", input: completion } - continue - } - if (signal instanceof Enter) { - for (const saved of signal.entries) stack.push(saved) - completion = signal.input - if (this.tick()) return { kind: "slice", input: completion } - continue - } - if (signal instanceof Await) { - const index = stack.findLastIndex((item) => item.boundary?.async === true) - if (index < 0) { - completion = abrupt(typeError("await is only valid in async functions.")) - continue - } - let promise: PromiseObj - try { - promise = this.promiseResolve(signal.value) - } catch (error) { - completion = abrupt(error) - continue - } - const boundary = stack[index]!.boundary! - const continuation = new Task(stack.splice(index)) - this.parked.set(continuation, { promise, node: signal.node }) - this.then(promise, { kind: "await", task: continuation }) - completion = boundary.suspended() - continue - } - if (signal instanceof Yield) { - const index = stack.findLastIndex((item) => item.boundary?.generator === true) - if (index < 0) { - completion = abrupt(typeError("yield is only valid inside a generator.")) - continue - } - completion = stack[index]!.boundary!.yielded(signal.value, () => stack.splice(index)) - continue - } - if (signal instanceof Host) return { kind: "host", effect: signal.effect } - return { kind: "slice", input: normal(undefined) } - } - this.current = undefined - if (!completion.ok) throw completion.error - return undefined - } - - // The innermost stuck `await`: an outer one waits on the result of a parked async body, so prefer one that does not. - // Bodies that wait on each other in a cycle leave no such await; any located one is then as good as another. - private stuckAwait(): AstNode | undefined { - const results = new Set() - for (const task of this.parked.keys()) { - const boundary = task.stack[0]?.boundary - if (boundary instanceof AsyncBoundary) results.add(boundary.result) - } - const located = [...this.parked.values()].filter((parked) => parked.node !== undefined) - return (located.find((parked) => !results.has(parked.promise)) ?? located[0])?.node - } - - private dequeue(): Job | undefined { - if (this.head === this.queue.length) { - if (this.head > 0) { - this.queue.length = 0 - this.head = 0 - } - return undefined - } - const job = this.queue[this.head++] - if (this.head > 4096 && this.head * 2 > this.queue.length) { - this.queue.splice(0, this.head) - this.head = 0 - } - return job - } - - private get yieldToHost(): Effect.Effect { - return Effect.map(Effect.yieldNow, () => { - this.sliceStart = performance.now() - }) - } - - private get hostSettled(): Effect.Effect { - return Effect.callback((resume) => { - if (this.inbox.length > 0) return resume(Effect.void) - this.wake = () => { - this.wake = undefined - resume(Effect.void) - } - return Effect.sync(() => { - this.wake = undefined - }) - }) - } -} diff --git a/packages/codemode/src/interpreter/signals.ts b/packages/codemode/src/interpreter/signals.ts index 60956262123..9050b598791 100644 --- a/packages/codemode/src/interpreter/signals.ts +++ b/packages/codemode/src/interpreter/signals.ts @@ -1,14 +1,15 @@ -import type { Cause, Effect } from "effect" +import type { Cause } from "effect" +import type { Frame } from "../machine/frame.js" +import type { HostRequest, Resolving } from "../machine/scheduler.js" import { type AstNode, GeneratorReturn, type GeneratorRequestKind, typeError } from "./model.js" import type { Value } from "./objects.js" -// The signals interpreter work sends its driver, and the activation records the driver keeps. Nothing here needs -// Effect at run time, so the machine can use them too. +// The signals interpreter work (a built-in's host generator) sends the machine. Nothing here needs Effect at run time. /** - * Interpreter work: a generator that yields signals to the driver and returns its result. Work inside one activation - * composes with `yield*`; a program call is a `Call`, so program recursion lives on the explicit stack below and never - * grows the host stack. + * Interpreter work: a generator that yields signals to the machine and returns its result. It runs in a native frame; + * a program call is an `Invoke`, which the machine runs as frames on its own stack, so program recursion never grows + * the host stack. */ export type Eval = Generator @@ -19,16 +20,13 @@ export type Completion = export const normal = (value: unknown): Completion => ({ ok: true, value }) export const abrupt = (error: unknown): Completion => ({ ok: false, error }) -/** - * What an activation's body is: interpreter work, or a machine task (`src/machine`), which answers the same `next` and - * `throw` as a generator. - */ +/** What a native frame runs: interpreter work, or a machine task (a generator's body), which answers the same calls. */ export interface Body { next(input?: unknown): IteratorResult throw(error: unknown): IteratorResult } -/** Run `body` as a new activation; a boundary marks where a suspended body splits off the stack. */ +/** Run `body` in a native frame of its own; a boundary marks where the stack splits when the body above suspends. */ export class Call { constructor( readonly body: Body, @@ -36,10 +34,10 @@ export class Call { ) {} } -/** Push saved activations back on the stack and continue the top one with `input`. */ +/** Push saved frames back on the stack and continue the top one with `input`. */ export class Enter { constructor( - readonly entries: Array, + readonly frames: Array, readonly input: Completion, ) {} } @@ -59,7 +57,7 @@ export class Yield { /** Block the current job on host work: no other job runs until it ends, as with a synchronous host call. */ export class Host { - constructor(readonly effect: Effect.Effect) {} + constructor(readonly request: HostRequest) {} } /** @@ -74,10 +72,7 @@ export class Interruption { /** Let the host run; the current job continues before any other job. */ export const Slice: unique symbol = Symbol("codemode.slice") -/** - * Call a program function from inside interpreter work. The machine evaluator answers it on its own stack; the - * runner, when it reaches it, starts the call as a new activation. - */ +/** Call a program function from inside interpreter work; the machine runs the call as frames on its own stack. */ export class Invoke { constructor( readonly callable: Value, @@ -100,40 +95,36 @@ export interface Boundary { readonly generator: boolean /** The body above returned or threw. */ finish(completion: Completion): Completion - /** The body above suspended at an `await`; its activations are saved in the continuation. */ + /** The body above suspended at an `await`; its frames are parked on the awaited promise. */ suspended(): Completion - /** The body above yielded; `split` takes its activations off the stack when it suspends. */ - yielded(value: Value, split: () => Array): Completion + /** The body above yielded; `split` takes its frames off the stack when it suspends. */ + yielded(value: Value, split: () => Array): Completion } -/** The boundary of an async body: the body's completion settles it, and the caller receives `result` at once. */ +/** + * The boundary of host work that crosses its arguments first (a tool call): the caller receives the call's promise + * at once, and a failure while crossing rejects it. Once the work starts, its outcome settles the promise. + */ export class AsyncBoundary implements Boundary { readonly async = true readonly generator = false - constructor( - private readonly settle: (completion: Completion) => void, - readonly result: Value, - ) {} + constructor(readonly resolving: Resolving) {} + get result(): Value { + return this.resolving.promise + } finish(completion: Completion): Completion { - this.settle(completion) - return normal(this.result) + if (!completion.ok) this.resolving.reject(completion.error) + return normal(this.resolving.promise) } suspended(): Completion { - return normal(this.result) + return normal(this.resolving.promise) } yielded(): Completion { return abrupt(typeError("yield is only valid inside a generator.")) } } -export type Entry = { readonly body: Body; readonly boundary: Boundary | undefined } - -/** Paused or running program code: activations on an explicit stack, innermost last. */ -export class Task { - constructor(readonly stack: Array) {} -} - -/** Most activations one task may hold, as the call-stack limit. */ +/** Most activations one stack may hold, as the call-stack limit. */ export const MAX_CALL_DEPTH = 10_000 // Program values never have a `next` method of their own: an object's properties live in its property map. diff --git a/packages/codemode/src/machine/calls.ts b/packages/codemode/src/machine/calls.ts index 97ab17e144a..367fa6a543c 100644 --- a/packages/codemode/src/machine/calls.ts +++ b/packages/codemode/src/machine/calls.ts @@ -30,19 +30,22 @@ import { import { isOpaque } from "../interpreter/references.js" import { ScopeStack } from "../interpreter/scope.js" import { - AsyncBoundary, - Call, + abrupt, + type Body, + type Boundary, + type Completion, type Eval, generatorBody, Invoke, isEval, MAX_CALL_DEPTH, + normal, type Signal, } from "../interpreter/signals.js" +import type { Resolving } from "./scheduler.js" import type { StatementResult } from "../interpreter/model.js" import { Frame, kind, type Out, PENDING, type Pending } from "./frame.js" import { ev } from "./expressions.js" -import type { Machine } from "./machine.js" import { bind, declaring } from "./patterns.js" import { blockStatement } from "./statements.js" import { MachineTask } from "./task.js" @@ -59,7 +62,7 @@ export function dispatch( if (callable instanceof Fn) return invoke(task, callable, thisValue, args, node) if (callable instanceof Builtin) { // A loop made only of built-in calls has no program call or loop back-edge to count, so count the call. - if (task.ctx.runner.tick()) task.slice = true + if (task.machine.tick()) task.slice = true // A bound function calls its target directly, so a program target runs as a frame rather than inside a native. if (callable instanceof BoundFn) { return dispatch(task, callable.target, callable.boundThis, [...callable.boundArgs, ...args], node) @@ -79,67 +82,36 @@ export function dispatch( throw notAFunction(callee, node) } -// A program call: a synchronous body runs as a counted `Call` frame on this stack. An async body runs as a task of its -// own behind an async boundary, as the runner keeps it today, and the caller receives its promise; a generator call -// binds its parameters on this stack and returns the generator. +// A program call runs as a counted `Call` frame on this stack. An async function's frame is an async boundary: an +// `await` in its body cuts the stack there and the caller receives its promise, which the body's completion settles. +// A generator call binds its parameters on this stack and returns the generator. function invoke(task: MachineTask, fn: Fn, thisValue: Value, args: Array, node: AstNode | undefined) { const machine = task.machine - const runner = task.ctx.runner - if (runner.depth + machine.calls > MAX_CALL_DEPTH) throw rangeError("Maximum call stack size exceeded", node) - if (runner.tick()) task.slice = true - if (fn.async && !fn.generator) return task.request(asyncCall(machine, fn, thisValue, args, node), fn.body) + if (machine.depth > MAX_CALL_DEPTH) throw rangeError("Maximum call stack size exceeded", node) + if (machine.tick()) task.slice = true + const resolving = fn.async && !fn.generator ? machine.resolving(machine.newPromise()) : undefined const counted = !fn.generator - if (counted) machine.calls++ - return task.push(CallKind, fn.body, [fn, thisValue, args, node, counted, undefined, 0]) -} - -/** A program call the runner starts as its own activation, so its frame is not counted again. */ -export function enter(machine: Machine, invocation: Invoke): Call { - const fn = invocation.callable as Fn - const runner = machine.ctx.runner - if (runner.depth + machine.calls > MAX_CALL_DEPTH) { - throw rangeError("Maximum call stack size exceeded", invocation.site) - } - if (fn.async && !fn.generator) return asyncCall(machine, fn, invocation.thisValue, invocation.args, invocation.site) - return new Call(callTask(machine, fn, invocation.thisValue, invocation.args, invocation.site)) -} - -const callTask = (machine: Machine, fn: Fn, thisValue: Value, args: Array, site: AstNode | undefined) => { - const task = new MachineTask(machine, new ScopeStack([], fn)) - task.stack.push(new Frame(CallKind, fn.body, 0, task.scopes, [fn, thisValue, args, site, false, undefined, 0])) - return task -} - -// An async function's body is a task behind an async boundary: its completion settles the promise the caller gets. -const asyncCall = (machine: Machine, fn: Fn, thisValue: Value, args: Array, site: AstNode | undefined) => { - const body = callTask(machine, fn, thisValue, args, site) - const runner = machine.ctx.runner - const promise = runner.newPromise() - const resolving = runner.resolving(promise) - return new Call( - body, - new AsyncBoundary( - (completion) => - completion.ok ? resolving.resolve(completion.value as Value) : resolving.reject(completion.error), - promise, - ), - ) + if (counted) machine.depth++ + return task.push(CallKind, fn.body, [fn, thisValue, args, node, counted, undefined, 0, resolving]) } // The phases of a `Call` frame: bind the parameters (resuming at the slot index after a pattern finishes), make the -// generator or start the body, and receive the body's result. A generator's own body task starts at `RUN`. +// generator or start the body, and receive the body's result. A generator's own body task starts at `RUN`. An async +// function's frame that took a throw has `SETTLED` its promise with it. const ENTER = 0 const BIND = 1 const BODY = 2 const RUN = 3 const RESULT = 4 +const SETTLED = 5 -// Slots: fn, this, args, call site, whether the frame counts toward the depth limit, the parameter scope, and the -// next parameter to bind. +// Slots: fn, this, args, call site, whether the frame counts toward the depth limit, the parameter scope, the next +// parameter to bind, and for an async function what settles its promise. function callStep(task: MachineTask, frame: Frame, _node: AstNode, input: unknown): Out { const slots = frame.slots const fn = slots[0] as Fn let phase = frame.phase + if (phase === SETTLED) return (slots[7] as Resolving).promise if (phase === ENTER) { const scopes = activate(task, fn, slots[1] as Value, slots[2] as Array) frame.scope = scopes @@ -168,10 +140,16 @@ function callStep(task: MachineTask, frame: Frame, _node: AstNode, input: unknow input = result } } - if (slots[4]) task.machine.calls-- - if (fn.body.type !== "BlockStatement") return input as Value - const result = input as StatementResult - return result.kind === "return" ? result.value : undefined + const result = + fn.body.type !== "BlockStatement" + ? (input as Value) + : (input as StatementResult).kind === "return" + ? (input as Extract).value + : undefined + const resolving = slots[7] as Resolving | undefined + if (resolving === undefined) return result + resolving.resolve(result) + return resolving.promise } // The activation's scope chain: the closure's scopes and a parameter scope holding `this`, `arguments`, and the @@ -249,10 +227,19 @@ const generator = (task: MachineTask, fn: Fn, frame: Frame): GeneratorObj => { } export const CallKind = kind("Call", callStep, { - unwind: (task, frame) => { - if (frame.slots[4]) task.machine.calls-- - return false + // An async function's body that throws rejects its promise; the caller still receives the promise. + unwind: (_task, frame, error) => { + const resolving = frame.slots[7] as Resolving | undefined + if (resolving === undefined || frame.phase === SETTLED) return false + resolving.reject(error) + frame.phase = SETTLED + return true }, + cut: (frame) => { + const resolving = frame.slots[7] as Resolving | undefined + return resolving === undefined ? undefined : normal(resolving.promise) + }, + counts: (frame) => frame.slots[4] === true, }) // The phases of a native frame: running (or receiving the answer it waits for), throwing a failure into the work, done. @@ -262,10 +249,10 @@ const FINISHED = 2 /** * Runs a built-in's interpreter work as a native frame: program calls it makes (`Invoke`) run as frames on this stack, - * and anything else it asks for goes to the driver. The frame holds a host generator, so it is opaque. + * and anything else it asks for is answered by the step loop. The frame holds a host generator, so it is opaque. */ export function native(task: MachineTask, work: Eval, site: AstNode | undefined, located: boolean): Out { - const frame = new Frame(Native, site, RUNNING, task.scopes, [work, located, undefined]) + const frame = new Frame(Native, site, RUNNING, task.scopes, [work, located, undefined, undefined, false]) task.stack.push(frame) let out: Out try { @@ -278,25 +265,41 @@ export function native(task: MachineTask, work: Eval, site: AstNode | unde return out } -// Slots: the work, whether its failures are located at the call site, and a failure to throw into it. +/** + * A native frame for a body that runs as an activation of its own: a job's work, or a body a built-in starts behind a + * boundary (`Call`). It counts toward the depth limit; the boundary decides what its completion gives the frame below. + */ +export const nativeFrame = (scopes: ScopeStack, body: Body, boundary: Boundary | undefined, counted: boolean): Frame => + new Frame(Native, undefined, RUNNING, scopes, [body, false, undefined, boundary, counted]) + +/** The boundary a native frame's body runs behind, if any. */ +export const boundaryOf = (frame: Frame): Boundary | undefined => + frame.kind === Native ? (frame.slots[3] as Boundary | undefined) : undefined + +// Slots: the work, whether its failures are located at the call site, a failure to throw into it, the boundary it +// runs behind, and whether it counts toward the depth limit. function nativeStep(task: MachineTask, frame: Frame, site: AstNode | undefined, input: unknown): Out { const slots = frame.slots - const work = slots[0] as Eval + const work = slots[0] as Body let ok = frame.phase !== THROWING let value = ok ? input : slots[2] slots[2] = undefined frame.phase = RUNNING while (true) { - let result: IteratorResult + let result: IteratorResult try { result = ok ? work.next(value) : work.throw(value) } catch (error) { frame.phase = FINISHED + const boundary = slots[3] as Boundary | undefined + if (boundary !== undefined) return outcome(boundary.finish(abrupt(error))) throw slots[1] ? locate(error, site) : error } if (result.done) { frame.phase = FINISHED - return result.value + const boundary = slots[3] as Boundary | undefined + if (boundary !== undefined) return outcome(boundary.finish(normal(result.value))) + return result.value as Value } const signal = result.value if (!(signal instanceof Invoke)) { @@ -317,6 +320,11 @@ function nativeStep(task: MachineTask, frame: Frame, site: AstNode | undefined, } } +const outcome = (completion: Completion): Value => { + if (!completion.ok) throw completion.error + return completion.value as Value +} + export const Native = kind("Native", nativeStep, { opaque: true, unwind: (_task, frame, error) => { @@ -325,6 +333,11 @@ export const Native = kind("Native", nativeStep, { frame.slots[2] = error return true }, + cut: (frame) => { + const boundary = frame.slots[3] as Boundary | undefined + return boundary?.async ? boundary.suspended() : undefined + }, + counts: (frame) => frame.slots[4] === true, }) export type Hint = "number" | "string" | "default" diff --git a/packages/codemode/src/machine/frame.ts b/packages/codemode/src/machine/frame.ts index 829f91d553d..9b827f12449 100644 --- a/packages/codemode/src/machine/frame.ts +++ b/packages/codemode/src/machine/frame.ts @@ -1,5 +1,6 @@ import type { AstNode } from "../interpreter/model.js" import type { ScopeStack } from "../interpreter/scope.js" +import type { Completion } from "../interpreter/signals.js" import type { MachineTask } from "./task.js" /** @@ -26,6 +27,13 @@ export type Kind = { // Method syntax, so each kind's step can name the node type it evaluates and the input it receives. step(task: MachineTask, frame: Frame | undefined, node: AstNode | undefined, input: unknown): unknown unwind?(task: MachineTask, frame: Frame, error: unknown): boolean + /** + * Whether the frame is an async boundary, where an `await` above cuts the stack: what the frame below receives once + * the frames from this one up are parked on the awaited promise, or undefined when the frame is no boundary. + */ + cut?(frame: Frame): Completion | undefined + /** Whether the frame counts toward the call-depth limit while it is on the stack. */ + counts?(frame: Frame): boolean /** A frame of this kind holds a host generator (a built-in that calls program code), so it cannot be saved. */ readonly opaque?: boolean } @@ -47,7 +55,11 @@ export class Frame { ) {} } -export const kind = (name: string, step: Kind["step"], options?: Pick): Kind => ({ +export const kind = ( + name: string, + step: Kind["step"], + options?: Pick, +): Kind => ({ name, step, ...options, diff --git a/packages/codemode/src/machine/machine.ts b/packages/codemode/src/machine/machine.ts index e51f97a2c53..08d4604c4f7 100644 --- a/packages/codemode/src/machine/machine.ts +++ b/packages/codemode/src/machine/machine.ts @@ -1,34 +1,364 @@ +import type { Diagnostic } from "../codemode.js" +import { fromJson, type Json } from "../data.js" +import { normalizeError } from "../interpreter/errors.js" import type { Interpreter } from "../interpreter/interpreter.js" +import { MAX_PENDING_PROMISES } from "../interpreter/limits.js" +import { type AstNode, rangeError, typeError } from "../interpreter/model.js" +import { type Callable, get, Obj, PromiseObj, type Value } from "../interpreter/objects.js" +import { adoptThenable, react } from "../interpreter/promises.js" +import { typeofValue } from "../interpreter/references.js" +import { ScopeStack } from "../interpreter/scope.js" +import { + abrupt, + Await, + type Completion, + type Eval, + Host, + Interruption, + normal, + Slice, + Yield, +} from "../interpreter/signals.js" import type { Source } from "../interpreter/source.js" -import type { Call, Invoke } from "../interpreter/signals.js" +import { nativeFrame } from "./calls.js" import { Frame } from "./frame.js" -import { enter } from "./calls.js" +import { HostCall, type HostRequest, type Job, type Reaction, Resolving, type Stop } from "./scheduler.js" import { ProgramKind } from "./statements.js" import { MachineTask } from "./task.js" -/** Which evaluator runs program code: the machine (`src/machine`) or the generator evaluator in `interpreter.ts`. */ -export type Evaluator = "machine" | "legacy" +/** Interpreter steps between clock checks, and how long one slice may run before the host gets a turn. */ +const FUEL = 64 +const SLICE_MS = 1 + +/** A host call's outcome, waiting to be delivered. */ +type Delivery = { readonly call: HostCall; readonly completion: Completion } + +/** Frames parked on a promise at an `await`, and what the frame below their boundary received when they were cut. */ +type Parked = { readonly promise: PromiseObj; readonly node: AstNode | undefined; readonly result: Value } /** - * The machine evaluator for one execution. Program code runs as explicit frames on `MachineTask` stacks, stepped by a - * loop, instead of host generators. Until the scheduler moves onto the machine too, each task is an activation body - * the runner drives: the program, an async function's body, a generator's body, or a call the runner starts. + * One execution's machine: program code runs as explicit frames on a stack, stepped by a loop, and the job queue, + * promises, and host calls it waits on are data it keeps. Program code runs only inside a job, one job at a time and + * each to completion, as in a JavaScript engine: promise reactions and resumed awaits queue in order, host work runs + * outside and settles its promise as a later job, and the program is stuck exactly when no job is queued and no host + * work is running. + * + * `run` steps until the machine has to stop, and says why (`Stop`); the driver (`driver.ts`) does what the stop asks + * and calls `run` again. Nothing here runs host work or knows about Effect. */ export class Machine { - /** Program calls on the running task's machine stacks, counted toward the call-depth limit with the runner's. */ - calls = 0 + /** Activations on the stack that count toward the call-depth limit. */ + depth = 0 + /** The intrinsic `Promise.prototype.then`, so adopting a native promise skips exposing its rejection first. */ + intrinsicThen: Callable | undefined + /** The stack every job runs on. */ + readonly root: MachineTask + /** How the program's body ended, once it has. */ + outcome: Completion | undefined + /** Frames suspended at an `await`, by the slice cut off the stack, so a stall can point at the one that is stuck. */ + readonly parked = new Map, Parked>() - constructor(readonly ctx: Interpreter) {} + private readonly queue: Array = [] + private head = 0 + private fuel = FUEL + private sliceStart = 0 + private pending = 0 + private nextID = 0 + private nextHostID = 0 + /** Rejections nothing handled yet, described when they happened so reporting them at the end is cheap. */ + private readonly unhandled = new Map() + /** Host calls queued since the last stop, for the driver to start. */ + private queued: Array = [] + /** Host calls the driver is running, by id. */ + private readonly running = new Map() + private readonly inbox: Array = [] + /** What the root stack continues with: the job in progress was paused, or has just started. */ + private input: Completion | undefined + /** A slice is due once the queued host calls are handed over. */ + private sliced = false - /** The program's body as a task: it runs the program, adopts a returned thenable, and copies the result out. */ - program(source: Source): MachineTask { - const task = new MachineTask(this, this.ctx.programScopes()) - task.stack.push(new Frame(ProgramKind, source.program, 0, task.scopes, [])) - return task + constructor(readonly ctx: Interpreter) { + this.root = new MachineTask(this, new ScopeStack([])) } - /** A call the runner reached (a built-in called a program function outside any task), as its own activation. */ - enter(invoke: Invoke): Call { - return enter(this, invoke) + /** Queues the program's body as the first job. */ + start(source: Source): void { + this.root.enter([new Frame(ProgramKind, source.program, 0, this.ctx.programScopes(), [])]) + this.input = normal(undefined) + } + + /** Steps until the machine has to stop: see `Stop`. A throw no frame handles escapes as a defect. */ + run(): Stop { + this.sliceStart = performance.now() + while (true) { + if (this.queued.length > 0) { + const calls = this.queued + this.queued = [] + for (const call of calls) { + call.status = "inFlight" + this.running.set(call.id, call) + } + return { kind: "host", calls } + } + if (this.sliced) { + this.sliced = false + return { kind: "slice" } + } + const input = this.input + if (input !== undefined) { + this.input = undefined + this.continue(input) + continue + } + // A blocking call is running: the job that made it continues once it completes. + if (this.root.stack.length > 0) return { kind: "host", calls: [] } + if (this.outcome !== undefined) return { kind: "done", completion: this.outcome } + const job = this.dequeue() + if (job !== undefined) { + this.begin(job) + if (this.input === undefined && this.tick()) this.sliced = true + continue + } + const delivery = this.inbox.shift() + if (delivery !== undefined) { + this.deliver(delivery) + continue + } + if (this.running.size > 0) return { kind: "host", calls: [] } + return { kind: "stuck", node: this.stuckAwait() } + } + } + + /** + * Reports how host work handed over by a stop ended. A blocking call's outcome continues the job that made it; a + * concurrent call's waits in the inbox and settles its promise in a later job, in completion order. + */ + complete(id: number, completion: Completion): void { + const call = this.running.get(id) + if (call === undefined) return + this.running.delete(id) + if (call.blocking) { + call.status = "completed" + this.input = completion + return + } + this.inbox.push({ call, completion }) + } + + /** Whether a host call's outcome is waiting to be delivered. */ + get ready(): boolean { + return this.inbox.length > 0 || this.input !== undefined + } + + /** + * Rejections no handler observed, in the order their promises were created. Host work that finished before the + * program did settles first, so its failures count even though no later job ran. + */ + diagnostics(): Array { + for (let delivery = this.inbox.shift(); delivery !== undefined; delivery = this.inbox.shift()) + this.deliver(delivery) + return [...this.unhandled].sort(([left], [right]) => left.id - right.id).map(([, diagnostic]) => diagnostic) + } + + /** Counts one interpreter step; true when the slice is over and the host should get a turn. */ + tick(): boolean { + if (--this.fuel > 0) return false + this.fuel = FUEL + return performance.now() - this.sliceStart >= SLICE_MS + } + + enqueue(job: Job): void { + this.queue.push(job) + } + + /** + * Queues concurrent host work, settling `resolving`'s promise in a later job, in completion order. The driver starts + * it at the next stop, before any other job runs. A defect is located at `site`, the call that started the work. + */ + host( + request: HostRequest, + call: { + readonly label: string + readonly result: "json" | "value" + readonly resolving: Resolving + readonly site: AstNode | undefined + }, + ): HostCall { + const record = new HostCall(this.nextHostID++, request, call.label, false, call.result, call.resolving, call.site) + this.queued.push(record) + return record + } + + /** A native frame that runs `body` as a job of its own. */ + job(body: Eval): Frame { + return nativeFrame(this.root.scopes, body, undefined, true) + } + + /** Parks frames cut off the stack at an `await` on the awaited promise; they resume as a job once it settles. */ + park(frames: Array, promise: PromiseObj, node: AstNode | undefined, result: Value): void { + this.parked.set(frames, { promise, node, result }) + this.then(promise, { kind: "await", frames }) + } + + // --- Promise abstract operations --- + + newPromise(): PromiseObj { + if (this.pending >= MAX_PENDING_PROMISES) { + throw rangeError( + `Too many pending promises (limit ${MAX_PENDING_PROMISES}); await promises before creating more.`, + ) + } + this.pending++ + return new PromiseObj(this.ctx.builtins.Promise, this.nextID++) + } + + /** CreateResolvingFunctions for `promise`; a failure without a location is located at `site`. */ + resolving(promise: PromiseObj, site?: AstNode): Resolving { + return new Resolving(this, promise, site) + } + + /** PerformPromiseThen with one reaction for either outcome; marks the promise handled. */ + then(promise: PromiseObj, reaction: Reaction): void { + if (promise.state === "pending") promise.reactions.push(reaction) + else { + if (promise.state === "rejected") this.unhandled.delete(promise) + const settled = promise.state === "fulfilled" ? normal(promise.result) : abrupt(promise.result) + this.enqueue({ kind: "reaction", reaction, settled }) + } + promise.handled = true + } + + /** Removes a reaction an internal consumer no longer needs, so a long-pending promise does not keep it. */ + detach(promise: PromiseObj, reaction: Reaction): void { + const index = promise.reactions.indexOf(reaction) + if (index >= 0) promise.reactions.splice(index, 1) + } + + /** PromiseResolve(%Promise%, value). */ + promiseResolve(value: Value): PromiseObj { + if (value instanceof PromiseObj) return value + const promise = this.newPromise() + this.resolveWith(promise, value) + return promise + } + + /** Rejects `promise` with a failure as thrown. */ + reject(promise: PromiseObj, reason: unknown): void { + this.settle(promise, abrupt(reason)) + } + + /** ResolvePromise: adopts a thenable, otherwise fulfills. */ + resolveWith(promise: PromiseObj, resolution: Value, site?: AstNode): void { + if (resolution === promise) { + const cycle = typeError("Chaining cycle detected: a promise cannot resolve with itself.", site) + return this.settle(promise, abrupt(cycle)) + } + if (!(resolution instanceof Obj)) return this.settle(promise, normal(resolution)) + const then = get(resolution, "then") + if (typeofValue(then) !== "function") return this.settle(promise, normal(resolution)) + this.enqueue({ kind: "thenable", promise, thenable: resolution, then }) + } + + private settle(promise: PromiseObj, completion: Completion): void { + if (promise.state !== "pending") return + this.pending-- + promise.state = completion.ok ? "fulfilled" : "rejected" + promise.result = completion.ok ? completion.value : completion.error + const reactions = promise.reactions + promise.reactions = [] + if (!completion.ok && !promise.handled && !(completion.error instanceof Interruption)) { + const failure = normalizeError(completion.error) + this.unhandled.set(promise, { + ...failure, + message: `Unhandled rejection from an un-awaited promise: ${failure.message}`, + }) + } + for (const reaction of reactions) this.enqueue({ kind: "reaction", reaction, settled: completion }) + } + + // --- Jobs --- + + // Starts a job on the root stack: parked frames resume with how their promise settled, and a reaction or thenable + // that calls program code runs its work in a native frame. One that finishes at once leaves the stack empty. + private begin(job: Job): void { + if (job.kind === "resume") return this.resume(job.frames, job.input) + if (job.kind === "thenable") return this.work(adoptThenable(this.ctx, job.promise, job.thenable, job.then)) + const reaction = job.reaction + if (reaction.kind !== "await") return this.work(react(this.ctx, reaction, job.settled)) + this.parked.delete(reaction.frames) + this.resume(reaction.frames, job.settled) + } + + private resume(frames: Array, input: Completion): void { + this.root.enter(frames) + this.input = input + } + + private work(body: Eval | void): void { + if (body === undefined) return + this.root.enter([this.job(body)]) + this.input = normal(undefined) + } + + // Continues the job on the root stack. It ends when the stack empties, or pauses for a slice or a blocking host call; + // either way it continues before any other job. + private continue(input: Completion): void { + const root = this.root + const result = root.drive(input.ok, input.ok ? input.value : input.error) + if (result.done) { + if (this.tick()) this.sliced = true + return + } + const signal = result.value + if (signal === Slice) { + this.input = normal(undefined) + this.sliced = true + return + } + if (signal instanceof Host) { + const call = new HostCall(this.nextHostID++, signal.request, "", true, "value", undefined, undefined) + this.queued.push(call) + return + } + // Nothing on the stack can take an `await` or `yield`: the frame that asked receives the failure. + if (signal instanceof Await) this.input = abrupt(typeError("await is only valid in async functions.")) + else if (signal instanceof Yield) this.input = abrupt(typeError("yield is only valid inside a generator.")) + else this.input = abrupt(new TypeError("The machine cannot answer this signal here.")) + } + + private deliver(delivery: Delivery): void { + const call = delivery.call + const completion = delivery.completion + call.status = "completed" + const resolving = call.resolving! + if (!completion.ok) return resolving.reject(completion.error) + resolving.resolve( + call.result === "json" ? fromJson(this.ctx, completion.value as Json | undefined) : (completion.value as Value), + ) + } + + // The innermost stuck `await`: an outer one waits on the result of a parked async body, so prefer one that does not. + // Bodies that wait on each other in a cycle leave no such await; any located one is then as good as another. + private stuckAwait(): AstNode | undefined { + const results = new Set() + for (const parked of this.parked.values()) if (parked.result instanceof PromiseObj) results.add(parked.result) + const located = [...this.parked.values()].filter((parked) => parked.node !== undefined) + return (located.find((parked) => !results.has(parked.promise)) ?? located[0])?.node + } + + private dequeue(): Job | undefined { + if (this.head === this.queue.length) { + if (this.head > 0) { + this.queue.length = 0 + this.head = 0 + } + return undefined + } + const job = this.queue[this.head++] + if (this.head > 4096 && this.head * 2 > this.queue.length) { + this.queue.splice(0, this.head) + this.head = 0 + } + return job } } diff --git a/packages/codemode/src/machine/scheduler.ts b/packages/codemode/src/machine/scheduler.ts new file mode 100644 index 00000000000..6293d904f5e --- /dev/null +++ b/packages/codemode/src/machine/scheduler.ts @@ -0,0 +1,136 @@ +import type { Json } from "../data.js" +import { locate } from "../interpreter/errors.js" +import type { AstNode } from "../interpreter/model.js" +import type { Callable, GeneratorObj, Obj, PromiseObj, Value } from "../interpreter/objects.js" +import type { Completion, Eval } from "../interpreter/signals.js" +import type { Frame } from "./frame.js" + +// The scheduler's records: jobs, reactions, resolvers, and host calls. All of them are data the machine keeps, so a +// run's queue and the work it waits on can be read (and later saved) at any step boundary. + +/** What the member reactions of one `Promise.all`/`allSettled`/`race`/`any` share. */ +export type Combination = { + readonly name: "all" | "allSettled" | "race" | "any" + /** Settles the combinator's result. */ + readonly made: Resolving + readonly values: Array + readonly reasons: Array + /** Members still to settle, plus one until iteration ends. */ + remaining: number + /** Every member's reaction, detached once the result is decided. */ + readonly attached: Array +} + +/** + * A settled promise's reaction, as data. It runs as a job (see `react` in promises.ts) and either finishes at once or + * starts interpreter work. The kinds: + * - `await`: resume the frames parked on the promise, cut off the stack at an `await`. + * - `settle`: pass the outcome on to `target`. + * - `handler`: `then`/`catch` handlers; the handler's result settles `derived`. + * - `combinator`: one member of a `Promise` combinator. + * - `finally`, `finallyCleaned`, `finallyOutcome`: the steps of `Promise.prototype.finally` with a cleanup. + * - `awaitReturn`: an async generator's `return(value)` once `value` settles. + * - `callback`: a host closure, still used by the async-from-sync iterator adapter. + */ +export type Reaction = + | { readonly kind: "await"; readonly frames: Array } + | { readonly kind: "settle"; readonly target: Resolving } + | { + readonly kind: "handler" + readonly onFulfilled: Callable | undefined + readonly onRejected: Callable | undefined + readonly derived: Resolving + } + | { readonly kind: "combinator"; readonly combination: Combination; readonly index: number; called: boolean } + | { + readonly kind: "finally" + readonly cleanup: Callable + readonly target: Resolving + readonly site: AstNode | undefined + } + | { readonly kind: "finallyCleaned"; readonly target: Resolving } + | { readonly kind: "finallyOutcome"; readonly target: Resolving; readonly outcome: Completion } + | { readonly kind: "awaitReturn"; readonly generator: GeneratorObj } + | { readonly kind: "callback"; readonly run: (settled: Completion) => Eval | void } + +/** One queued job, as data: run a settled promise's reaction, resume parked frames, or adopt a thenable. */ +export type Job = + | { readonly kind: "reaction"; readonly reaction: Reaction; readonly settled: Completion } + | { readonly kind: "resume"; readonly frames: Array; readonly input: Completion } + | { readonly kind: "thenable"; readonly promise: PromiseObj; readonly thenable: Obj; readonly then: Value } + +/** + * What host work a call asks for. A tool call or a search is data; an extension's work is a live host value the driver + * runs as it is. + */ +export type HostRequest = + | { readonly kind: "tool"; readonly path: ReadonlyArray; readonly args: Array } + | { readonly kind: "search"; readonly args: Array } + | { readonly kind: "effect"; readonly effect: unknown } + +/** + * Host work the program waits on. The machine queues it and hands it to the driver at its next stop; the driver runs + * it and reports how it ended with `Machine.complete`. A concurrent call's outcome waits in the inbox and settles its + * promise in a later job; a blocking call stops the machine, and no other job runs until it ends. + */ +export class HostCall { + status: "queued" | "inFlight" | "completed" = "queued" + constructor( + readonly id: number, + readonly request: HostRequest, + /** What runs: a tool path, or an extension function's label. */ + readonly label: string, + /** The running job waits for it: its outcome continues the frame that asked, before any other job. */ + readonly blocking: boolean, + /** How a concurrent call's result becomes a program value: a tool's crosses as JSON, an extension's is one. */ + readonly result: "json" | "value", + /** Settles a concurrent call's promise. */ + readonly resolving: Resolving | undefined, + /** The call that started the work, where a defect it raises is located. */ + readonly site: AstNode | undefined, + ) {} +} + +/** + * CreateResolvingFunctions as data: settles one promise at most once, however many `Resolver`s share it. `reject` takes + * the failure as thrown; one without a location is located at `site`, the call that created the promise. + */ +export class Resolving { + done = false + constructor( + private readonly scheduler: Settles, + readonly promise: PromiseObj, + readonly site: AstNode | undefined, + ) {} + + resolve(resolution: Value): void { + if (this.done) return + this.done = true + this.scheduler.resolveWith(this.promise, resolution, this.site) + } + + reject(reason: unknown): void { + if (this.done) return + this.done = true + this.scheduler.reject(this.promise, locate(reason, this.site)) + } +} + +type Settles = { + resolveWith(promise: PromiseObj, resolution: Value, site?: AstNode): void + reject(promise: PromiseObj, reason: unknown): void +} + +/** + * Why `Machine.run` stopped: + * - `done`: the program's body ended, with its completion. + * - `host`: start `calls`, newly queued host work, in order (a blocking call is last, and the machine continues only + * once it completes). With no calls, the machine waits for host work already running to complete. + * - `slice`: the step budget is used up; let the host run, then continue. + * - `stuck`: nothing can run or settle any more; `node` is the `await` that waits forever, when one is known. + */ +export type Stop = + | { readonly kind: "done"; readonly completion: Completion } + | { readonly kind: "host"; readonly calls: ReadonlyArray } + | { readonly kind: "slice" } + | { readonly kind: "stuck"; readonly node: AstNode | undefined } diff --git a/packages/codemode/src/machine/statements.ts b/packages/codemode/src/machine/statements.ts index cd65b9cbb26..39980b67d6e 100644 --- a/packages/codemode/src/machine/statements.ts +++ b/packages/codemode/src/machine/statements.ts @@ -49,7 +49,7 @@ import { predeclarePattern, } from "../interpreter/operations.js" import { describeValue } from "../interpreter/references.js" -import { Await, Interruption } from "../interpreter/signals.js" +import { abrupt, Await, Interruption, normal } from "../interpreter/signals.js" import { type Frame, kind, type Out, PENDING, type Pending } from "./frame.js" import { ev, named } from "./expressions.js" import { type Iter, iteratorClose, iteratorStep, openIterator } from "./iteration.js" @@ -357,7 +357,7 @@ const Labeled = kind("LabeledStatement", labeledStatement) // The machine checks for a slice at each loop back-edge, so a long loop lets the host run. const backEdge = (task: MachineTask, frame: Frame, phase: number): boolean => { - if (!task.ctx.runner.tick() && !task.slice) return false + if (!task.machine.tick() && !task.slice) return false task.slice = true frame.phase = phase return true @@ -866,10 +866,15 @@ const Try = kind("TryStatement", tryStatement, { // Slots: the statement running and the value of the last expression statement. Phases: 1 has a statement's result, // 2 has the last expression's value, 3 has a returned thenable settled, 4 has the result copied out. +// The phases of the program's frame: 0 starts, 1 has a statement's result, 2 has the body's value, 3 has it adopted, +// 4 has it copied out, and `ENDED` has ended the program, its outcome recorded on the machine. +const ENDED = 5 + function programStep(task: MachineTask, frame: Frame, node: Program, input: unknown) { const slots = frame.slots const body = node.body let phase = frame.phase + if (phase === ENDED) return undefined let value: Value = undefined let index = 0 if (phase === 0) { @@ -930,9 +935,12 @@ function programStep(task: MachineTask, frame: Frame, node: Program, input: unkn } if (phase === 3) { frame.phase = 4 - return task.native(toBoundary(task.ctx, input as Value), node, false) + const copied = task.native(toBoundary(task.ctx, input as Value), node, false) + if (copied === PENDING) return PENDING + input = copied } - return input + task.machine.outcome = normal(input) + return undefined } const outsideLoop = (result: StatementResult, statement: Statement | ModuleDeclaration) => { @@ -941,4 +949,17 @@ const outsideLoop = (result: StatementResult, statement: Statement | ModuleDecla } } -export const ProgramKind = kind("Program", programStep) +/** + * The program's body, the first job. It is an async boundary: an `await` in it parks the whole body, and how the body + * ends, normally or by a throw, is the program's outcome. + */ +export const ProgramKind = kind("Program", programStep, { + unwind: (task, frame, error) => { + if (frame.phase === ENDED) return false + task.machine.outcome = abrupt(error) + frame.phase = ENDED + return true + }, + cut: () => normal(undefined), + counts: () => true, +}) diff --git a/packages/codemode/src/machine/task.ts b/packages/codemode/src/machine/task.ts index cf09fec71cf..4150264c685 100644 --- a/packages/codemode/src/machine/task.ts +++ b/packages/codemode/src/machine/task.ts @@ -3,24 +3,40 @@ import type { Interpreter } from "../interpreter/interpreter.js" import type { AstNode } from "../interpreter/model.js" import type { Value } from "../interpreter/objects.js" import type { ScopeStack } from "../interpreter/scope.js" -import { type Eval, type Signal, Slice } from "../interpreter/signals.js" +import { + abrupt, + Await, + Call, + type Completion, + Enter, + type Eval, + normal, + type Signal, + Slice, + Yield, +} from "../interpreter/signals.js" import { Frame, type Kind, PASS, PENDING, type Pending } from "./frame.js" -import { dispatch, native } from "./calls.js" +import { boundaryOf, dispatch, native, nativeFrame } from "./calls.js" import type { Machine } from "./machine.js" /** Steps run by every machine task in this process, so tests can see that a program ran on the machine. */ export const stats = { steps: 0 } /** - * A stack of machine frames and the loop that steps it. To the runner it is an activation body, like interpreter - * work: `next` and `throw` run it until it finishes or needs the driver (`Await`, `Yield`, a host call, a slice, or a - * call it cannot make on its own stack), and the driver's answer resumes the frame that asked. + * A stack of machine frames and the loop that steps it. The machine runs every job on its root task. A generator's + * body is a task of its own for now, run by a native frame on the stack that resumes it; to that frame it answers + * `next` and `throw` like interpreter work. + * + * The loop answers what frames ask for on the stack itself: an `await` cuts the stack at the innermost async boundary + * and parks the frames it cut on the promise, a `yield` splits it at the innermost generator body, and a body a + * built-in starts (`Call`) or a generator resumes (`Enter`) is pushed. What it cannot answer (host work, a slice, or an + * `await` or `yield` with no boundary on this stack) it hands to whoever runs it. */ export class MachineTask { readonly stack: Array = [] /** The scope chain of the frame running now. */ scopes: ScopeStack - /** What the running frame asked the driver for; the step loop hands it over once the frame parks. */ + /** What the running frame asked for; the step loop answers it once the frame parks. */ signal: Signal | undefined = undefined /** The host should get a turn at the next step boundary. */ slice = false @@ -103,15 +119,28 @@ export class MachineTask { return native(this, work, site, locate) } - /** Asks the driver for `signal`; its answer (or failure) is the result of the frame this pushes. */ + /** Asks for `signal`; its answer (or failure) is the result of the frame this pushes. */ request(signal: Signal, node: AstNode | undefined): Pending { this.signal = signal this.stack.push(new Frame(passKind, node, PASS, this.scopes, [])) return PENDING } - private drive(ok: boolean, input: unknown): IteratorResult { + /** Puts frames taken off a stack back on this one, counting their calls again. */ + enter(frames: ReadonlyArray): void { + for (const frame of frames) { + this.stack.push(frame) + if (frame.kind.counts?.(frame)) this.machine.depth++ + } + } + + /** + * Runs the stack from the top frame with `input` until it empties (its last result) or a frame asks for something + * this stack cannot answer (the signal). A throw that unwinds past the bottom frame is thrown. + */ + drive(ok: boolean, input: unknown): IteratorResult { const stack = this.stack + const machine = this.machine let value = input while (true) { stats.steps++ @@ -130,6 +159,7 @@ export class MachineTask { } } stack.pop() + if (frame.kind.counts?.(frame)) machine.depth-- } ok = true value = undefined @@ -151,6 +181,7 @@ export class MachineTask { } if (out !== PENDING) { stack.pop() + if (frame.kind.counts?.(frame)) machine.depth-- value = out continue } @@ -158,7 +189,11 @@ export class MachineTask { const signal = this.signal if (signal !== undefined) { this.signal = undefined - return { done: false, value: signal } + const answer = this.answer(signal) + if (answer === undefined) return { done: false, value: signal } + ok = answer.ok + value = answer.ok ? answer.value : answer.error + continue } if (this.slice) { this.slice = false @@ -166,7 +201,59 @@ export class MachineTask { } } } + + // Answers a signal on this stack, giving what the top frame continues with; undefined hands the signal on. + private answer(signal: Signal): Completion | undefined { + const machine = this.machine + if (signal instanceof Call) { + this.stack.push(nativeFrame(this.scopes, signal.body, signal.boundary, true)) + machine.depth++ + if (machine.tick()) this.slice = true + return normal(undefined) + } + if (signal instanceof Enter) { + this.enter(signal.frames) + if (machine.tick()) this.slice = true + return signal.input + } + if (signal instanceof Await) return this.await(signal) + if (signal instanceof Yield) { + const stack = this.stack + for (let index = stack.length - 1; index >= 0; index--) { + const boundary = boundaryOf(stack[index]!) + if (boundary?.generator) return boundary.yielded(signal.value, () => this.split(index)) + } + } + return undefined + } + + // Cuts the stack at the innermost async boundary and parks the frames cut off on the awaited promise; the frame + // below the boundary continues with what the boundary gives it (an async function's promise, say). + private await(signal: Await): Completion | undefined { + const stack = this.stack + for (let index = stack.length - 1; index >= 0; index--) { + const frame = stack[index]! + const cut = frame.kind.cut?.(frame) + if (cut === undefined) continue + let promise + try { + promise = this.machine.promiseResolve(signal.value) + } catch (error) { + return abrupt(error) + } + this.machine.park(this.split(index), promise, signal.node, cut.ok ? (cut.value as Value) : undefined) + return cut + } + return undefined + } + + // Takes the frames from `index` up off the stack; their calls no longer count toward its depth. + private split(index: number): Array { + const frames = this.stack.splice(index) + for (const frame of frames) if (frame.kind.counts?.(frame)) this.machine.depth-- + return frames + } } -// A frame waiting on the driver's answer to a request; the answer passes straight to the frame below. +// A frame waiting on the answer to a request; the answer passes straight to the frame below. const passKind: Kind = { name: "Request", step: (_task, _frame, _node, input) => input } diff --git a/packages/codemode/src/stdlib/array.ts b/packages/codemode/src/stdlib/array.ts index a600cf8db5d..98f0992db91 100644 --- a/packages/codemode/src/stdlib/array.ts +++ b/packages/codemode/src/stdlib/array.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, type Method, methods, prototypeFrom, receiver } from "../interpreter/native.js" import { checkArrayLength, checkStringLength, MAX_ARRAY_LENGTH } from "../interpreter/limits.js" import { invalidData, IteratorSymbol, rangeError, typeError } from "../interpreter/model.js" diff --git a/packages/codemode/src/stdlib/bytes.ts b/packages/codemode/src/stdlib/bytes.ts index 6e6ff0e1f77..7f37b2bca22 100644 --- a/packages/codemode/src/stdlib/bytes.ts +++ b/packages/codemode/src/stdlib/bytes.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { checkArrayLength, checkStringLength } from "../interpreter/limits.js" import { constructor, methods, prototypeFrom, receiver, requiresNew } from "../interpreter/native.js" import { IteratorSymbol, rangeError, syntaxError, typeError } from "../interpreter/model.js" diff --git a/packages/codemode/src/stdlib/collections.ts b/packages/codemode/src/stdlib/collections.ts index 3bec536dc40..37f5eca798b 100644 --- a/packages/codemode/src/stdlib/collections.ts +++ b/packages/codemode/src/stdlib/collections.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, fn, type Method, methods, prototypeFrom, receiver, requiresNew } from "../interpreter/native.js" import { invalidData, IteratorSymbol, typeError } from "../interpreter/model.js" import { diff --git a/packages/codemode/src/stdlib/date.ts b/packages/codemode/src/stdlib/date.ts index 5fd2b12d565..ebf5b4c5193 100644 --- a/packages/codemode/src/stdlib/date.ts +++ b/packages/codemode/src/stdlib/date.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, type Method, methods, prototypeFrom, receiver } from "../interpreter/native.js" import { rangeError } from "../interpreter/model.js" import { DateObj, Obj, coerceToNumber, coerceToString, type Value } from "../interpreter/objects.js" diff --git a/packages/codemode/src/stdlib/headers.ts b/packages/codemode/src/stdlib/headers.ts index 0e052733077..67c156df799 100644 --- a/packages/codemode/src/stdlib/headers.ts +++ b/packages/codemode/src/stdlib/headers.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, methods, prototypeFrom, receiver, requiresNew } from "../interpreter/native.js" import { IteratorSymbol, typeError } from "../interpreter/model.js" import { diff --git a/packages/codemode/src/stdlib/iterator.ts b/packages/codemode/src/stdlib/iterator.ts index 99210a8553e..bdb05bb8f0b 100644 --- a/packages/codemode/src/stdlib/iterator.ts +++ b/packages/codemode/src/stdlib/iterator.ts @@ -16,7 +16,7 @@ import { type Value, } from "../interpreter/objects.js" import { describeValue } from "../interpreter/references.js" -import { type Eval, Interruption } from "../interpreter/runner.js" +import { type Eval, Interruption } from "../interpreter/signals.js" const finished: Step = { done: true, value: undefined } diff --git a/packages/codemode/src/stdlib/json.ts b/packages/codemode/src/stdlib/json.ts index c90c123fcdf..a1aef194525 100644 --- a/packages/codemode/src/stdlib/json.ts +++ b/packages/codemode/src/stdlib/json.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { methods } from "../interpreter/native.js" import { applyCollectionCallback } from "../interpreter/callback.js" import type { Interpreter } from "../interpreter/interpreter.js" diff --git a/packages/codemode/src/stdlib/math.ts b/packages/codemode/src/stdlib/math.ts index 5cb9e11a542..9f979bab658 100644 --- a/packages/codemode/src/stdlib/math.ts +++ b/packages/codemode/src/stdlib/math.ts @@ -3,7 +3,7 @@ import { typeError } from "../interpreter/model.js" import { Obj, coerceToNumber } from "../interpreter/objects.js" import { preserveConsumerError, withPrimitives } from "../interpreter/callback.js" import type { Interpreter } from "../interpreter/interpreter.js" -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" // Bun exposes ES2026 Math.sumPrecise before TypeScript's standard library types. declare global { diff --git a/packages/codemode/src/stdlib/object.ts b/packages/codemode/src/stdlib/object.ts index 825e48c29c7..9ccfcf3486e 100644 --- a/packages/codemode/src/stdlib/object.ts +++ b/packages/codemode/src/stdlib/object.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, methods, receiver } from "../interpreter/native.js" import { type AstNode, AsyncIteratorSymbol, invalidData, IteratorSymbol, typeError } from "../interpreter/model.js" import { diff --git a/packages/codemode/src/stdlib/string.ts b/packages/codemode/src/stdlib/string.ts index c532a890d43..26e506cc7be 100644 --- a/packages/codemode/src/stdlib/string.ts +++ b/packages/codemode/src/stdlib/string.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, fn, type Impl, type Method, methods } from "../interpreter/native.js" import { checkArrayLength, checkStringLength } from "../interpreter/limits.js" import { invalidData, IteratorSymbol, rangeError, typeError } from "../interpreter/model.js" diff --git a/packages/codemode/src/stdlib/url.ts b/packages/codemode/src/stdlib/url.ts index 1957d358116..930d7b1a664 100644 --- a/packages/codemode/src/stdlib/url.ts +++ b/packages/codemode/src/stdlib/url.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, fn, type Method, methods, prototypeFrom, receiver, requiresNew } from "../interpreter/native.js" import { IteratorSymbol, PendingThrow, typeError, uriError } from "../interpreter/model.js" import { diff --git a/packages/codemode/test/limits.test.ts b/packages/codemode/test/limits.test.ts index db5175379d5..3dbe6071d80 100644 --- a/packages/codemode/test/limits.test.ts +++ b/packages/codemode/test/limits.test.ts @@ -91,7 +91,7 @@ describe("the host stays in control", () => { }), ) expect(result).toMatchObject({ ok: true, value: 999_999 }) - // The loop takes well over 10 ms anywhere, and the runner yields to the host every millisecond; a job that kept + // The loop takes well over 10 ms anywhere, and the machine yields to the host every millisecond; a job that kept // the thread for its whole run would let the counter tick at most a couple of times around it. expect(ticks).toBeGreaterThan(10) }) diff --git a/packages/codemode/test/machine.test.ts b/packages/codemode/test/machine.test.ts index 18f117b362c..48a9c8fd375 100644 --- a/packages/codemode/test/machine.test.ts +++ b/packages/codemode/test/machine.test.ts @@ -2,59 +2,65 @@ import { describe, expect, test } from "bun:test" import { Glob } from "bun" import { Effect, Schema } from "effect" import { CodeMode, Tool } from "../src/index.js" +import { parseProgram } from "../src/interpreter/execute.js" +import { Interpreter } from "../src/interpreter/interpreter.js" +import { createBuiltins } from "../src/interpreter/intrinsics.js" +import { normal } from "../src/interpreter/signals.js" +import { Frame } from "../src/machine/frame.js" +import type { Machine } from "../src/machine/machine.js" import { stats } from "../src/machine/task.js" +import { ToolRuntime } from "../src/tool-runtime.js" -// The machine evaluator (`src/machine`) runs programs as explicit frames. These run each program on both evaluators, -// check the machine really ran it, and check both give the same result. +// The machine (`src/machine`) runs programs as explicit frames, and keeps the jobs, promises, and host calls they wait +// on as data. The Effect driver only does what the machine's stops ask. -const runtime = CodeMode.make({ - tools: { - echo: Tool.make({ - description: "Echo a value", - input: Schema.Struct({ value: Schema.Number }), - output: Schema.Number, - execute: ({ value }) => Effect.succeed(value), - }), - }, -}) - -const run = async (code: string, evaluator: "machine" | "legacy") => { - const previous = process.env.CODEMODE_EVALUATOR - process.env.CODEMODE_EVALUATOR = evaluator - try { - return await Effect.runPromise(runtime.execute(code)) - } finally { - if (previous === undefined) delete process.env.CODEMODE_EVALUATOR - else process.env.CODEMODE_EVALUATOR = previous - } +const runtimeTools = { + echo: Tool.make({ + description: "Echo a value", + input: Schema.Struct({ value: Schema.Number }), + output: Schema.Number, + execute: ({ value }) => Effect.succeed(value), + }), } -const both = async (code: string) => { - const legacy = await run(code, "legacy") - const before = stats.steps - const machine = await run(code, "machine") - expect(stats.steps).toBeGreaterThan(before) - expect(machine).toEqual(legacy) - return machine -} +const runtime = CodeMode.make({ tools: runtimeTools }) + +const run = (code: string) => Effect.runPromise(runtime.execute(code)) const value = async (code: string) => { - const result = await both(code) + const before = stats.steps + const result = await run(code) + expect(stats.steps).toBeGreaterThan(before) if (!result.ok) throw new Error(`expected success, got ${result.error.kind}: ${result.error.message}`) return result.value } const error = async (code: string) => { - const result = await both(code) + const result = await run(code) if (result.ok) throw new Error(`expected failure, got value ${JSON.stringify(result.value)}`) return result.error } -describe("machine evaluator", () => { - test("runs programs as frames, and the generator evaluator runs none", async () => { - const before = stats.steps - await run(`return 1 + 2`, "legacy") - expect(stats.steps).toBe(before) +// A machine for `code`, stepped by hand instead of by the Effect driver. +const machineFor = (code: string) => { + const parsed = parseProgram(code) + if (!parsed.ok) throw new Error(parsed.error.message) + const tools = ToolRuntime.make(ToolRuntime.prepare(runtimeTools), undefined, {}) + const ctx = new Interpreter({ tools, builtins: createBuiltins() }) + ctx.start(parsed.source) + return ctx.machine +} + +// The next stop that is not a slice. +const stop = (machine: Machine) => { + for (;;) { + const next = machine.run() + if (next.kind !== "slice") return next + } +} + +describe("machine", () => { + test("runs programs as frames", async () => { expect(await value(`const f = (n) => (n < 2 ? n : f(n - 1) + f(n - 2)); return f(12)`)).toBe(144) }) @@ -151,12 +157,66 @@ describe("machine evaluator", () => { ]) }) - test("runaway recursion stops at the same depth with the same location", async () => { + test("runaway recursion stops with the location of the call that overflowed", async () => { expect((await error(`const f = (n) => f(n + 1); return f(0)`)).message).toBe( "RangeError: Maximum call stack size exceeded (line 1, col 18)", ) }) + test("concurrent tool calls stop the machine once, with every call in one batch", () => { + const machine = machineFor(`return await Promise.all([1, 2, 3].map((value) => tools.echo({ value })))`) + const started = stop(machine) + if (started.kind !== "host") throw new Error(`expected a host stop, got ${started.kind}`) + expect( + started.calls.map((call) => ({ request: call.request, blocking: call.blocking, status: call.status })), + ).toEqual( + [1, 2, 3].map((value) => ({ + request: { kind: "tool", path: ["echo"], args: [{ value }] }, + blocking: false, + status: "inFlight", + })), + ) + // Nothing else can run until host work completes. + expect(stop(machine)).toEqual({ kind: "host", calls: [] }) + for (const [index, call] of [...started.calls.entries()].reverse()) + machine.complete(call.id, normal((index + 1) * 10)) + expect(stop(machine)).toEqual({ kind: "done", completion: normal([10, 20, 30]) }) + expect(started.calls.map((call) => call.status)).toEqual(["completed", "completed", "completed"]) + }) + + test("a blocking host call stops the job that made it until it completes", () => { + const machine = machineFor(`const found = search({ query: "echo" }); return found.items.length`) + const started = stop(machine) + if (started.kind !== "host") throw new Error(`expected a host stop, got ${started.kind}`) + expect(started.calls.map((call) => [call.request.kind, call.blocking])).toEqual([["search", true]]) + expect(stop(machine)).toEqual({ kind: "host", calls: [] }) + machine.complete(started.calls[0]!.id, normal({ items: [1, 2], remaining: 0, next: null })) + expect(stop(machine)).toEqual({ kind: "done", completion: normal(2) }) + }) + + test("await parks the frames it cuts off the stack as data on the promise", () => { + const machine = machineFor(` + const inner = async (x) => { const y = x + 1; return await new Promise(() => {}) } + const outer = async () => [await inner(1)] + return await outer() + `) + const stuck = stop(machine) + expect(stuck.kind).toBe("stuck") + const slices = [...machine.parked.keys()] + expect(slices.length).toBe(3) + for (const frames of slices) { + expect(Array.isArray(frames)).toBe(true) + for (const frame of frames) { + expect(frame).toBeInstanceOf(Frame) + expect(frame.kind.opaque).toBeUndefined() + } + } + // Each slice starts at its async boundary: the program's body, or an async function's call. + expect(slices.map((frames) => frames[0]!.kind.name).sort()).toEqual(["Call", "Call", "Program"]) + expect(machine.depth).toBe(0) + expect(machine.root.stack).toEqual([]) + }) + test("the machine does not import Effect", async () => { const files = await Array.fromAsync(new Glob("src/machine/*.ts").scan({ cwd: `${import.meta.dir}/..` })) expect(files.length).toBeGreaterThan(0) diff --git a/packages/codemode/test/promise.test.ts b/packages/codemode/test/promise.test.ts index 947e3a50d69..c9514c09b17 100644 --- a/packages/codemode/test/promise.test.ts +++ b/packages/codemode/test/promise.test.ts @@ -1344,7 +1344,7 @@ describe("promise construction", () => { }) describe("jobs run to completion", () => { - // The runner pauses a long job to let the host run; the paused job still finishes before any other job starts. + // The machine pauses a long job to let the host run; the paused job still finishes before any other job starts. test("updates to shared state between awaits are not lost", async () => { expect( await value(` diff --git a/packages/codemode/test/resumable.test.ts b/packages/codemode/test/resumable.test.ts index cf863c987ff..7e9b022b2bc 100644 --- a/packages/codemode/test/resumable.test.ts +++ b/packages/codemode/test/resumable.test.ts @@ -1,5 +1,4 @@ import { describe, expect, test } from "bun:test" -import { Context } from "effect" import { parseProgram } from "../src/interpreter/execute.js" import { Interpreter } from "../src/interpreter/interpreter.js" import { createBuiltins } from "../src/interpreter/intrinsics.js" @@ -14,7 +13,6 @@ import { nodeId, sourceOf } from "../src/interpreter/source.js" const interpreter = () => new Interpreter({ tools: ToolRuntime.make(ToolRuntime.prepare({}), undefined, {}), - services: Context.empty(), builtins: createBuiltins(), })