From ad4fd72c55a476edb21a8ddcaa2dc5e7f5d9006f Mon Sep 17 00:00:00 2001 From: James Long Date: Wed, 4 Nov 2026 21:10:43 +0000 Subject: [PATCH] refactor(codemode): run every job on the machine and drive it from a thin Effect loop The machine now owns the scheduler: job queue, inbox, parked awaits, unhandled rejections, and promise operations. An await cuts the stack at the innermost async boundary (the program's frame or an async function's call frame) and parks the frames on the promise as data. Host calls are HostCall records the machine hands to the driver at its stops; blocking host work is a blocking HostCall. The generator evaluator, runner.ts, and CODEMODE_EVALUATOR are gone. --- packages/codemode/package.json | 3 +- packages/codemode/src/data.ts | 4 +- packages/codemode/src/interpreter/bridge.ts | 378 ++++ packages/codemode/src/interpreter/callback.ts | 2 +- packages/codemode/src/interpreter/driver.ts | 128 ++ packages/codemode/src/interpreter/errors.ts | 2 +- packages/codemode/src/interpreter/execute.ts | 28 +- .../codemode/src/interpreter/extensions.ts | 9 +- .../codemode/src/interpreter/generators.ts | 37 +- .../codemode/src/interpreter/interpreter.ts | 1982 +---------------- packages/codemode/src/interpreter/objects.ts | 8 +- packages/codemode/src/interpreter/promises.ts | 82 +- packages/codemode/src/interpreter/runner.ts | 590 ----- packages/codemode/src/interpreter/signals.ts | 65 +- packages/codemode/src/machine/calls.ts | 137 +- packages/codemode/src/machine/frame.ts | 14 +- packages/codemode/src/machine/machine.ts | 366 ++- packages/codemode/src/machine/scheduler.ts | 136 ++ packages/codemode/src/machine/statements.ts | 31 +- packages/codemode/src/machine/task.ts | 107 +- packages/codemode/src/stdlib/array.ts | 2 +- packages/codemode/src/stdlib/bytes.ts | 2 +- packages/codemode/src/stdlib/collections.ts | 2 +- packages/codemode/src/stdlib/date.ts | 2 +- packages/codemode/src/stdlib/headers.ts | 2 +- packages/codemode/src/stdlib/iterator.ts | 2 +- packages/codemode/src/stdlib/json.ts | 2 +- packages/codemode/src/stdlib/math.ts | 2 +- packages/codemode/src/stdlib/object.ts | 2 +- packages/codemode/src/stdlib/string.ts | 2 +- packages/codemode/src/stdlib/url.ts | 2 +- packages/codemode/test/limits.test.ts | 2 +- packages/codemode/test/machine.test.ts | 136 +- packages/codemode/test/promise.test.ts | 2 +- packages/codemode/test/resumable.test.ts | 2 - 35 files changed, 1476 insertions(+), 2797 deletions(-) create mode 100644 packages/codemode/src/interpreter/bridge.ts create mode 100644 packages/codemode/src/interpreter/driver.ts delete mode 100644 packages/codemode/src/interpreter/runner.ts create mode 100644 packages/codemode/src/machine/scheduler.ts diff --git a/packages/codemode/package.json b/packages/codemode/package.json index 6736f2415a7..34a41d3b3b3 100644 --- a/packages/codemode/package.json +++ b/packages/codemode/package.json @@ -24,8 +24,7 @@ "pack": "bun run script/pack.ts", "typecheck": "bun check --no-script", "typecheck:tsc": "tsc --noEmit", - "test": "bun test && bun run test:machine", - "test:machine": "CODEMODE_EVALUATOR=machine bun test" + "test": "bun test" }, "dependencies": { "acorn": "8.15.0", diff --git a/packages/codemode/src/data.ts b/packages/codemode/src/data.ts index b4aa3fe2ff8..6a09f748f17 100644 --- a/packages/codemode/src/data.ts +++ b/packages/codemode/src/data.ts @@ -21,7 +21,7 @@ import { HeadersObj, } from "./interpreter/objects.js" import { typeofValue } from "./interpreter/references.js" -import { type Eval, Slice } from "./interpreter/runner.js" +import { type Eval, Slice } from "./interpreter/signals.js" export type Json = Schema.Json @@ -50,7 +50,7 @@ const walk = ( const stack = new Set() const visit = function* (holder: Obj, key: string, depth: number): Eval { // Shared references expand into a tree, so the work is not bounded by the value's size or depth. - if (ctx.runner.tick()) yield Slice + if (ctx.machine.tick()) yield Slice if (depth > MAX_VALUE_DEPTH) throw rangeError(`Value exceeds the maximum depth of ${MAX_VALUE_DEPTH}.`) const raw = get(holder, key) if (raw instanceof PromiseObj && !boundary) { diff --git a/packages/codemode/src/interpreter/bridge.ts b/packages/codemode/src/interpreter/bridge.ts new file mode 100644 index 00000000000..b632734ffcf --- /dev/null +++ b/packages/codemode/src/interpreter/bridge.ts @@ -0,0 +1,378 @@ +import type { Expression } from "acorn" +import { ToolReference } from "../tool-runtime.js" +import { locate } from "./errors.js" +import type { Interpreter } from "./interpreter.js" +import type { Builtins } from "./intrinsics.js" +import { type AstNode, AsyncIteratorSymbol, GeneratorReturn, IteratorSymbol, Throw, typeError } from "./model.js" +import { native } from "./native.js" +import { + Builtin, + type Cursor, + define, + Fn, + get, + hidden, + IteratorObj, + iteratorResult, + type NativeCall, + type NativeResult, + Obj, + type PromiseObj, + type Step, + type Value, +} from "./objects.js" +import { + builtinCursor, + type CustomIterator, + notAFunction, + requireIteratorMethod, + requireIteratorObject, +} from "./operations.js" +import { capability } from "./promises.js" +import { describeValue } from "./references.js" +import { + Await, + type Eval, + type GeneratorRequest, + Interruption, + Invoke, + isEval, + normal, + Slice, + Yield, +} from "./signals.js" + +/** + * Interpreter work for the built-ins that still run as host generators in native frames: calls into the program, + * iterating, `yield*`, and the async-from-sync iterator adapter. A program function it calls is an `Invoke` the + * machine runs on its own stack. + * + * Phase 2c moves `yield*` and the async-from-sync adapter onto machine frames, and phase 3 replaces the built-ins that + * call program code with the prelude, after which this goes away. + */ +export class Bridge { + constructor(private readonly ctx: Interpreter) {} + + /** Await: suspends the innermost async body until `value` settles. */ + *awaitValue(value: Value, node?: AstNode): Eval { + return (yield new Await(value, node)) as Value + } + + *call(callable: Value, thisValue: Value, args: Array, node?: AstNode, callee?: Expression): Eval { + const result = this.dispatch(callable, thisValue, args, node, callee) + if (isEval(result)) return yield* result + return result instanceof Invoke ? ((yield result) as Value) : result + } + + // The single dispatch for every call a built-in makes. A program function is an `Invoke` for the machine to run on + // its stack; a built-in that returns a value needs no generator. + private dispatch( + callable: Value, + thisValue: Value, + args: Array, + node?: AstNode, + callee?: Expression, + ): Value | Eval | Invoke { + if (callable instanceof Fn) return new Invoke(callable, thisValue, args, node) + if (callable instanceof Builtin) { + const ctx = this.ctx + // A loop made only of built-in calls has no program call or loop back-edge to count, so count the call. + if (ctx.machine.tick()) + return afterSlice(() => this.native(() => callable.invoke(ctx, thisValue, args, node), node)) + return this.native(() => callable.invoke(ctx, thisValue, args, node), node) + } + if (callable instanceof ToolReference) { + if (callable.path.length === 0) { + throw typeError("The tools root is not callable.", callee ?? node) + } + return this.ctx.toolCall(callable.path, args, node) + } + throw notAFunction(callee, node) + } + + // Built-ins throw without a location, synchronously or inside their work; the call site supplies it. + private native(body: () => NativeResult, node?: AstNode): Value | Eval { + let result: NativeResult + try { + result = body() + } catch (error) { + throw locate(error, node) + } + return isEval(result) ? located(result, node) : result + } + + // CreateAsyncFromSyncIterator: an async iterator over a synchronous one. Each `next()` promise settles a job after + // the value does, and a rejected value closes the synchronous iterator first; `return()` calls the synchronous + // iterator's own `return`, or settles at once when it has none. + asyncFromSync( + builtin: Cursor | undefined, + custom: CustomIterator | undefined, + node: AstNode | undefined, + ): CustomIterator { + const ctx = this.ctx + const machine = ctx.machine + const builtins = ctx.builtins + const result = (value: Value, done: boolean) => iteratorResult(builtins.Object, value, done) + const self = this + // IteratorNext passes the value `next` received, when it received one. + const syncNext = function* (args: Array): Eval { + if (!custom) return yield* builtin!.next() + const object = requireIteratorObject( + yield* self.call(custom.next, custom.iterator, args, node), + "Iterator next() result", + node, + ) + return { done: Boolean(get(object, "done")), value: get(object, "value") } + } + // The synchronous iterator's `return`, looked up when it is needed; built-in collection iterators have none. + const syncReturn = (): Value => (custom ? get(custom.iterator, "return") : undefined) + const settle = (step: Step, closeOnRejection: boolean, made: ReturnType) => { + const rejected = (reason: unknown) => { + if (step.done || !closeOnRejection || reason instanceof Interruption) return made.reject(reason) + const body = function* (): Eval { + try { + if (custom) yield* self.closeIterator(custom, node) + else yield* builtin!.close() + } catch { + // The rejection wins over a failing close. + } + made.reject(reason) + } + return body() + } + let wrapper: PromiseObj + try { + wrapper = machine.promiseResolve(step.value) + } catch (error) { + const body = rejected(error) + if (body !== undefined) + machine.enqueue({ kind: "resume", frames: [machine.job(body)], input: normal(undefined) }) + return + } + machine.then(wrapper, { + kind: "callback", + run: (settled) => + settled.ok ? made.resolve(result(settled.value as Value, step.done)) : rejected(settled.error), + }) + } + const next = asyncFromSyncMethod(builtins, "next", function* (_, args) { + const made = capability(ctx, node) + try { + settle(yield* syncNext(args), true, made) + } catch (error) { + made.reject(error) + } + return made.promise + }) + const close = asyncFromSyncMethod(builtins, "return", function* (_, args) { + const made = capability(ctx, node) + try { + const method = syncReturn() + if (method === undefined || method === null) { + if (builtin !== undefined) yield* builtin.close() + made.resolve(result(args[0], true)) + return made.promise + } + const returned = requireIteratorObject( + yield* self.call(requireIteratorMethod(method, "Iterator return", node), custom!.iterator, args, node), + "Iterator return() result", + node, + ) + settle({ done: Boolean(get(returned, "done")), value: get(returned, "value") }, false, made) + } catch (error) { + made.reject(error) + } + return made.promise + }) + const raise = asyncFromSyncMethod(builtins, "throw", function* (_, args) { + const made = capability(ctx, node) + try { + const method = custom ? get(custom.iterator, "throw") : undefined + if (method === undefined || method === null) { + if (custom) yield* self.closeIterator(custom, node) + else yield* builtin!.close() + made.reject(typeError("The delegated iterator does not provide a throw() method.", node)) + return made.promise + } + const thrown = requireIteratorObject( + yield* self.call(requireIteratorMethod(method, "Iterator throw", node), custom!.iterator, args, node), + "Iterator throw() result", + node, + ) + settle({ done: Boolean(get(thrown, "done")), value: get(thrown, "value") }, true, made) + } catch (error) { + made.reject(error) + } + return made.promise + }) + const adapter = new Obj(builtins.Object) + define(adapter, "next", next, hidden) + define(adapter, "return", close, hidden) + define(adapter, "throw", raise, hidden) + return { iterator: adapter, next, asynchronous: true } + } + + *iterate(value: Value, node?: AstNode): Eval { + const cursor = builtinCursor(this.ctx.builtins, value) + if (cursor !== undefined) return cursor + const iterator = yield* this.customIterator(value, node, false) + return iterator === undefined ? undefined : this.customCursor(iterator, node) + } + + /** GetIteratorDirect: drive an iterator by its own `next`, without asking for `[Symbol.iterator]`. */ + iterateDirect(value: Value, node?: AstNode): Cursor { + if (value instanceof IteratorObj) return value.cursor + if (!(value instanceof Obj)) { + throw typeError(`An iterator must be an object, received ${describeValue(value)}.`, node) + } + return this.customCursor( + { + iterator: value, + next: requireIteratorMethod(get(value, "next"), "Iterator next", node), + asynchronous: false, + }, + node, + ) + } + + private customCursor(iterator: CustomIterator, node: AstNode | undefined): Cursor { + return { + next: () => this.nextIteratorResult(iterator, node), + close: () => this.closeIterator(iterator, node), + } + } + + private *customIterator( + value: Value, + node: AstNode | undefined, + allowAsync = true, + ): Eval { + if (!(value instanceof Obj)) return undefined + const asyncMethod = allowAsync ? get(value, AsyncIteratorSymbol) : undefined + const method = asyncMethod ?? get(value, IteratorSymbol) + if (method === undefined || method === null) return undefined + const iterator = yield* this.call(requireIteratorMethod(method, "Iterator method", node), value, [], node) + const object = requireIteratorObject(iterator, "Iterator method result", node) + return { + iterator: object, + next: requireIteratorMethod(get(object, "next"), "Iterator next", node), + asynchronous: asyncMethod !== undefined && asyncMethod !== null, + } + } + + private *nextIteratorResult(iterator: CustomIterator, node: AstNode | undefined): Eval { + const called = yield* this.call(iterator.next, iterator.iterator, [], node) + const object = requireIteratorObject( + iterator.asynchronous ? yield* this.awaitValue(called, node) : called, + "Iterator next() result", + node, + ) + return { done: Boolean(get(object, "done")), value: get(object, "value") } + } + + private *closeIterator(iterator: CustomIterator, node: AstNode | undefined): Eval { + const close = get(iterator.iterator, "return") + if (close === undefined || close === null) return + const method = requireIteratorMethod(close, "Iterator return", node) + const called = yield* this.call(method, iterator.iterator, [], node) + requireIteratorObject( + iterator.asynchronous ? yield* this.awaitValue(called, node) : called, + "Iterator return() result", + node, + ) + } + + // Yield `value` and continue with the request that resumes the body: `next` supplies the value of the `yield`, + // `throw` throws there, and `return` unwinds through `finally` blocks (an async generator awaits the value first). + private *suspendGenerator(value: Value, asynchronous: boolean): Eval { + const request = (yield new Yield(value)) as GeneratorRequest + if (request.kind === "next") return request.value + if (request.kind === "throw") throw new Throw(request.value) + throw new GeneratorReturn(asynchronous ? yield* this.awaitValue(request.value) : request.value) + } + + *delegateYield(value: Value, node: AstNode, asynchronous: boolean): Eval { + const builtin = builtinCursor(this.ctx.builtins, value) + if (builtin !== undefined && !asynchronous) { + while (true) { + const step = yield* builtin.next() + if (step.done) return undefined + try { + yield* this.suspendGenerator(step.value, asynchronous) + } catch (error) { + if (error instanceof GeneratorReturn) { + yield* builtin.close() + throw error + } + if (error instanceof Throw) { + yield* builtin.close() + throw typeError("The delegated iterator does not provide a throw() method.", node) + } + throw error + } + } + } + + const custom = builtin === undefined ? yield* this.customIterator(value, node, asynchronous) : undefined + if (builtin === undefined && !custom) throw typeError("yield* requires a compatible iterable value.", node) + // An async generator delegates to a synchronous iterator through CreateAsyncFromSyncIterator, as JS does. + const iterator = asynchronous && !custom?.asynchronous ? this.asyncFromSync(builtin, custom, node) : custom! + let kind: GeneratorRequest["kind"] = "next" + let input: Value = undefined + while (true) { + const method = kind === "next" ? iterator.next : get(iterator.iterator, kind) + if (method === undefined || method === null) { + if (kind === "return") throw new GeneratorReturn(asynchronous ? yield* this.awaitValue(input, node) : input) + yield* this.closeIterator(iterator, node) + throw typeError("The delegated iterator does not provide a throw() method.", node) + } + const called = yield* this.call( + requireIteratorMethod(method, `Iterator ${kind}`, node), + iterator.iterator, + [input], + node, + ) + const result = requireIteratorObject( + iterator.asynchronous ? yield* this.awaitValue(called, node) : called, + `Iterator ${kind}() result`, + node, + ) + const done = Boolean(get(result, "done")) + const resultValue = get(result, "value") + if (done) { + if (kind === "return") throw new GeneratorReturn(resultValue) + return resultValue + } + + try { + // Delegated values are yielded as they are; an async generator's inner promise already settled them. + input = yield* this.suspendGenerator(resultValue, asynchronous) + kind = "next" + } catch (error) { + if (!(error instanceof GeneratorReturn) && !(error instanceof Throw)) throw error + kind = error instanceof GeneratorReturn ? "return" : "throw" + input = error.value + } + } + } +} + +// The async-from-sync adapter still keeps its iterator in host closures, so its methods are made per iteration. +const asyncFromSyncMethod = (builtins: Builtins, name: string, call: NativeCall) => + native(builtins, { name, length: 1, call, runtime: "async-from-sync" }) + +// Finishes a built-in's work, locating a failure at the call that started it. +function* located(work: Eval, node: AstNode | undefined): Eval { + try { + return yield* work + } catch (error) { + throw locate(error, node) + } +} + +// Lets the host run first, then starts the work. +function* afterSlice(start: () => Value | Eval): Eval { + yield Slice + const result = start() + return isEval(result) ? yield* result : result +} diff --git a/packages/codemode/src/interpreter/callback.ts b/packages/codemode/src/interpreter/callback.ts index c5319c22967..4de68426071 100644 --- a/packages/codemode/src/interpreter/callback.ts +++ b/packages/codemode/src/interpreter/callback.ts @@ -14,7 +14,7 @@ import { type Value, } from "./objects.js" import { isOpaque, typeofValue } from "./references.js" -import { type Eval, Interruption, isEval } from "./runner.js" +import { type Eval, Interruption, isEval } from "./signals.js" /** IteratorClose: a consumer failure closes the iterator and wins over any close failure, except that a generator's * return() is a return completion, so a failing close wins over it, as after `break`. */ diff --git a/packages/codemode/src/interpreter/driver.ts b/packages/codemode/src/interpreter/driver.ts new file mode 100644 index 00000000000..074d4571033 --- /dev/null +++ b/packages/codemode/src/interpreter/driver.ts @@ -0,0 +1,128 @@ +import { Cause, type Context, Effect, Exit, Fiber } from "effect" +import type { Machine } from "../machine/machine.js" +import type { HostCall } from "../machine/scheduler.js" +import type { ToolRuntime } from "../tool-runtime.js" +import { locate } from "./errors.js" +import { PendingThrow } from "./model.js" +import { abrupt, type Completion, Interruption, normal } from "./signals.js" + +/** + * Runs a machine in Effect: the only place program execution meets the host. It calls `machine.run` until the program + * ends, and does what each stop asks: yields to the Effect scheduler for a slice, and starts the host calls the machine + * queued, as fibers with the caller's services, reporting each outcome back with `machine.complete`. Fibers live only + * here, by host call id, and never become part of the machine's state. + */ +export class Driver { + private readonly fibers = new Map>() + private wake: (() => void) | undefined + private stopping = false + + constructor( + private readonly machine: Machine, + private readonly tools: ToolRuntime, + /** The services host work runs with: the caller's, so tools see its context, tracing, and clock. */ + private readonly services: Context.Context, + ) {} + + /** + * Runs the machine until the program's body completes, then returns how it ended. Host work it started may still be + * running; `interrupt` stops it. Host work interrupted on its own that reaches the program's body, or blocking host + * work that is interrupted, interrupts the execution. + */ + run(): Effect.Effect { + const self = this + const machine = this.machine + return Effect.gen(function* () { + while (true) { + const stop = machine.run() + if (stop.kind === "slice") { + yield* Effect.yieldNow + continue + } + if (stop.kind === "host") { + if (stop.calls.length === 0) { + yield* self.settled + continue + } + for (const call of stop.calls) { + if (!call.blocking) { + self.start(call) + continue + } + const exit = yield* Effect.exit(self.work(call)) + if (Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) return yield* Effect.failCause(exit.cause) + machine.complete(call.id, Exit.isSuccess(exit) ? normal(exit.value) : abrupt(Cause.squash(exit.cause))) + } + continue + } + if (stop.kind === "stuck") { + return abrupt( + new PendingThrow( + "Error", + "The program is waiting on a promise that can never settle: nothing else is running, so nothing can resolve it. Code Mode has no timers or sleep, so don't wait or poll inside a program.", + stop.node, + ), + ) + } + const outcome = stop.completion + if (!outcome.ok && outcome.error instanceof Interruption) return yield* Effect.failCause(outcome.error.cause) + return outcome + } + }) + } + + /** Stops host work still running, waiting for its cleanup. */ + interrupt(): Effect.Effect { + const self = this + return Effect.gen(function* () { + self.stopping = true + while (self.fibers.size > 0) yield* Fiber.interruptAll([...self.fibers.values()]) + }) + } + + private work(call: HostCall): Effect.Effect { + const request = call.request + if (request.kind === "tool") return this.tools.execute(request.path, request.args) + if (request.kind === "search") return this.tools.search(request.args) + // An extension's work, made by the extension bridge with the program's services in its requirements. + return request.effect as Effect.Effect + } + + // Starts concurrent host work now; its outcome goes back to the machine's inbox, in completion order. + private start(call: HostCall): void { + // Starts synchronously, so admission and hooks happen before the machine runs another job. + const fiber = Effect.runForkWith(this.services)(this.work(call)) + this.fibers.set(call.id, fiber) + fiber.addObserver((exit) => { + this.fibers.delete(call.id) + // Interrupted by the program ending: nothing is left to settle. + if (this.stopping) return + this.machine.complete( + call.id, + Exit.isSuccess(exit) + ? normal(exit.value) + : abrupt( + Cause.hasInterruptsOnly(exit.cause) + ? new Interruption(exit.cause) + : Cause.hasDies(exit.cause) + ? locate(Cause.squash(exit.cause), call.site) + : Cause.squash(exit.cause), + ), + ) + this.wake?.() + }) + } + + private get settled(): Effect.Effect { + return Effect.callback((resume) => { + if (this.machine.ready) return resume(Effect.void) + this.wake = () => { + this.wake = undefined + resume(Effect.void) + } + return Effect.sync(() => { + this.wake = undefined + }) + }) + } +} diff --git a/packages/codemode/src/interpreter/errors.ts b/packages/codemode/src/interpreter/errors.ts index 34d9eb5b332..2896b1ed213 100644 --- a/packages/codemode/src/interpreter/errors.ts +++ b/packages/codemode/src/interpreter/errors.ts @@ -19,7 +19,7 @@ import { type Value, } from "./objects.js" import type { Interpreter } from "./interpreter.js" -import type { Eval } from "./runner.js" +import type { Eval } from "./signals.js" import { toPrimitiveString } from "./callback.js" import { formatValue } from "../stdlib/console.js" diff --git a/packages/codemode/src/interpreter/execute.ts b/packages/codemode/src/interpreter/execute.ts index f2969d85641..becfb04c56b 100644 --- a/packages/codemode/src/interpreter/execute.ts +++ b/packages/codemode/src/interpreter/execute.ts @@ -7,9 +7,9 @@ import { createBuiltins } from "./intrinsics.js" import { Interpreter } from "./interpreter.js" import type { Value } from "./objects.js" import { formatLocation, sourceLocation } from "./model.js" -import type { Runner } from "./runner.js" +import { Driver } from "./driver.js" +import type { Machine } from "../machine/machine.js" import { Source } from "./source.js" -import type { Evaluator } from "../machine/machine.js" export const executeProgram = ( code: string, @@ -33,24 +33,25 @@ export const executeProgram = ( const logs: Array = [] const logged = () => (logs.length > 0 ? { logs: [...logs] } : {}) // Set only after copy-out so timeouts cannot report invalid values as completed. - let returned: { value: DataValue; runner: Runner } | undefined - let runner: Runner | undefined + let returned: { value: DataValue; machine: Machine } | undefined + let driver: Driver | undefined const base = Effect.gen(function* () { const parsed = parseProgram(code) if (!parsed.ok) return { ok: false, error: parsed.error, toolCalls: [] } satisfies Result const services = yield* Effect.context() - const ctx = new Interpreter({ tools, services, builtins, logs, globals, evaluator: evaluator() }) - runner = ctx.runner - const outcome = yield* runner.run(ctx.main(parsed.source)) + const ctx = new Interpreter({ tools, builtins, logs, globals }) + ctx.start(parsed.source) + driver = new Driver(ctx.machine, tools, services) + const outcome = yield* driver.run() if (!outcome.ok) { return { ok: false, error: normalizeError(outcome.error), ...logged(), toolCalls: tools.calls } satisfies Result } const result = (outcome.value as DataValue | undefined) ?? null - returned = { value: result, runner } + returned = { value: result, machine: ctx.machine } // Host work the program left running is interrupted, not awaited. - yield* runner.interrupt() - const warnings = runner.diagnostics() + yield* driver.interrupt() + const warnings = ctx.machine.diagnostics() return { ok: true, value: result, @@ -58,7 +59,7 @@ export const executeProgram = ( ...logged(), toolCalls: tools.calls, } satisfies Result - }).pipe(Effect.ensuring(Effect.suspend(() => runner?.interrupt() ?? Effect.void))) + }).pipe(Effect.ensuring(Effect.suspend(() => driver?.interrupt() ?? Effect.void))) const timeoutMs = limits.timeoutMs const operation = timeoutMs === undefined @@ -85,7 +86,7 @@ export const executeProgram = ( kind: "TimeoutExceeded", message: `The program returned, but background work was still running at the ${timeoutMs}ms timeout and was interrupted. Await all started promises.`, }, - ...returned.runner.diagnostics(), + ...returned.machine.diagnostics(), ], ...logged(), toolCalls: tools.calls, @@ -112,9 +113,6 @@ export const executeProgram = ( }) } -// Internal while the machine evaluator is ported: `CODEMODE_EVALUATOR=machine` runs programs on it. -const evaluator = (): Evaluator => (process.env.CODEMODE_EVALUATOR === "machine" ? "machine" : "legacy") - export const parseProgram = ( code: string, ): { ok: true; program: Program; source: Source } | { ok: false; error: Diagnostic } => { diff --git a/packages/codemode/src/interpreter/extensions.ts b/packages/codemode/src/interpreter/extensions.ts index 22218606b14..7ac50eadd58 100644 --- a/packages/codemode/src/interpreter/extensions.ts +++ b/packages/codemode/src/interpreter/extensions.ts @@ -28,6 +28,7 @@ import { type Value, } from "./objects.js" import { describeValue, isOpaque } from "./references.js" +import { Host } from "./signals.js" /** * The global bindings of one run's extensions. Everything crossing the boundary is converted: plain data and built-in @@ -184,18 +185,18 @@ export const extensionGlobals = ( label, site, ) - if (before) yield ctx.runner.blocking(before) + if (before) yield new Host({ kind: "effect", effect: before }) const host = callHost(value, args) if (host.async) return pending(settle(host.effect), label, site) - return (yield ctx.runner.blocking(settle(host.effect))) as Value + return (yield new Host({ kind: "effect", effect: settle(host.effect) })) as Value }, }) } // An asynchronous host call: the program holds a promise that settles when the host's work does. const pending = (work: Effect.Effect, label: string, site: AstNode | undefined) => { - const resolving = ctx.runner.resolving(ctx.runner.newPromise(), site) - ctx.runner.host(work, { label, result: "value", resolving, site }) + const resolving = ctx.machine.resolving(ctx.machine.newPromise(), site) + ctx.machine.host({ kind: "effect", effect: work }, { label, result: "value", resolving, site }) return resolving.promise } diff --git a/packages/codemode/src/interpreter/generators.ts b/packages/codemode/src/interpreter/generators.ts index aee63489aa0..f13d6067dd6 100644 --- a/packages/codemode/src/interpreter/generators.ts +++ b/packages/codemode/src/interpreter/generators.ts @@ -5,13 +5,14 @@ import type { Interpreter } from "./interpreter.js" import { abrupt, type Boundary, + Call, type Completion, Enter, - type Entry, type Eval, type GeneratorRequest, normal, -} from "./runner.js" +} from "./signals.js" +import type { Frame } from "../machine/frame.js" /** `next`/`return`/`throw` on the generator prototypes; async generators answer with promises. */ export const generatorGlobals = (ctx: Interpreter): void => { @@ -49,17 +50,15 @@ export const generatorGlobals = (ctx: Interpreter): void => { const result = (ctx: Interpreter, value: Value, done: boolean) => iteratorResult(ctx.builtins.Object, value, done) -// Puts the suspended body back on the stack and continues it with `request`; the boundary answers the caller when -// the body next yields or ends. -const resume = (ctx: Interpreter, generator: GeneratorObj, request: GeneratorRequest): Enter => { +// Starts the body behind its boundary, or puts the suspended body's frames back on the stack and continues it with +// `request`; the boundary answers the caller when the body next yields or ends. +const resume = (ctx: Interpreter, generator: GeneratorObj, request: GeneratorRequest): Call | Enter => { const started = generator.state !== "suspendedStart" generator.state = "executing" - if (!started) { - return new Enter([{ body: generator.body, boundary: new GeneratorBoundary(ctx, generator) }], normal(undefined)) - } - const entries = generator.entries! - generator.entries = undefined - return new Enter(entries, normal(request)) + if (!started) return new Call(generator.body, new GeneratorBoundary(ctx, generator)) + const frames = generator.frames! + generator.frames = undefined + return new Enter(frames, normal(request)) } // A synchronous generator step: the body runs on the caller's stack until it yields, returns, or throws. @@ -76,9 +75,9 @@ function* step(ctx: Interpreter, generator: GeneratorObj, kind: GeneratorR // AsyncGenerator.prototype.next/return/throw: queue the request with its promise, and run the body now if it is // waiting for one. A request that arrives while the body runs is answered when the body reaches it. function* request(ctx: Interpreter, thisValue: Value, kind: GeneratorRequestKind, value: Value): Eval { - const runner = ctx.runner - const promise = runner.newPromise() - const resolving = runner.resolving(promise) + const machine = ctx.machine + const promise = machine.newPromise() + const resolving = machine.resolving(promise) if (!(thisValue instanceof GeneratorObj) || !thisValue.asynchronous) { resolving.reject(typeError(`AsyncGenerator.prototype.${kind} called on incompatible receiver.`)) return promise @@ -138,9 +137,9 @@ const drainQueue = (ctx: Interpreter, generator: GeneratorObj) => { // AsyncGeneratorAwaitReturn: a `return(value)` on an ended or unstarted generator settles once `value` does. const awaitReturn = (ctx: Interpreter, generator: GeneratorObj) => { - const runner = ctx.runner + const machine = ctx.machine try { - runner.then(runner.promiseResolve(generator.queue[0]!.request.value), { kind: "awaitReturn", generator }) + machine.then(machine.promiseResolve(generator.queue[0]!.request.value), { kind: "awaitReturn", generator }) } catch (error) { finishAwaitReturn(ctx, generator, abrupt(error)) } @@ -179,10 +178,10 @@ class GeneratorBoundary implements Boundary { return normal(undefined) } - yielded(value: Value, split: () => Array): Completion { + yielded(value: Value, split: () => Array): Completion { const generator = this.target if (!generator.asynchronous) { - generator.entries = split() + generator.frames = split() generator.state = "suspendedYield" return normal(result(this.ctx, value, false)) } @@ -190,7 +189,7 @@ class GeneratorBoundary implements Boundary { completeStep(this.ctx, generator, normal(value), false) const next = generator.queue[0] if (next !== undefined) return normal(next.request) - generator.entries = split() + generator.frames = split() generator.state = "suspendedYield" return normal(undefined) } diff --git a/packages/codemode/src/interpreter/interpreter.ts b/packages/codemode/src/interpreter/interpreter.ts index 60028d2190e..53a9d5c9b4e 100644 --- a/packages/codemode/src/interpreter/interpreter.ts +++ b/packages/codemode/src/interpreter/interpreter.ts @@ -1,216 +1,56 @@ -import type { - ArrayExpression, - ArrayPattern, - AssignmentPattern, - AssignmentExpression, - AssignmentProperty, - AwaitExpression, - BinaryExpression, - BlockStatement, - BreakStatement, - CallExpression, - ChainExpression, - ConditionalExpression, - ContinueStatement, - DoWhileStatement, - Expression, - ForInStatement, - ForOfStatement, - ForStatement, - Identifier, - IfStatement, - ImportExpression, - LabeledStatement, - LogicalExpression, - MemberExpression, - ModuleDeclaration, - NewExpression, - ObjectExpression, - Pattern, - Program, - Property, - ReturnStatement, - SequenceExpression, - SpreadElement, - Statement, - SwitchStatement, - TaggedTemplateExpression, - TemplateLiteral, - ThrowStatement, - TryStatement, - UnaryExpression, - UpdateExpression, - VariableDeclaration, - WhileStatement, - YieldExpression, -} from "acorn" -import type { Context } from "effect" +import type { TaggedTemplateExpression } from "acorn" import { fromJson, type Json, toBoundary } from "../data.js" -import { ToolReference, type ToolRuntime } from "../tool-runtime.js" -import { - type AstNode, - AsyncIteratorSymbol, - type Binding, - GeneratorReturn, - IteratorSymbol, - OptionalShortCircuit, - invalidData, - PendingThrow, - syntaxError, - Throw, - rangeError, - type StatementResult, - typeError, - unsupportedSyntax, -} from "./model.js" -import { checkStringLength } from "./limits.js" -import { locate, materialize } from "./errors.js" -import type { Builtins } from "./intrinsics.js" +import type { ToolRuntime } from "../tool-runtime.js" +import { Machine } from "../machine/machine.js" +import { Bridge } from "./bridge.js" import { globals } from "./globals.js" -import { - assign, - define, - get, - IteratorObj, - type Cursor, - has, - hidden, - Builtin, - nativeRule, - Arguments, - Arr, - Fn, - GeneratorObj, - Obj, - PromiseObj, - iteratorResult, - remove, - coerceToNumber, - coerceToString, - entries, - type NativeCall, - type NativeResult, - type Step, - type Value, -} from "./objects.js" -import { type Hint, preserveConsumerError, toPrimitive } from "./callback.js" -import { - AsyncBoundary, - Await, - type Body, - Call, - type Eval, - generatorBody, - type GeneratorRequest, - Interruption, - Invoke, - isEval, - MAX_CALL_DEPTH, - Runner, - Slice, - Task, - Yield, - normal, -} from "./runner.js" -import { moduleName, resolveModule } from "./modules.js" -import { describeValue, isOpaque, typeofValue } from "./references.js" -import { native } from "./native.js" -import { capability, promiseLike } from "./promises.js" -import { ScopeStack } from "./scope.js" -import { type Evaluator, Machine } from "../machine/machine.js" +import type { Builtins } from "./intrinsics.js" import { IntrinsicNames } from "./intrinsic-names.js" +import { type AstNode, type Binding, PendingThrow } from "./model.js" +import { moduleName, resolveModule } from "./modules.js" +import { type Arr, type Cursor, define, entries, nativeRule, Obj, type Value } from "./objects.js" +import type { CustomIterator } from "./operations.js" +import { promiseLike } from "./promises.js" +import { ScopeStack } from "./scope.js" +import { AsyncBoundary, Call, type Eval, Host } from "./signals.js" import type { NodeId, Source } from "./source.js" -import { constructRegExp } from "../stdlib/regexp.js" -import { enumerableSource } from "../stdlib/object.js" -import { - applyBinaryOperator, - assignToReference, - bindImport, - builtinCursor, - collectPatternNames, - createFunction, - createsClosure, - type CustomIterator, - declaresLexically, - destructuredProperty, - enumerableKeys, - hoistFunctions, - hoistVars, - instanceofValue, - isAbrupt, - isAnonymousFunction, - isSimple, - isThenable, - literal, - loopDeclaration, - loopExit, - type MemberReference, - none, - notAFunction, - notConstructible, - operatorConversion, - predeclareLexical, - predeclarePattern, - propertyKey, - readReference, - requireIteratorMethod, - requireIteratorObject, - requireMemberBase, - resolveProperty, - templateObject, - usesArguments, - writableReference, -} from "./operations.js" -import { compoundOperators } from "../stdlib/value.js" -/** Marks an operand a fast path has not evaluated yet, since `undefined` is a value. */ -const unevaluated: unique symbol = Symbol("codemode.unevaluated") -type Reference = MemberReference | ToolReference | { value: Value } | typeof OptionalShortCircuit - -/** One program execution: the tool bridge, job queue, captured logs, and the global scope built once. */ +/** One program execution: the tool bridge, the machine that runs it, captured logs, and the global scope built once. */ export class Interpreter { readonly tools: ToolRuntime - readonly runner: Runner readonly builtins: Builtins readonly logs: Array + /** Runs program code, and keeps the job queue, promises, and host calls the program waits on. */ + readonly machine: Machine /** Names for this runtime's intrinsics, taken before any program code can change a built-in. */ readonly intrinsics: IntrinsicNames /** Template objects by site: a tag sees the same `strings` array every time its literal is evaluated, as in JS. */ readonly templates = new WeakMap() /** The parsed sources this execution runs, by id, so a `NodeId` resolves to its node. */ readonly sources = new Map() - /** The machine evaluator, when this execution runs on it instead of the generator evaluator. */ - readonly machine: Machine | undefined - private readonly root: Frame + private readonly bridge: Bridge private readonly globalScope: Map private readonly modules = new Map() private readonly namespaces = new Map() constructor(options: { readonly tools: ToolRuntime - /** The caller's services, which host work runs with. */ - readonly services: Context.Context readonly builtins: Builtins readonly logs?: Array readonly globals?: (ctx: Interpreter) => ReadonlyArray - /** Which evaluator runs program code; the generator evaluator unless the machine is asked for. */ - readonly evaluator?: Evaluator }) { this.tools = options.tools this.builtins = options.builtins this.logs = options.logs ?? [] - this.runner = new Runner(this, options.services) - // The machine never runs host work with the caller's services itself; it asks the runner, which is typed by them. - this.machine = options.evaluator === "machine" ? new Machine(this as Interpreter) : undefined + // The machine never runs host work itself, so it does not need the services that work runs with. + this.machine = new Machine(this as Interpreter) + this.bridge = new Bridge(this) // Program code has no receiver: top-level `this` is undefined, as in a module. - const globalScope = new Map([["this", { mutable: false, value: undefined }]]) - // Calling back into the program never reads frame state, so any frame serves; the root is always alive. - this.root = new Frame(this, new ScopeStack([globalScope])) - this.globalScope = globalScope + this.globalScope = new Map([["this", { mutable: false, value: undefined }]]) const intrinsics = globals(this) this.intrinsics = new IntrinsicNames(intrinsics, this.builtins) for (const [name, value] of [...intrinsics, ...(options.globals?.(this) ?? [])]) { - globalScope.set(name, { mutable: false, value }) + this.globalScope.set(name, { mutable: false, value }) } nativeRule.sealed.add(this.builtins.Function) } @@ -250,14 +90,10 @@ export class Interpreter { return namespace } - /** The program's body, as the first job: its completion value, copied out for the host. */ - main(source: Source): Body { + /** Queues the program's body as the first job; its completion value is copied out for the host. */ + start(source: Source): void { this.sources.set(source.id, source) - return this.machine === undefined ? this.legacyMain(source) : this.machine.program(source) - } - - private *legacyMain(source: Source): Eval { - return yield* toBoundary(this, yield* this.root.run(source.program)) + this.machine.start(source) } /** A new chain over the global scope, where the program's own scope is pushed. */ @@ -265,47 +101,29 @@ export class Interpreter { return new ScopeStack([this.globalScope]) } - /** Starts a program call the runner reached outside the machine as its own activation. */ - enter(invoke: Invoke): Call { - if (this.machine === undefined) throw new TypeError("Invoke reached the runner without the machine evaluator.") - return this.machine.enter(invoke) - } - // The call returns its promise at once. Arguments cross on the caller's stack, since `toJSON` is program code, and a - // promise argument is awaited first; the host call starts once they have crossed, so admission and hooks happen at - // the call site, and runs concurrently with the program. + // promise argument is awaited first; the host call is queued once they have crossed, and the driver starts it, so + // admission and hooks happen before any other job runs, and it runs concurrently with the program. *toolCall(path: ReadonlyArray, args: Array, site: AstNode | undefined): Eval { const ctx = this - const runner = ctx.runner - const promise = runner.newPromise() - const resolving = runner.resolving(promise) + const machine = ctx.machine + const resolving = machine.resolving(machine.newPromise()) const body = function* (): Eval { const json: Array = [] for (const arg of args) json.push(yield* toBoundary(ctx, arg)) - runner.host(ctx.tools.execute(path, json), { - label: path.join("."), - args: json, - result: "json", - resolving, - site, - }) + machine.host({ kind: "tool", path, args: json }, { label: path.join("."), result: "json", resolving, site }) } - return (yield new Call( - body(), - new AsyncBoundary((completion) => { - if (!completion.ok) resolving.reject(completion.error) - }, promise), - )) as Value + return (yield new Call(body(), new AsyncBoundary(resolving))) as Value } - /** `yield*` in a generator body, as interpreter work; the machine evaluator runs it as a native frame for now. */ + /** `yield*` in a generator body, as interpreter work; the machine runs it as a native frame for now. */ delegateYield(value: Value, node: AstNode, asynchronous: boolean): Eval { - return this.root.delegateYield(value, node, asynchronous) + return this.bridge.delegateYield(value, node, asynchronous) } /** CreateAsyncFromSyncIterator, for `for await` over a synchronous iterable. */ asyncFromSync(builtin: Cursor | undefined, custom: CustomIterator | undefined, node: AstNode | undefined) { - return this.root.asyncFromSync(builtin, custom, node) + return this.bridge.asyncFromSync(builtin, custom, node) } /** The node a `NodeId` names, in one of this execution's sources. */ @@ -315,1744 +133,26 @@ export class Interpreter { /** Calls into the program; a built-in passes its own call site so failures inside are located there. */ call(callable: Value, thisValue: Value, args: Array, site?: AstNode): Eval { - return this.root.call(callable, thisValue, args, site) + return this.bridge.call(callable, thisValue, args, site) } /** Await: suspends the innermost async body until `value` settles. */ - *awaitValue(value: Value, node?: AstNode): Eval { - return (yield new Await(value, node)) as Value + awaitValue(value: Value, node?: AstNode): Eval { + return this.bridge.awaitValue(value, node) } iterate(value: Value) { - return this.root.iterate(value) + return this.bridge.iterate(value) } iterateDirect(value: Value) { - return this.root.iterateDirect(value) - } - - /** Runs one host call to completion before any other job: arguments cross as JSON, the result comes back as values. */ - *tool(run: ToolRuntime["search"], args: Array) { - const json: Array = [] - for (const arg of args) json.push(yield* toBoundary(this, arg)) - return fromJson(this, (yield this.runner.blocking(run(json))) as Json | undefined) + return this.bridge.iterateDirect(value) } /** The built-in search: its result is usable directly, and can also be awaited or chained like a Promise. */ *search(args: Array) { - return promiseLike(this, yield* this.tool(this.tools.search, args)) + const json: Array = [] + for (const arg of args) json.push(yield* toBoundary(this, arg)) + return promiseLike(this, fromJson(this, (yield new Host({ kind: "search", args: json })) as Json | undefined)) } } - -/** One activation: the top-level program or a single function call, evaluating against its own scope chain. */ -class Frame { - /** The body of a generator function, where `yield` suspends. */ - private generator = false - private generatorAsync = false - - constructor( - private readonly ctx: Interpreter, - private scopes: ScopeStack, - ) {} - - *run(program: Program): Eval { - // Keep top-level declarations separate so they can shadow builtins. - this.scopes.push() - try { - // Imports load before any code runs, as in a module. - for (const statement of program.body) - if (statement.type === "ImportDeclaration") bindImport(this.ctx, this.scopes, statement) - predeclareLexical(this.scopes, program.body) - hoistFunctions(this.ctx.builtins, this.scopes, program.body) - hoistVars(this.scopes, program.body) - let value: Value = undefined - for (const [index, statement] of program.body.entries()) { - if (statement.type === "ImportDeclaration") continue - if (index === program.body.length - 1 && statement.type === "ExpressionStatement") { - value = yield* this.evaluateExpression(statement.expression) - break - } - const result = yield* this.evaluateStatement(statement) - - if (result.kind === "return") { - value = result.value - break - } - - if (result.kind === "break" || result.kind === "continue") { - throw typeError(`Unexpected '${result.kind}' outside of a loop.`, statement) - } - } - - // The implicit async body adopts returned promises before copy-out. - return isThenable(value) ? yield* this.awaitValue(value) : value - } finally { - this.scopes.pop() - } - } - - awaitValue(value: Value, node?: AstNode): Eval { - return this.ctx.awaitValue(value, node) - } - - private *evaluateStatement(node: Statement | ModuleDeclaration): Eval { - const result = this.execute(node) - return isEval(result) ? yield* result : result - } - - // Like `evaluate`: statements that cannot run program code complete without a generator. - private execute(node: Statement | ModuleDeclaration): StatementResult | Eval { - switch (node.type) { - case "ExpressionStatement": { - const result = this.evaluate(node.expression) - return isEval(result) ? this.finishExpressionStatement(result) : none - } - case "VariableDeclaration": - return this.evaluateVariableStatement(node) - case "ReturnStatement": - return this.evaluateReturnStatement(node) - case "BlockStatement": - return this.evaluateBlock(node) - case "IfStatement": - return this.evaluateIfStatement(node) - case "SwitchStatement": - return this.evaluateSwitchStatement(node) - case "LabeledStatement": - return this.evaluateLabeledStatement(node) - case "WhileStatement": - return this.evaluateWhileStatement(node) - case "DoWhileStatement": - return this.evaluateDoWhileStatement(node) - case "ForStatement": - return this.evaluateForStatement(node) - case "ForOfStatement": - return this.evaluateForOfStatement(node) - case "ForInStatement": - return this.evaluateForInStatement(node) - case "BreakStatement": - return this.evaluateBreakStatement(node) - case "ContinueStatement": - return this.evaluateContinueStatement(node) - case "ThrowStatement": - return this.evaluateThrowStatement(node) - case "TryStatement": - return this.evaluateTryStatement(node) - case "EmptyStatement": - return none - case "FunctionDeclaration": - return none - case "ImportDeclaration": - throw syntaxError("An import declaration may only appear at the top level of the program.", node) - default: - throw unsupportedSyntax(node.type, node) - } - } - - private *finishExpressionStatement(work: Eval): Eval { - yield* work - return none - } - - private *evaluateVariableStatement(node: VariableDeclaration): Eval { - yield* this.evaluateVariableDeclaration(node) - return none - } - - private *evaluateReturnStatement(node: ReturnStatement): Eval { - const argumentNode = node.argument - if (!argumentNode) return { kind: "return", value: undefined } - const evaluated = this.evaluate(argumentNode) - const value = isEval(evaluated) ? yield* evaluated : evaluated - // An async generator awaits what it returns, as it awaits what it yields. - return { kind: "return", value: this.generatorAsync ? yield* this.awaitValue(value, node) : value } - } - - private *evaluateBlock(node: BlockStatement): Eval { - const body = node.body - const scoped = declaresLexically(body) - if (scoped) { - this.scopes.push() - predeclareLexical(this.scopes, body) - hoistFunctions(this.ctx.builtins, this.scopes, body) - } - try { - for (const statement of body) { - const executed = this.execute(statement) - const result = isEval(executed) ? yield* executed : executed - if (result.kind !== "none") return result - } - - return none - } finally { - if (scoped) this.scopes.pop() - } - } - - // NamedEvaluation: an anonymous function definition takes the name of what it is assigned to. - private *evaluateNamed(node: Expression, name: string): Eval { - if (isAnonymousFunction(node)) { - return createFunction(this.ctx.builtins, this.scopes, node, name) - } - return yield* this.evaluateExpression(node) - } - - private *evaluateIfStatement(node: IfStatement): Eval { - const test = this.evaluate(node.test) - const branch = (isEval(test) ? yield* test : test) ? node.consequent : node.alternate - if (!branch) return none - const result = this.execute(branch) - return isEval(result) ? yield* result : result - } - - private *evaluateSwitchStatement(node: SwitchStatement): Eval { - const discriminant = yield* this.evaluateExpression(node.discriminant) - this.scopes.push() - try { - const cases = node.cases - const statements = cases.flatMap((branch) => branch.consequent) - predeclareLexical(this.scopes, statements) - hoistFunctions(this.ctx.builtins, this.scopes, statements) - let defaultIndex: number | undefined - let selected: number | undefined - for (const [index, branch] of cases.entries()) { - const test = branch.test - if (!test) { - defaultIndex = index - continue - } - if ((yield* this.evaluateExpression(test)) === discriminant) { - selected = index - break - } - } - const start = selected ?? defaultIndex - if (start === undefined) return none - for (let index = start; index < cases.length; index += 1) { - for (const statement of cases[index]!.consequent) { - const result = yield* this.evaluateStatement(statement) - if (result.kind === "break") { - if (result.label === undefined) return none - return result - } - if (result.kind === "return" || result.kind === "continue") return result - } - } - return none - } finally { - this.scopes.pop() - } - } - - private *evaluateWhileStatement(node: WhileStatement, labels?: ReadonlySet): Eval { - const runner = this.ctx.runner - while (true) { - const test = this.evaluate(node.test) - if (!(isEval(test) ? yield* test : test)) break - const executed = this.execute(node.body) - const exit = loopExit(isEval(executed) ? yield* executed : executed, labels) - if (exit !== undefined) return exit - if (runner.tick()) yield Slice - } - - return none - } - - private *evaluateDoWhileStatement(node: DoWhileStatement, labels?: ReadonlySet): Eval { - const runner = this.ctx.runner - do { - const exit = loopExit(yield* this.evaluateStatement(node.body), labels) - if (exit !== undefined) return exit - if (runner.tick()) yield Slice - } while (yield* this.evaluateExpression(node.test)) - - return none - } - - private *evaluateForStatement(node: ForStatement, labels?: ReadonlySet): Eval { - const runner = this.ctx.runner - this.scopes.push() - try { - const initNode = node.init - const testNode = node.test - const updateNode = node.update - - if (initNode?.type === "VariableDeclaration" && initNode.kind !== "var") { - predeclareLexical(this.scopes, [initNode]) - } - - if (initNode) { - if (initNode.type === "VariableDeclaration") { - yield* this.evaluateVariableDeclaration(initNode) - } else { - yield* this.evaluateExpression(initNode) - } - } - - // Each iteration gets fresh copies of the loop's `let` bindings, but only closures can tell them apart. - const perIterationBindings = - initNode?.type === "VariableDeclaration" && - initNode.kind !== "var" && - [node.init, node.test, node.update, node.body].some((part) => part && createsClosure(part)) - ? Array.from(this.scopes.current().keys()) - : [] - - const nextIteration = () => { - if (perIterationBindings.length === 0) return - const current = this.scopes.current() - this.scopes.pop() - this.scopes.push( - new Map(perIterationBindings.map((name): [string, Binding] => [name, { ...current.get(name)! }])), - ) - } - nextIteration() - - while (true) { - if (testNode) { - const test = this.evaluate(testNode) - if (!(isEval(test) ? yield* test : test)) break - } - const executed = this.execute(node.body) - const exit = loopExit(isEval(executed) ? yield* executed : executed, labels) - if (exit !== undefined) return exit - - nextIteration() - if (updateNode) { - const update = this.evaluate(updateNode) - if (isEval(update)) yield* update - } - if (runner.tick()) yield Slice - } - - return none - } finally { - this.scopes.pop() - } - } - - private *evaluateForOfStatement(node: ForOfStatement, labels?: ReadonlySet): Eval { - const runner = this.ctx.runner - const awaiting = node.await - const left = node.left - const declared = loopDeclaration(left, "for...of") - if (declared?.lexical) this.scopes.push() - - try { - if (declared?.lexical) predeclarePattern(this.scopes, declared.pattern, declared.mutable, left) - const right = yield* this.evaluateExpression(node.right) - - const builtin = builtinCursor(this.ctx.builtins, right) - const custom = builtin === undefined ? yield* this.customIterator(right, node, awaiting) : undefined - if (custom === undefined && builtin === undefined) { - throw invalidData( - `${awaiting ? "for await...of" : "for...of"} requires an iterable value, received ${describeValue(right)}.`, - node, - ) - } - // `for await` over a synchronous iterator drives it through CreateAsyncFromSyncIterator, as JS does. - const iterator = awaiting && !custom?.asynchronous ? this.asyncFromSync(builtin, custom, node) : custom - const cursor = iterator === undefined ? builtin : undefined - const self = this - const close = function* (): Eval { - if (iterator) return yield* self.closeIterator(iterator, node) - yield* cursor!.close() - } - - if (left.type === "RestElement" || left.type === "AssignmentPattern") { - throw typeError("Unsupported for...of binding.", left) - } - const assignment = left.type === "VariableDeclaration" ? undefined : left - // Each iteration gets a fresh binding, but only a closure in the body can tell; without one, a single - // `for (const x of ...)` binding is reused. - const reused = - declared?.lexical && declared.pattern.type === "Identifier" && !createsClosure(node.body) - ? { mutable: declared.mutable, value: undefined as Value, initialized: false } - : undefined - if (reused !== undefined) this.scopes.push(new Map([[(declared!.pattern as Identifier).name, reused]])) - - try { - while (true) { - const step: Step = iterator ? yield* this.nextIteratorResult(iterator, node) : yield* cursor!.next() - if (step.done) return none - let result: StatementResult - try { - if (reused !== undefined) { - reused.value = step.value - reused.initialized = true - const executed = this.execute(node.body) - result = isEval(executed) ? yield* executed : executed - } else if (declared?.lexical) { - this.scopes.push() - try { - predeclarePattern(this.scopes, declared.pattern, declared.mutable, left) - yield* this.declarePattern(declared.pattern, step.value, declared.mutable, left, true) - result = yield* this.evaluateStatement(node.body) - } finally { - this.scopes.pop() - } - } else { - if (declared) yield* this.assignPattern(declared.pattern, step.value, left) - else if (assignment) yield* this.assignPattern(assignment, step.value, left) - result = yield* this.evaluateStatement(node.body) - } - } catch (error) { - return yield* preserveConsumerError(close, thrower(error)) - } - const exit = loopExit(result, labels) - if (exit !== undefined) { - yield* close() - return exit - } - if (runner.tick()) yield Slice - } - } finally { - if (reused !== undefined) this.scopes.pop() - } - } finally { - if (declared?.lexical) this.scopes.pop() - } - } - - // CreateAsyncFromSyncIterator: an async iterator over a synchronous one. Each `next()` promise settles a job after - // the value does, and a rejected value closes the synchronous iterator first; `return()` calls the synchronous - // iterator's own `return`, or settles at once when it has none. - asyncFromSync( - builtin: Cursor | undefined, - custom: CustomIterator | undefined, - node: AstNode | undefined, - ): CustomIterator { - const ctx = this.ctx - const runner = ctx.runner - const builtins = ctx.builtins - const result = (value: Value, done: boolean) => iteratorResult(builtins.Object, value, done) - const self = this - // IteratorNext passes the value `next` received, when it received one. - const syncNext = function* (args: Array): Eval { - if (!custom) return yield* builtin!.next() - const object = requireIteratorObject( - yield* self.call(custom.next, custom.iterator, args, node), - "Iterator next() result", - node, - ) - return { done: Boolean(get(object, "done")), value: get(object, "value") } - } - // The synchronous iterator's `return`, looked up when it is needed; built-in collection iterators have none. - const syncReturn = (): Value => (custom ? get(custom.iterator, "return") : undefined) - const settle = (step: Step, closeOnRejection: boolean, made: ReturnType) => { - const rejected = (reason: unknown) => { - if (step.done || !closeOnRejection || reason instanceof Interruption) return made.reject(reason) - const body = function* (): Eval { - try { - if (custom) yield* self.closeIterator(custom, node) - else yield* builtin!.close() - } catch { - // The rejection wins over a failing close. - } - made.reject(reason) - } - return { task: new Task([{ body: body(), boundary: undefined }]), input: normal(undefined) } - } - let wrapper: PromiseObj - try { - wrapper = runner.promiseResolve(step.value) - } catch (error) { - const start = rejected(error) - if (start !== undefined) runner.enqueue({ kind: "resume", task: start.task, input: start.input }) - return - } - runner.then(wrapper, { - kind: "callback", - run: (settled) => - settled.ok ? made.resolve(result(settled.value as Value, step.done)) : rejected(settled.error), - }) - } - const next = asyncFromSyncMethod(builtins, "next", function* (_, args) { - const made = capability(ctx, node) - try { - settle(yield* syncNext(args), true, made) - } catch (error) { - made.reject(error) - } - return made.promise - }) - const close = asyncFromSyncMethod(builtins, "return", function* (_, args) { - const made = capability(ctx, node) - try { - const method = syncReturn() - if (method === undefined || method === null) { - if (builtin !== undefined) yield* builtin.close() - made.resolve(result(args[0], true)) - return made.promise - } - const returned = requireIteratorObject( - yield* self.call(requireIteratorMethod(method, "Iterator return", node), custom!.iterator, args, node), - "Iterator return() result", - node, - ) - settle({ done: Boolean(get(returned, "done")), value: get(returned, "value") }, false, made) - } catch (error) { - made.reject(error) - } - return made.promise - }) - const raise = asyncFromSyncMethod(builtins, "throw", function* (_, args) { - const made = capability(ctx, node) - try { - const method = custom ? get(custom.iterator, "throw") : undefined - if (method === undefined || method === null) { - if (custom) yield* self.closeIterator(custom, node) - else yield* builtin!.close() - made.reject(typeError("The delegated iterator does not provide a throw() method.", node)) - return made.promise - } - const thrown = requireIteratorObject( - yield* self.call(requireIteratorMethod(method, "Iterator throw", node), custom!.iterator, args, node), - "Iterator throw() result", - node, - ) - settle({ done: Boolean(get(thrown, "done")), value: get(thrown, "value") }, true, made) - } catch (error) { - made.reject(error) - } - return made.promise - }) - const adapter = new Obj(builtins.Object) - define(adapter, "next", next, hidden) - define(adapter, "return", close, hidden) - define(adapter, "throw", raise, hidden) - return { iterator: adapter, next, asynchronous: true } - } - - *iterate(value: Value, node?: AstNode): Eval { - const cursor = builtinCursor(this.ctx.builtins, value) - if (cursor !== undefined) return cursor - const iterator = yield* this.customIterator(value, node, false) - return iterator === undefined ? undefined : this.customCursor(iterator, node) - } - - /** GetIteratorDirect: drive an iterator by its own `next`, without asking for `[Symbol.iterator]`. */ - iterateDirect(value: Value, node?: AstNode): Cursor { - if (value instanceof IteratorObj) return value.cursor - if (!(value instanceof Obj)) { - throw typeError(`An iterator must be an object, received ${describeValue(value)}.`, node) - } - return this.customCursor( - { - iterator: value, - next: requireIteratorMethod(get(value, "next"), "Iterator next", node), - asynchronous: false, - }, - node, - ) - } - - private customCursor(iterator: CustomIterator, node: AstNode | undefined): Cursor { - return { - next: () => this.nextIteratorResult(iterator, node), - close: () => this.closeIterator(iterator, node), - } - } - - private *customIterator( - value: Value, - node: AstNode | undefined, - allowAsync = true, - ): Eval { - if (!(value instanceof Obj)) return undefined - const asyncMethod = allowAsync ? get(value, AsyncIteratorSymbol) : undefined - const method = asyncMethod ?? get(value, IteratorSymbol) - if (method === undefined || method === null) return undefined - const iterator = yield* this.call(requireIteratorMethod(method, "Iterator method", node), value, [], node) - const object = requireIteratorObject(iterator, "Iterator method result", node) - return { - iterator: object, - next: requireIteratorMethod(get(object, "next"), "Iterator next", node), - asynchronous: asyncMethod !== undefined && asyncMethod !== null, - } - } - - private *nextIteratorResult(iterator: CustomIterator, node: AstNode | undefined): Eval { - const called = yield* this.call(iterator.next, iterator.iterator, [], node) - const object = requireIteratorObject( - iterator.asynchronous ? yield* this.awaitValue(called, node) : called, - "Iterator next() result", - node, - ) - return { done: Boolean(get(object, "done")), value: get(object, "value") } - } - - private *closeIterator(iterator: CustomIterator, node: AstNode | undefined): Eval { - const close = get(iterator.iterator, "return") - if (close === undefined || close === null) return - const method = requireIteratorMethod(close, "Iterator return", node) - const called = yield* this.call(method, iterator.iterator, [], node) - requireIteratorObject( - iterator.asynchronous ? yield* this.awaitValue(called, node) : called, - "Iterator return() result", - node, - ) - } - - private *evaluateForInStatement(node: ForInStatement, labels?: ReadonlySet): Eval { - const runner = this.ctx.runner - const left = node.left - const declared = loopDeclaration(left, "for...in") - if (declared?.lexical) this.scopes.push() - - try { - if (declared?.lexical) predeclarePattern(this.scopes, declared.pattern, declared.mutable, left) - const right = yield* this.evaluateExpression(node.right) - - const keys = enumerableKeys(this.ctx, right, node.right) - - if (left.type === "RestElement" || left.type === "AssignmentPattern") { - throw typeError("Unsupported for...in binding.", left) - } - const assignment = left.type === "VariableDeclaration" ? undefined : left - - for (const key of keys) { - let result: StatementResult - if (declared?.lexical) { - this.scopes.push() - try { - predeclarePattern(this.scopes, declared.pattern, declared.mutable, left) - yield* this.declarePattern(declared.pattern, key, declared.mutable, left, true) - result = yield* this.evaluateStatement(node.body) - } finally { - this.scopes.pop() - } - } else { - if (declared) yield* this.assignPattern(declared.pattern, key, left) - else if (assignment) yield* this.assignPattern(assignment, key, left) - result = yield* this.evaluateStatement(node.body) - } - - const exit = loopExit(result, labels) - if (exit !== undefined) return exit - if (runner.tick()) yield Slice - } - - return none - } finally { - if (declared?.lexical) this.scopes.pop() - } - } - - private evaluateBreakStatement(node: BreakStatement): StatementResult { - return node.label ? { kind: "break", label: node.label.name } : { kind: "break" } - } - - private evaluateContinueStatement(node: ContinueStatement): StatementResult { - return node.label ? { kind: "continue", label: node.label.name } : { kind: "continue" } - } - - private *evaluateLabeledStatement(node: LabeledStatement): Eval { - const labels = new Set() - let body: Statement = node - while (body.type === "LabeledStatement") { - labels.add(body.label.name) - body = body.body - } - - const result = - body.type === "WhileStatement" - ? yield* this.evaluateWhileStatement(body, labels) - : body.type === "DoWhileStatement" - ? yield* this.evaluateDoWhileStatement(body, labels) - : body.type === "ForStatement" - ? yield* this.evaluateForStatement(body, labels) - : body.type === "ForOfStatement" - ? yield* this.evaluateForOfStatement(body, labels) - : body.type === "ForInStatement" - ? yield* this.evaluateForInStatement(body, labels) - : yield* this.evaluateStatement(body) - - return result.kind === "break" && result.label !== undefined && labels.has(result.label) ? none : result - } - - private *evaluateThrowStatement(node: ThrowStatement): Eval { - throw new Throw(yield* this.evaluateExpression(node.argument)) - } - - private *evaluateTryStatement(node: TryStatement): Eval { - const handler = node.handler - const finalizer = node.finalizer - let result: StatementResult = none - let failure: { readonly error: unknown } | undefined - - try { - result = yield* this.evaluateStatement(node.block) - } catch (error) { - if (error instanceof Interruption) throw error - // A generator's return() unwinds through `finally` but is not an exception `catch` sees. - if (error instanceof GeneratorReturn || !handler) { - if (!finalizer) throw error - failure = { error } - } else { - const caught = materialize(this.ctx, error) - const parameter = handler.param - this.scopes.push() - try { - if (parameter) yield* this.declarePattern(parameter, caught, true, handler) - result = yield* this.evaluateStatement(handler.body) - } catch (error) { - if (!finalizer || error instanceof Interruption) throw error - failure = { error } - } finally { - this.scopes.pop() - } - } - } - - if (!finalizer) return result - const final = yield* this.evaluateStatement(finalizer) - if (isAbrupt(final)) return final - if (failure) throw failure.error - return result - } - - private *evaluateVariableDeclaration(node: VariableDeclaration): Eval { - const kind = node.kind - for (const declaration of node.declarations) { - if (declaration.type !== "VariableDeclarator") { - throw typeError("Unsupported variable declaration shape.", declaration) - } - - const init = declaration.init - // `var x` alone is a no-op: the binding was hoisted on function entry. - const id = declaration.id - if (kind === "var") { - if (init) { - const value = - id.type === "Identifier" ? yield* this.evaluateNamed(init, id.name) : yield* this.evaluateExpression(init) - yield* this.assignPattern(id, value, declaration) - } - continue - } - const value = !init - ? undefined - : id.type === "Identifier" - ? yield* this.evaluateNamed(init, id.name) - : yield* this.evaluateExpression(init) - yield* this.declarePattern(declaration.id, value, kind !== "const", declaration, true) - } - } - - private *declarePattern( - pattern: Pattern, - value: Value, - mutable: boolean, - node: AstNode, - initialize = false, - ): Eval { - if (pattern.type === "Identifier") { - const name = pattern.name - if (initialize) this.scopes.initialize(name, value, node) - else this.scopes.declare(name, value, mutable, node) - return - } - - if (pattern.type === "AssignmentPattern") { - const resolved = value === undefined ? yield* this.evaluateDefault(pattern) : value - yield* this.declarePattern(pattern.left, resolved, mutable, node, initialize) - return - } - - if (pattern.type === "ObjectPattern") { - if (value === null || value === undefined) { - throw typeError(`Cannot destructure ${describeValue(value)} as it is ${value}.`, pattern) - } - - const consumed = new Set() - for (const property of pattern.properties) { - if (property.type === "RestElement") { - const rest = new Obj(this.ctx.builtins.Object) - assign(rest, enumerableSource(this.ctx, "Object destructuring", value, pattern), consumed) - yield* this.declarePattern(property.argument, rest, mutable, property, initialize) - continue - } - - const key = yield* this.destructuringPropertyKey(property) - consumed.add(typeof key === "symbol" ? key : String(key)) - yield* this.declarePattern( - property.value, - destructuredProperty(this.ctx.builtins, value, key, property), - mutable, - property, - initialize, - ) - } - return - } - - if (pattern.type === "ArrayPattern") { - return yield* this.destructureArrayPattern(pattern, value, (target, item, context) => - this.declarePattern(target, item, mutable, context, initialize), - ) - } - - throw typeError(`Unsupported binding pattern '${pattern.type}'.`, pattern) - } - - private *assignPattern(pattern: Pattern, value: Value, node: AstNode): Eval { - if (pattern.type === "Identifier") { - this.scopes.set(pattern.name, value, pattern) - return - } - - if (pattern.type === "MemberExpression") { - yield* this.writeMember(pattern, value) - return - } - - if (pattern.type === "AssignmentPattern") { - const resolved = value === undefined ? yield* this.evaluateDefault(pattern) : value - yield* this.assignPattern(pattern.left, resolved, node) - return - } - - if (pattern.type === "ObjectPattern") { - if (value === null || value === undefined) { - throw typeError(`Cannot destructure ${describeValue(value)} as it is ${value}.`, pattern) - } - - const consumed = new Set() - for (const property of pattern.properties) { - if (property.type === "RestElement") { - const rest = new Obj(this.ctx.builtins.Object) - assign(rest, enumerableSource(this.ctx, "Object destructuring", value, pattern), consumed) - yield* this.assignPattern(property.argument, rest, property) - continue - } - const key = yield* this.destructuringPropertyKey(property) - consumed.add(typeof key === "symbol" ? key : String(key)) - yield* this.assignPattern( - property.value, - destructuredProperty(this.ctx.builtins, value, key, property), - property, - ) - } - return - } - - if (pattern.type === "ArrayPattern") { - return yield* this.destructureArrayPattern(pattern, value, (target, item, context) => - this.assignPattern(target, item, context), - ) - } - - throw typeError(`Unsupported assignment pattern '${pattern.type}'.`, node) - } - - private evaluateDefault(pattern: AssignmentPattern): Eval { - return pattern.left.type === "Identifier" - ? this.evaluateNamed(pattern.right, pattern.left.name) - : this.evaluateExpression(pattern.right) - } - - private *destructureArrayPattern( - pattern: ArrayPattern, - value: Value, - consume: (target: Pattern, value: Value, context: AstNode) => Eval, - ): Eval { - const cursor = yield* this.iterate(value, pattern) - if (cursor === undefined) { - throw typeError("Array destructuring requires a supported iterable value.", pattern) - } - let done = false - for (const element of pattern.elements) { - if (done) { - if (element === null) continue - yield* consume( - element.type === "RestElement" ? element.argument : element, - element.type === "RestElement" ? new Arr(this.ctx.builtins.Array) : undefined, - element, - ) - if (element.type === "RestElement") return - continue - } - const step = yield* cursor.next() - done = step.done - if (element === null) continue - if (element.type === "RestElement") { - const rest: Array = [] - if (!step.done) rest.push(step.value) - while (!done) { - const next = yield* cursor.next() - done = next.done - if (!done) rest.push(next.value) - } - yield* consume(element.argument, new Arr(this.ctx.builtins.Array, rest), element) - return - } - const consumed = consume(element, step.done ? undefined : step.value, pattern) - yield* step.done ? consumed : preserveConsumerError(cursor.close, consumed) - } - if (!done) yield* cursor.close() - } - - private *destructuringPropertyKey(property: Property | AssignmentProperty): Eval { - if (property.type !== "Property" || property.kind !== "init") { - throw typeError("Unsupported object destructuring property.", property) - } - const keyNode = property.key - if (property.computed) return yield* this.toPropertyKey(yield* this.evaluateExpression(keyNode), keyNode) - if (keyNode.type === "Identifier") return keyNode.name - if (keyNode.type === "Literal") return String(keyNode.value) - throw unsupportedSyntax(keyNode.type, keyNode) - } - - private *evaluateExpression(node: Expression): Eval { - const result = this.evaluate(node) - return isEval(result) ? yield* result : result - } - - // Evaluates leaves directly and returns the work for anything that may run program code, so the common cases cost - // no generator; callers finish the work with `yield*`. - private evaluate(node: Expression): Value | Eval { - switch (node.type) { - case "Literal": { - const regex = node.regex - if (regex) return constructRegExp(this.ctx.builtins, [regex.pattern, regex.flags]) - return literal(node) - } - case "Identifier": - return this.scopes.get(node.name, node) - case "ThisExpression": - return this.scopes.get("this", node) - case "BinaryExpression": { - const operator = node.operator - if (node.left.type === "PrivateIdentifier") throw unsupportedSyntax(node.left.type, node.left) - const left = this.evaluate(node.left) - if (isEval(left)) return this.evaluateBinaryExpression(node, left) - const right = this.evaluate(node.right) - if (isEval(right)) return this.evaluateBinaryExpression(node, left, right) - if (operator === "instanceof") return instanceofValue(left, right, node) - if (left instanceof Obj || right instanceof Obj) return this.applyOperator(operator, left, right, node) - return applyBinaryOperator(operator, left, right, node) - } - case "LogicalExpression": - return this.evaluateLogicalExpression(node) - case "UnaryExpression": - return this.evaluateUnaryExpression(node) - case "AssignmentExpression": { - const left = node.left - if (left.type !== "Identifier") return this.evaluateAssignmentExpression(node) - if (node.operator === "=") { - const right = - node.right.type === "ArrowFunctionExpression" || - (node.right.type === "FunctionExpression" && !node.right.id) - ? createFunction(this.ctx.builtins, this.scopes, node.right, left.name) - : this.evaluate(node.right) - return isEval(right) ? this.assignAfter(left, right) : this.scopes.set(left.name, right, left) - } - if (!compoundOperators.has(node.operator)) return this.evaluateAssignmentExpression(node) - const current = this.scopes.get(left.name, left) - const right = this.evaluate(node.right) - if (isEval(right) || current instanceof Obj || right instanceof Obj) - return this.compoundAfter(node, current, right) - return this.scopes.set(left.name, applyBinaryOperator(node.operator.slice(0, -1), current, right, node), left) - } - case "SequenceExpression": - return this.evaluateSequenceExpression(node) - case "CallExpression": - return this.evaluateCallExpression(node) - case "ArrowFunctionExpression": - case "FunctionExpression": - return createFunction(this.ctx.builtins, this.scopes, node) - case "MemberExpression": { - const objectNode = node.object - const propertyNode = node.property - if (objectNode.type === "Super") throw unsupportedSyntax(objectNode.type, objectNode) - if (propertyNode.type === "PrivateIdentifier") throw unsupportedSyntax(propertyNode.type, propertyNode) - const object = this.evaluate(objectNode) - if (isEval(object)) return this.readMember(node, object) - if (object === OptionalShortCircuit) return OptionalShortCircuit - if ((object === null || object === undefined) && node.optional) return OptionalShortCircuit - const key = - !node.computed && propertyNode.type === "Identifier" ? propertyNode.name : this.evaluate(propertyNode) - if (isEval(key)) return this.readMember(node, object, key) - const reference = this.memberReference(node, object, key) - return isEval(reference) ? this.readMember(node, object, key) : this.readResolved(node, reference) - } - case "ChainExpression": - return this.evaluateChainExpression(node) - case "ObjectExpression": - return this.evaluateObjectExpression(node) - case "ArrayExpression": - return this.evaluateArrayExpression(node) - case "TemplateLiteral": - return this.evaluateTemplateLiteral(node) - case "TaggedTemplateExpression": - return this.evaluateTaggedTemplate(node) - case "ConditionalExpression": - return this.evaluateConditionalExpression(node) - case "UpdateExpression": { - const argument = node.argument - if (argument.type !== "Identifier") return this.evaluateUpdateExpression(node) - const current = this.scopes.get(argument.name, argument) - if (current instanceof Obj) return this.evaluateUpdateExpression(node) - return this.update(node, argument, current) - } - case "AwaitExpression": - return this.evaluateAwaitExpression(node) - case "YieldExpression": - return this.evaluateYieldExpression(node) - case "NewExpression": - return this.evaluateNewExpression(node) - case "ImportExpression": - return this.evaluateImportExpression(node) - default: - throw unsupportedSyntax(node.type, node) - } - } - - private *evaluateSequenceExpression(node: SequenceExpression): Eval { - let result: Value - for (const expression of node.expressions) { - const value = this.evaluate(expression) - result = isEval(value) ? yield* value : value - } - return result - } - - private *evaluateChainExpression(node: ChainExpression): Eval { - const value = yield* this.evaluateExpression(node.expression) - return value === OptionalShortCircuit ? undefined : value - } - - private *evaluateConditionalExpression(node: ConditionalExpression): Eval { - const test = this.evaluate(node.test) - const branch = this.evaluate((isEval(test) ? yield* test : test) ? node.consequent : node.alternate) - return isEval(branch) ? yield* branch : branch - } - - private *evaluateAwaitExpression(node: AwaitExpression): Eval { - // Await always suspends, including for plain values. - return yield* this.awaitValue(yield* this.evaluateExpression(node.argument), node) - } - - private *evaluateNewExpression(node: NewExpression): Eval { - const callee = yield* this.evaluateExpression(node.callee) - const construct = callee instanceof Builtin ? callee.construct : undefined - if (construct === undefined) throw notConstructible(callee, node) - const args = yield* this.evaluateCallArguments(node.arguments) - const result = this.native(() => construct(args, callee as Builtin, node), node) - return isEval(result) ? yield* result : result - } - - // Finishes a binary expression whose left operand (and maybe right) `evaluate` already started. - private *evaluateBinaryExpression( - node: BinaryExpression, - leftValue: Value | Eval, - started: Value | Eval | typeof unevaluated = unevaluated, - ): Eval { - const operator = node.operator - const lhs = isEval(leftValue) ? yield* leftValue : leftValue - const rightValue = started === unevaluated ? this.evaluate(node.right) : started - const rhs = isEval(rightValue) ? yield* rightValue : rightValue - if (operator === "instanceof") return instanceofValue(lhs, rhs, node) - if (lhs instanceof Obj || rhs instanceof Obj) return yield* this.applyOperator(operator, lhs, rhs, node) - return applyBinaryOperator(operator, lhs, rhs, node) - } - - /** ToPrimitive for an operand: data objects run their own methods; opaque values stay for the data gates below. */ - private *toPrimitive(value: Value, hint: Hint, node: AstNode): Eval { - const result = this.native(() => toPrimitive(this.ctx, value, hint), node) - return isEval(result) ? yield* result : result - } - - // Arithmetic, relational, and bitwise operators convert both operands first, left then right, so a `valueOf` - // runs (and throws) in spec order; `+` asks for the default hint and the rest for a number. - private *applyOperator(operator: string, lhs: Value, rhs: Value, node: AstNode): Eval { - if (!(lhs instanceof Obj || rhs instanceof Obj)) return applyBinaryOperator(operator, lhs, rhs, node) - const conversion = operatorConversion(operator, lhs, rhs) - if (conversion === undefined) return applyBinaryOperator(operator, lhs, rhs, node) - if (conversion.kind === "key") return has(rhs as Obj, yield* this.toPropertyKey(lhs, node)) - const l = yield* this.toPrimitive(lhs, conversion.hint, node) - const r = yield* this.toPrimitive(rhs, conversion.hint, node) - return applyBinaryOperator(operator, l, r, node) - } - - private *evaluateLogicalExpression(node: LogicalExpression): Eval { - const operator = node.operator - const leftValue = this.evaluate(node.left) - const left = isEval(leftValue) ? yield* leftValue : leftValue - if (operator === "&&") return left ? yield* this.evaluateExpression(node.right) : left - if (operator === "||") return left ? left : yield* this.evaluateExpression(node.right) - if (operator === "??") - return left !== null && left !== undefined ? left : yield* this.evaluateExpression(node.right) - throw typeError(`Unsupported logical operator '${operator}'.`, node) - } - - private *evaluateUnaryExpression(node: UnaryExpression): Eval { - const operator = node.operator - const argument = node.argument - if (operator === "delete") return yield* this.evaluateDeleteExpression(argument) - // Undeclared names short-circuit, but declared TDZ bindings must still throw. - if (operator === "typeof" && argument.type === "Identifier" && !this.scopes.resolve(argument.name)) { - return "undefined" - } - const value = yield* this.evaluateExpression(argument) - if (operator === "typeof") return typeofValue(value) - if (operator === "!") return !value - if (operator === "void") return undefined - const operand = yield* this.toPrimitive(value, "number", node) - if (isOpaque(operand)) { - throw invalidData("Unary operators require data values.", node) - } - switch (operator) { - case "+": - return +(operand as number) - case "-": - return -(operand as number) - case "~": - return ~(operand as number) - default: - throw typeError(`Unsupported unary operator '${operator}'.`, node) - } - } - - private *evaluateAssignmentExpression(node: AssignmentExpression): Eval { - const left = node.left - const operator = node.operator - // The binary operator a compound assignment applies: `+=` is `+`. - const binary = operator.slice(0, -1) - if (operator === "??=" || operator === "||=" || operator === "&&=") { - return yield* this.evaluateLogicalAssignment(node, left, operator) - } - if (operator !== "=" && !compoundOperators.has(operator)) { - throw typeError(`Unsupported assignment operator '${operator}'.`, node) - } - if (operator === "=" && (left.type === "ObjectPattern" || left.type === "ArrayPattern")) { - const rightValue = yield* this.evaluateExpression(node.right) - yield* this.assignPattern(left, rightValue, node) - return rightValue - } - if (left.type === "Identifier") { - const name = left.name - if (operator !== "=") { - const current = this.scopes.get(name, left) - const evaluated = this.evaluate(node.right) - const rightValue = isEval(evaluated) ? yield* evaluated : evaluated - const next = - current instanceof Obj || rightValue instanceof Obj - ? yield* this.applyOperator(binary, current, rightValue, node) - : applyBinaryOperator(binary, current, rightValue, node) - return this.scopes.set(name, next, left) - } - const rightValue = yield* this.evaluateNamed(node.right, name) - return this.scopes.set(name, rightValue, left) - } - if (left.type === "MemberExpression") { - const self = this - return yield* this.modifyMember(left, function* (current) { - const rightValue = yield* self.evaluateExpression(node.right) - if (operator === "=") return { write: true, next: rightValue, result: rightValue } - const next = yield* self.applyOperator(binary, current, rightValue, node) - return { write: true, next, result: next } - }) - } - throw typeError("Assignment target must be an Identifier or MemberExpression.", left) - } - - private *assignAfter(left: Identifier, right: Eval): Eval { - return this.scopes.set(left.name, yield* right, left) - } - - private *compoundAfter(node: AssignmentExpression, current: Value, started: Value | Eval): Eval { - const left = node.left as Identifier - const binary = node.operator.slice(0, -1) - const right = isEval(started) ? yield* started : started - const next = - current instanceof Obj || right instanceof Obj - ? yield* this.applyOperator(binary, current, right, node) - : applyBinaryOperator(binary, current, right, node) - return this.scopes.set(left.name, next, left) - } - - // `++`/`--` on a name whose value is already primitive. - private update(node: UpdateExpression, argument: Identifier, current: Value): Value { - if (isOpaque(current)) throw invalidData(`'${node.operator}' requires a data value.`, argument) - const before = coerceToNumber(current) - const next = node.operator === "++" ? before + 1 : before - 1 - this.scopes.set(argument.name, next, argument) - return node.prefix ? next : before - } - - private *evaluateLogicalAssignment(node: AssignmentExpression, left: Pattern, operator: string): Eval { - const shouldAssign = (current: Value): boolean => - operator === "??=" ? current === null || current === undefined : operator === "||=" ? !current : Boolean(current) - if (left.type === "Identifier") { - const name = left.name - const current = this.scopes.get(name, left) - if (!shouldAssign(current)) return current - const rightValue = yield* this.evaluateNamed(node.right, name) - return this.scopes.set(name, rightValue, left) - } - if (left.type === "MemberExpression") { - const self = this - return yield* this.modifyMember(left, function* (current) { - if (!shouldAssign(current)) return { write: false, next: current, result: current } - const rightValue = yield* self.evaluateExpression(node.right) - return { write: true, next: rightValue, result: rightValue } - }) - } - throw typeError("Assignment target must be an Identifier or MemberExpression.", left) - } - - private *evaluateUpdateExpression(node: UpdateExpression): Eval { - const operator = node.operator - const argument = node.argument - const prefix = node.prefix - - const increment = operator === "++" ? 1 : operator === "--" ? -1 : undefined - - if (increment === undefined) { - throw typeError(`Unsupported update operator '${operator}'.`, node) - } - - // CodeMode numeric coercion, not host Number(), so opaque runtime references reject clearly. - const operand = (current: Value): number => { - if (isOpaque(current)) { - throw invalidData(`'${operator}' requires a data value.`, argument) - } - return coerceToNumber(current) - } - - if (argument.type === "Identifier") { - const name = argument.name - const current = this.scopes.get(name, argument) - const before = operand(current instanceof Obj ? yield* this.toPrimitive(current, "number", argument) : current) - const next = before + increment - this.scopes.set(name, next, argument) - return prefix ? next : before - } - - if (argument.type === "MemberExpression") { - const self = this - return yield* this.modifyMember(argument, function* (current) { - const value = operand(yield* self.toPrimitive(current, "number", argument)) - const next = value + increment - return { write: true, next, result: prefix ? next : value } - }) - } - - throw typeError("Update target must be an Identifier or MemberExpression.", argument) - } - - // A module that fails to load rejects the promise, as a failed dynamic import does. - private *evaluateImportExpression(node: ImportExpression): Eval { - const specifier = coerceToString(yield* this.evaluateExpression(node.source)) - if (node.options) yield* this.evaluateExpression(node.options) - const promise = capability(this.ctx, node) - try { - promise.resolve(this.ctx.namespace(specifier, node)) - } catch (error) { - if (!(error instanceof PendingThrow)) throw error - promise.reject(error) - } - return promise.promise - } - - // EvaluateCall: a member callee supplies its base object as `this`; anything else calls with undefined. - private *evaluateCallExpression(node: CallExpression): Eval { - const callee = node.callee - if (callee.type === "Super") throw unsupportedSyntax(callee.type, callee) - const { callable, thisValue } = - callee.type === "MemberExpression" - ? yield* this.readMethod(callee) - : { callable: yield* this.evaluateExpression(callee), thisValue: undefined } - if (callable === OptionalShortCircuit) return OptionalShortCircuit - if ((callable === null || callable === undefined) && node.optional) return OptionalShortCircuit - - const args = yield* this.evaluateCallArguments(node.arguments) - const result = this.dispatch(callable, thisValue, args, node, callee) - if (isEval(result)) return yield* result - return result instanceof Call || result instanceof Invoke ? ((yield result) as Value) : result - } - - private *readMethod(node: MemberExpression): Eval<{ callable: Value; thisValue: Value }> { - const reference = yield* this.getMemberReference(node) - if (reference === OptionalShortCircuit) return { callable: OptionalShortCircuit, thisValue: undefined } - if (reference instanceof ToolReference) return { callable: reference, thisValue: undefined } - if ("value" in reference) return { callable: reference.value, thisValue: undefined } - return { callable: readReference(reference, node), thisValue: reference.receiver } - } - - *call(callable: Value, thisValue: Value, args: Array, node?: AstNode, callee?: Expression): Eval { - const result = this.dispatch(callable, thisValue, args, node, callee) - if (isEval(result)) return yield* result - return result instanceof Call || result instanceof Invoke ? ((yield result) as Value) : result - } - - // The single dispatch for every invocation: call expressions and callbacks share it. A program function is a `Call` - // for the caller to yield; a built-in that returns a value needs no generator. - private dispatch( - callable: Value, - thisValue: Value, - args: Array, - node?: AstNode, - callee?: Expression, - ): Value | Eval | Call | Invoke { - if (callable instanceof Fn) { - if (this.ctx.machine !== undefined) return new Invoke(callable, thisValue, args, node) - return this.invokeFunction(callable, thisValue, args, node) - } - if (callable instanceof Builtin) { - const ctx = this.ctx - // A loop made only of built-in calls has no program call or loop back-edge to count, so count the call. - if (ctx.runner.tick()) - return afterSlice(() => this.native(() => callable.invoke(ctx, thisValue, args, node), node)) - return this.native(() => callable.invoke(ctx, thisValue, args, node), node) - } - if (callable instanceof ToolReference) { - if (callable.path.length === 0) { - throw typeError("The tools root is not callable.", callee ?? node) - } - return this.ctx.toolCall(callable.path, args, node) - } - throw notAFunction(callee, node) - } - - // Built-ins throw without a location, synchronously or inside their work; the call site supplies it. - private native(body: () => NativeResult, node?: AstNode): Value | Eval { - let result: NativeResult - try { - result = body() - } catch (error) { - throw locate(error, node) - } - return isEval(result) ? located(result, node) : result - } - - private *evaluateCallArguments(argNodes: ReadonlyArray): Eval> { - const args: Array = [] - for (const argNode of argNodes) { - if (argNode.type === "SpreadElement") { - const spread = yield* this.evaluateExpression(argNode.argument) - const cursor = yield* this.iterate(spread, argNode) - if (cursor === undefined) throw typeError("Spread arguments require a synchronous iterable.", argNode) - while (true) { - const step = yield* cursor.next() - if (step.done) break - args.push(step.value) - } - } else { - const value = this.evaluate(argNode) - args.push(isEval(value) ? yield* value : value) - } - } - return args - } - - // A program call runs as its own activation on the explicit stack, so it is a `Call` for the caller to yield. An - // async call returns its promise when the body first suspends or ends; a generator call binds its parameters and - // returns the generator without running the body. - invokeFunction(fn: Fn, thisValue: Value, args: Array, node?: AstNode): Call | Value | Eval { - const runner = this.ctx.runner - // The program's own activation is at the bottom of the stack. - if (runner.depth > MAX_CALL_DEPTH) throw rangeError("Maximum call stack size exceeded", node) - const invocation = new Frame(this.ctx, new ScopeStack([...fn.capturedScopes, new Map()])) - const paramScope = invocation.scopes.current() - // `this` and `arguments` are scope bindings so arrows resolve them lexically; a parameter named - // `arguments` shadows the object, as in JS. - if (!fn.arrow) paramScope.set("this", { mutable: false, value: thisValue, initialized: true }) - if (!fn.arrow && usesArguments(fn)) { - paramScope.set("arguments", { - mutable: true, - value: new Arguments(this.ctx.builtins.Object, args), - initialized: true, - }) - } - const simple = isSimple(fn.parameters) - if (simple) { - // A repeated name takes the last argument for it, as in JS. - for (const [index, parameter] of fn.parameters.entries()) { - paramScope.set((parameter as Identifier).name, { mutable: true, value: args[index], initialized: true }) - } - } else { - // Seed all parameters first so defaults cannot fall through to same-named outer bindings. - for (const parameter of fn.parameters) { - for (const name of collectPatternNames(parameter)) { - paramScope.set(name, { mutable: true, value: undefined, initialized: false }) - } - } - } - if (fn.generator) { - invocation.generator = true - invocation.generatorAsync = fn.async - // The prototype is read after the parameters bind, since a default may replace it. - const create = () => { - const proto = get(fn, "prototype") - return new GeneratorObj( - proto instanceof Obj ? proto : fn.async ? this.ctx.builtins.AsyncGenerator : this.ctx.builtins.Generator, - fn.async, - generatorBody(invocation.functionBody(fn, paramScope)), - ) - } - if (simple) return create() - return (function* () { - yield* invocation.bindParameters(fn, args) - return create() - })() - } - const body = simple ? invocation.functionBody(fn, paramScope) : invocation.enter(fn, args, paramScope) - if (!fn.async) return new Call(body) - const promise = runner.newPromise() - const resolving = runner.resolving(promise) - return new Call( - body, - new AsyncBoundary( - (completion) => - completion.ok ? resolving.resolve(completion.value as Value) : resolving.reject(completion.error), - promise, - ), - ) - } - - private *enter(fn: Fn, args: Array, paramScope: Map): Eval { - yield* this.bindParameters(fn, args) - return yield* this.functionBody(fn, paramScope) - } - - private *bindParameters(fn: Fn, args: Array): Eval { - const parameters = fn.parameters - for (const [index, parameter] of parameters.entries()) { - if (parameter.type === "RestElement") { - yield* this.declarePattern( - parameter.argument, - new Arr(this.ctx.builtins.Array, args.slice(index)), - true, - parameter, - true, - ) - break - } - if (parameter.type === "Identifier") { - // A sloppy simple parameter list may repeat a name; the last occurrence wins, as in JS. - if (parameters.findLastIndex((other) => other.type === "Identifier" && other.name === parameter.name) !== index) - continue - this.scopes.initialize(parameter.name, args[index], parameter) - continue - } - yield* this.declarePattern(parameter, args[index], true, parameter, true) - } - } - - private *functionBody(fn: Fn, paramScope: Map): Eval { - if (fn.body.type === "BlockStatement") { - this.scopes.push() - hoistVars(this.scopes, fn.body.body, paramScope) - const result = yield* this.evaluateBlock(fn.body) - return result.kind === "return" ? result.value : undefined - } - const result = this.evaluate(fn.body) - return isEval(result) ? yield* result : result - } - - private *evaluateYieldExpression(node: YieldExpression): Eval { - const argument = node.argument - if (!this.generator) throw typeError("yield is only valid inside a generator.", node) - const value = argument ? yield* this.evaluateExpression(argument) : undefined - if (node.delegate) return yield* this.delegateYield(value, node) - return yield* this.suspendGenerator(this.generatorAsync ? yield* this.awaitValue(value, node) : value) - } - - // Yield `value` and continue with the request that resumes the body: `next` supplies the value of the `yield`, - // `throw` throws there, and `return` unwinds through `finally` blocks (an async generator awaits the value first). - private *suspendGenerator(value: Value, asynchronous = this.generatorAsync): Eval { - const request = (yield new Yield(value)) as GeneratorRequest - if (request.kind === "next") return request.value - if (request.kind === "throw") throw new Throw(request.value) - throw new GeneratorReturn(asynchronous ? yield* this.awaitValue(request.value) : request.value) - } - - *delegateYield(value: Value, node: AstNode, asynchronous = this.generatorAsync): Eval { - const builtin = builtinCursor(this.ctx.builtins, value) - if (builtin !== undefined && !asynchronous) { - while (true) { - const step = yield* builtin.next() - if (step.done) return undefined - try { - yield* this.suspendGenerator(step.value, asynchronous) - } catch (error) { - if (error instanceof GeneratorReturn) { - yield* builtin.close() - throw error - } - if (error instanceof Throw) { - yield* builtin.close() - throw typeError("The delegated iterator does not provide a throw() method.", node) - } - throw error - } - } - } - - const custom = builtin === undefined ? yield* this.customIterator(value, node, asynchronous) : undefined - if (builtin === undefined && !custom) throw typeError("yield* requires a compatible iterable value.", node) - // An async generator delegates to a synchronous iterator through CreateAsyncFromSyncIterator, as JS does. - const iterator = asynchronous && !custom?.asynchronous ? this.asyncFromSync(builtin, custom, node) : custom! - let kind: GeneratorRequest["kind"] = "next" - let input: Value = undefined - while (true) { - const method = kind === "next" ? iterator.next : get(iterator.iterator, kind) - if (method === undefined || method === null) { - if (kind === "return") throw new GeneratorReturn(asynchronous ? yield* this.awaitValue(input, node) : input) - yield* this.closeIterator(iterator, node) - throw typeError("The delegated iterator does not provide a throw() method.", node) - } - const called = yield* this.call( - requireIteratorMethod(method, `Iterator ${kind}`, node), - iterator.iterator, - [input], - node, - ) - const result = requireIteratorObject( - iterator.asynchronous ? yield* this.awaitValue(called, node) : called, - `Iterator ${kind}() result`, - node, - ) - const done = Boolean(get(result, "done")) - const resultValue = get(result, "value") - if (done) { - if (kind === "return") throw new GeneratorReturn(resultValue) - return resultValue - } - - try { - // Delegated values are yielded as they are; an async generator's inner promise already settled them. - input = yield* this.suspendGenerator(resultValue, asynchronous) - kind = "next" - } catch (error) { - if (!(error instanceof GeneratorReturn) && !(error instanceof Throw)) throw error - kind = error instanceof GeneratorReturn ? "return" : "throw" - input = error.value - } - } - } - - private *evaluateObjectExpression(node: ObjectExpression): Eval { - const objectValue = new Obj(this.ctx.builtins.Object) - for (const property of node.properties) { - if (property.type === "SpreadElement") { - const spread = yield* this.evaluateExpression(property.argument) - if (spread === null || spread === undefined) continue - assign(objectValue, enumerableSource(this.ctx, "Object spread", spread, property)) - continue - } - - if (property.kind !== "init") { - throw typeError("Only init object properties are supported.", property) - } - - const keyNode = property.key - - let key: PropertyKey - - if (property.computed) { - key = yield* this.toPropertyKey(yield* this.evaluateExpression(keyNode), keyNode) - } else if (keyNode.type === "Identifier") { - key = keyNode.name - } else if (keyNode.type === "Literal") { - key = propertyKey(literal(keyNode)) - } else { - throw typeError("Unsupported object property key shape.", keyNode) - } - - const name = - key === IteratorSymbol - ? "[Symbol.iterator]" - : key === AsyncIteratorSymbol - ? "[Symbol.asyncIterator]" - : String(key) - define(objectValue, key, yield* this.evaluateNamed(property.value, name)) - } - - return objectValue - } - - private *evaluateArrayExpression(node: ArrayExpression): Eval { - const values: Array = [] - - for (const element of node.elements) { - if (element === null) { - // A literal elision is a real hole, like JS: extend length without an own index. - values.length += 1 - continue - } - if (element.type === "SpreadElement") { - const spread = yield* this.evaluateExpression(element.argument) - const cursor = yield* this.iterate(spread, element) - if (cursor === undefined) throw typeError("Array spread requires a synchronous iterable.", element) - while (true) { - const step = yield* cursor.next() - if (step.done) break - values.push(step.value) - } - } else { - values.push(yield* this.evaluateExpression(element)) - } - } - return new Arr(this.ctx.builtins.Array, values) - } - - private *evaluateTemplateLiteral(node: TemplateLiteral): Eval { - const quasis = node.quasis - const expressions = node.expressions - - let output = "" - - for (let index = 0; index < quasis.length; index += 1) { - const quasi = quasis[index]! - // acorn only omits `cooked` for invalid escapes, which are a parse error outside a tagged template. - if (typeof quasi.value.cooked !== "string") { - throw typeError("Invalid template literal quasi.", quasi) - } - output += quasi.value.cooked - checkStringLength(output.length) - - if (index < expressions.length) { - const raw = yield* this.evaluateExpression(expressions[index]) - output += coerceToString(yield* this.toPrimitive(raw, "string", expressions[index])) - checkStringLength(output.length) - } - } - - return output - } - - // `tag\`a${x}b\`` is `tag(strings, x)`: the tag is read like a callee (a member keeps its receiver), the - // substitutions are evaluated in order, and `strings` carries the escaped source as `strings.raw`. - private *evaluateTaggedTemplate(node: TaggedTemplateExpression): Eval { - const { callable, thisValue } = - node.tag.type === "MemberExpression" - ? yield* this.readMethod(node.tag) - : { callable: yield* this.evaluateExpression(node.tag), thisValue: undefined } - const strings = templateObject(this.ctx, node) - const values: Array = [] - for (const expression of node.quasi.expressions) values.push(yield* this.evaluateExpression(expression)) - return yield* this.call(callable, thisValue, [strings, ...values], node, node.tag) - } - - private *getMemberReference( - node: MemberExpression, - object: Value | Eval | typeof unevaluated = unevaluated, - key: Value | Eval | typeof unevaluated = unevaluated, - ): Eval { - const objectNode = node.object - const propertyNode = node.property - if (objectNode.type === "Super") throw unsupportedSyntax(objectNode.type, objectNode) - if (propertyNode.type === "PrivateIdentifier") throw unsupportedSyntax(propertyNode.type, propertyNode) - const evaluated = object === unevaluated ? this.evaluate(objectNode) : object - const objectValue = isEval(evaluated) ? yield* evaluated : evaluated - if (objectValue === OptionalShortCircuit) return OptionalShortCircuit - if ((objectValue === null || objectValue === undefined) && node.optional) return OptionalShortCircuit - - const computed = - key !== unevaluated - ? key - : !node.computed && propertyNode.type === "Identifier" - ? propertyNode.name - : this.evaluate(propertyNode) - const keyValue = isEval(computed) ? yield* computed : computed - const reference = this.memberReference(node, objectValue, keyValue) - return isEval(reference) ? yield* reference : reference - } - - // The reference once the object and key are evaluated: ToPropertyKey on an object key may run program code. - // GetValue applies ToObject to the base before ToPropertyKey, so a nullish base throws before the key's own - // toString runs. - private memberReference(node: MemberExpression, objectValue: Value, keyValue: Value): Reference | Eval { - requireMemberBase(node, objectValue, keyValue) - if (keyValue instanceof Obj) return this.resolveAfterKey(node, objectValue, keyValue) - return resolveProperty(this.ctx.builtins, objectValue, propertyKey(keyValue), node.object, node.property) - } - - private *resolveAfterKey(node: MemberExpression, objectValue: Value, keyValue: Obj): Eval { - const key = yield* this.toPropertyKey(keyValue, node.property) - return resolveProperty(this.ctx.builtins, objectValue, key, node.object, node.property) - } - - private readResolved(node: MemberExpression, reference: Reference): Value { - if (reference === OptionalShortCircuit) return OptionalShortCircuit - if (reference instanceof ToolReference) return reference - if ("value" in reference) return reference.value - return readReference(reference, node) - } - - private *readMember( - node: MemberExpression, - object: Value | Eval | typeof unevaluated = unevaluated, - key: Value | Eval | typeof unevaluated = unevaluated, - ): Eval { - return this.readResolved(node, yield* this.getMemberReference(node, object, key)) - } - - private writeMember(node: MemberExpression, value: Value): Eval { - return this.modifyMember(node, function* () { - return { write: true, next: value, result: value } - }) - } - - private *evaluateDeleteExpression(argument: Expression): Eval { - const target = argument.type === "ChainExpression" ? argument.expression : argument - if (target.type === "Identifier") throw typeError("Only data fields may be deleted.", argument) - // `delete ` evaluates the operand and is true, as in JS. - if (target.type !== "MemberExpression") { - yield* this.evaluateExpression(target) - return true - } - const reference = yield* this.getMemberReference(target) - if (reference === OptionalShortCircuit) return true - if (reference instanceof ToolReference || "value" in reference || reference.receiver !== reference.target) { - throw invalidData("Only data fields may be deleted.", target) - } - if (remove(reference.target, reference.key)) return true - throw typeError(`Cannot delete property '${String(reference.key)}'.`, target) - } - - // Resolve side-effecting object and key expressions exactly once. - private *modifyMember( - node: MemberExpression, - compute: (current: Value) => Eval<{ write: boolean; next: Value; result: Value }>, - ): Eval { - const reference = writableReference(yield* this.getMemberReference(node), node) - const key = reference.key - const { write, next, result } = yield* compute(readReference(reference, node)) - if (write) assignToReference(reference.target, key, next, node) - return result - } - - // ToPropertyKey: a data object converts through its own `toString`/`valueOf` first; anything else becomes its - // string form synchronously, so `counts[row.category]` works when the field is null. - private *toPropertyKey(value: Value, node: AstNode): Eval { - if (!(value instanceof Obj)) return propertyKey(value) - return propertyKey(yield* this.toPrimitive(value, "string", node)) - } -} - -// The async-from-sync adapter still keeps its iterator in host closures, so its methods are made per iteration. -const asyncFromSyncMethod = (builtins: Builtins, name: string, call: NativeCall) => - native(builtins, { name, length: 1, call, runtime: "async-from-sync" }) - -// Rethrows inside interpreter work, so a failure already caught can pass through `preserveConsumerError`. -function* thrower(error: unknown): Eval { - throw error -} - -// Finishes a built-in's work, locating a failure at the call that started it. -function* located(work: Eval, node: AstNode | undefined): Eval { - try { - return yield* work - } catch (error) { - throw locate(error, node) - } -} - -// Lets the host run first, then starts the work. -function* afterSlice(start: () => Value | Eval): Eval { - yield Slice - const result = start() - return isEval(result) ? yield* result : result -} diff --git a/packages/codemode/src/interpreter/objects.ts b/packages/codemode/src/interpreter/objects.ts index 575452b7b36..096bda0f586 100644 --- a/packages/codemode/src/interpreter/objects.ts +++ b/packages/codemode/src/interpreter/objects.ts @@ -12,7 +12,9 @@ import { Throw, typeError, } from "./model.js" -import type { Entry, Eval, GeneratorRequest, Reaction, Resolving } from "./runner.js" +import type { Eval, GeneratorRequest } from "./signals.js" +import type { Frame } from "../machine/frame.js" +import type { Reaction, Resolving } from "../machine/scheduler.js" import { OrderedTable, type TableCursor } from "./table.js" /** Property attributes, as in a JS property descriptor. */ @@ -347,8 +349,8 @@ export type GeneratorState = export class GeneratorObj extends Opaque { override readonly tag = "Generator" state: GeneratorState = "suspendedStart" - /** The suspended body's activations, kept while it waits for the next request. */ - entries: Array | undefined + /** The suspended body's frames, kept while it waits for the next request. */ + frames: Array | undefined /** An async generator's requests, oldest first, each with the functions that settle its promise. */ readonly queue: Array<{ readonly request: GeneratorRequest; readonly resolving: Resolving }> = [] constructor( diff --git a/packages/codemode/src/interpreter/promises.ts b/packages/codemode/src/interpreter/promises.ts index 54804dfed53..e3dce972436 100644 --- a/packages/codemode/src/interpreter/promises.ts +++ b/packages/codemode/src/interpreter/promises.ts @@ -5,17 +5,8 @@ import { createAggregateErrorValue, materialize } from "./errors.js" import { describeValue, typeofValue } from "./references.js" import { isSupportedCallback } from "./callback.js" import type { Interpreter } from "./interpreter.js" -import { - type Combination, - type Completion, - type Eval, - Interruption, - normal, - type Reaction, - type Resolving, - type Start, - Task, -} from "./runner.js" +import { type Completion, type Eval, Interruption } from "./signals.js" +import type { Combination, Reaction, Resolving } from "../machine/scheduler.js" import { finishAwaitReturn } from "./generators.js" // A `resolve`/`reject` handed to an executor or thenable: calling it settles the capability. @@ -29,18 +20,18 @@ const resolvingFunctions = (ctx: Interpreter, resolving: Resolving) => ({ * it is located at `site`, the call that created it. */ export const capability = (ctx: Interpreter, site: AstNode | undefined): Resolving => - ctx.runner.resolving(ctx.runner.newPromise(), site) + ctx.machine.resolving(ctx.machine.newPromise(), site) /** Settles `resolving` with a completion: a fulfillment value is resolved, so a thenable is adopted. */ const settleWith = (resolving: Resolving, completion: Completion) => completion.ok ? resolving.resolve(completion.value as Value) : resolving.reject(completion.error) -/** Runs a settled promise's reaction as a job: it finishes at once, or returns the task that calls program code. */ +/** Runs a settled promise's reaction as a job: it finishes at once, or returns the work that calls program code. */ export const react = ( ctx: Interpreter, reaction: Exclude, settled: Completion, -): Start | void => { +): Eval | void => { switch (reaction.kind) { case "settle": return settleWith(reaction.target, settled) @@ -61,8 +52,33 @@ export const react = ( } } -/** A task that runs `body` as its own job. */ -const job = (body: Eval): Start => ({ task: new Task([{ body, boundary: undefined }]), input: normal(undefined) }) +/** + * NewPromiseResolveThenableJob: the work that calls the thenable's `then`. Adopting a native promise through the + * intrinsic `then` takes the same jobs as calling it, and keeps the rejection as thrown so its diagnostic survives. + */ +export const adoptThenable = ( + ctx: Interpreter, + promise: PromiseObj, + thenable: Obj, + then: Value, +): Eval | void => { + const machine = ctx.machine + const resolving = machine.resolving(promise) + if (then === machine.intrinsicThen && thenable instanceof PromiseObj) { + machine.then(thenable, { kind: "settle", target: resolving }) + return + } + const proto = ctx.builtins.Function + const resolve = new Resolver(proto, "", resolving, "resolve") + const reject = new Resolver(proto, "", resolving, "reject") + return (function* (): Eval { + try { + yield* ctx.call(then, thenable, [resolve, reject]) + } catch (error) { + resolving.reject(error) + } + })() +} // A `then`/`catch` reaction calls program code: the handler runs as a job, and its result or failure settles the // derived promise. A missing handler passes the outcome through. @@ -70,7 +86,7 @@ const handle = ( ctx: Interpreter, reaction: Extract, settled: Completion, -): Start | void => { +): Eval | void => { const handler = settled.ok ? reaction.onFulfilled : reaction.onRejected const derived = reaction.derived if (handler === undefined || (!settled.ok && settled.error instanceof Interruption)) { @@ -87,7 +103,7 @@ const handle = ( } derived.resolve(result) } - return job(body()) + return body() } const promiseStatics = ["all", "allSettled", "race", "any", "resolve", "reject", "withResolvers", "try"] as const @@ -98,8 +114,8 @@ function* invokePromiseMethod( args: Array, site: AstNode | undefined, ): Eval { - const runner = ctx.runner - if (name === "resolve") return runner.promiseResolve(args[0]) + const machine = ctx.machine + if (name === "resolve") return machine.promiseResolve(args[0]) if (name === "reject") { const made = capability(ctx, site) made.reject(new Throw(args[0])) @@ -134,7 +150,7 @@ function* combinator( iterable: Value, site: AstNode | undefined, ): Eval { - const runner = ctx.runner + const machine = ctx.machine const made = capability(ctx, site) try { const cursor = yield* ctx.iterate(iterable) @@ -146,13 +162,13 @@ function* combinator( while (true) { const step = yield* cursor.next() if (step.done) break - const item = runner.promiseResolve(step.value) + const item = machine.promiseResolve(step.value) const reaction: Reaction = { kind: "combinator", combination, index: index++, called: false } combination.values.push(undefined) combination.reasons.push(undefined) combination.remaining++ combination.attached.push([item, reaction]) - runner.then(item, reaction) + machine.then(item, reaction) } if (name === "race" && index === 0) { throw typeError("Promise.race([]) would never settle; provide at least one promise or value.") @@ -203,7 +219,7 @@ const combine = ( } const detachAll = (ctx: Interpreter, combination: Combination) => { - for (const [item, reaction] of combination.attached) ctx.runner.detach(item, reaction) + for (const [item, reaction] of combination.attached) ctx.machine.detach(item, reaction) } const finishCombination = (ctx: Interpreter, combination: Combination) => { @@ -230,7 +246,7 @@ const instanceMethod = ( const onFulfilled = name === "then" ? reactionHandler(args[0], method) : undefined const onRejected = reactionHandler(name === "then" ? args[1] : args[0], method) const made = capability(ctx, site) - ctx.runner.then(promise, { kind: "handler", onFulfilled, onRejected, derived: made }) + ctx.machine.then(promise, { kind: "handler", onFulfilled, onRejected, derived: made }) return made.promise } @@ -243,7 +259,7 @@ const chainFinally = ( site: AstNode | undefined, ): PromiseObj => { const made = capability(ctx, site) - ctx.runner.then( + ctx.machine.then( source, cleanup === undefined ? { kind: "settle", target: made } : { kind: "finally", cleanup, target: made, site }, ) @@ -255,7 +271,7 @@ const cleanUp = ( reaction: Extract, settled: Completion, ) => { - const runner = ctx.runner + const machine = ctx.machine const made = reaction.target if (!settled.ok && settled.error instanceof Interruption) return made.reject(settled.error) const cleanup = reaction.cleanup @@ -269,17 +285,17 @@ const cleanUp = ( } let after: PromiseObj try { - after = runner.promiseResolve(result) + after = machine.promiseResolve(result) } catch (error) { made.reject(error) return } // The spec chains `then(valueThunk)` onto the cleanup's promise: one more job before the outcome passes. const derived = capability(ctx, reaction.site) - runner.then(after, { kind: "finallyCleaned", target: derived }) - runner.then(derived.promise, { kind: "finallyOutcome", target: made, outcome: settled }) + machine.then(after, { kind: "finallyCleaned", target: derived }) + machine.then(derived.promise, { kind: "finallyOutcome", target: made, outcome: settled }) } - return job(body()) + return body() } const reactionHandler = (value: Value, method: string): Callable | undefined => { @@ -321,7 +337,7 @@ const promiseLikeMethods = (ctx: Interpreter) => { const source = receiver(Obj, thisValue, name) const copy = new Obj(ctx.builtins.Object) for (const [key, slot] of source.props) copy.props.set(key, { ...slot }) - return ctx.runner.promiseResolve(copy) + return ctx.machine.promiseResolve(copy) } methods( ctx.builtins, @@ -363,7 +379,7 @@ export const promiseGlobal = (ctx: Interpreter) => { ["finally", 1, (thisValue, args, site) => instanceMethod(ctx, "finally", thisValue, args, site)], ]) const then = get(proto, "then") - if (then instanceof Callable) ctx.runner.intrinsicThen = then + if (then instanceof Callable) ctx.machine.intrinsicThen = then promiseLikeMethods(ctx) return promise } diff --git a/packages/codemode/src/interpreter/runner.ts b/packages/codemode/src/interpreter/runner.ts deleted file mode 100644 index 22c6efc01b2..00000000000 --- a/packages/codemode/src/interpreter/runner.ts +++ /dev/null @@ -1,590 +0,0 @@ -import { Cause, type Context, Effect, Exit, Fiber } from "effect" -import type { Diagnostic } from "../codemode.js" -import { locate, normalizeError } from "./errors.js" -import type { Interpreter } from "./interpreter.js" -import { MAX_PENDING_PROMISES } from "./limits.js" -import { type AstNode, PendingThrow, rangeError, typeError } from "./model.js" -import { type Callable, type GeneratorObj, get, Obj, PromiseObj, Resolver, type Value } from "./objects.js" -import { react } from "./promises.js" -import { fromJson, type Json } from "../data.js" -import { typeofValue } from "./references.js" - -export { - abrupt, - AsyncBoundary, - type Body, - type Boundary, - Call, - type Completion, - Enter, - type Entry, - type Eval, - Await, - generatorBody, - type GeneratorRequest, - Host, - Interruption, - Invoke, - isEval, - MAX_CALL_DEPTH, - normal, - type Signal, - Slice, - Task, - Yield, -} from "./signals.js" -import { - abrupt, - AsyncBoundary, - Await, - type Body, - Call, - type Completion, - Enter, - type Eval, - Host, - Interruption, - Invoke, - normal, - type Signal, - Task, - Yield, -} from "./signals.js" - -export type Start = { readonly task: Task; readonly input: Completion } - -/** What the member reactions of one `Promise.all`/`allSettled`/`race`/`any` share. */ -export type Combination = { - readonly name: "all" | "allSettled" | "race" | "any" - /** Settles the combinator's result. */ - readonly made: Resolving - readonly values: Array - readonly reasons: Array - /** Members still to settle, plus one until iteration ends. */ - remaining: number - /** Every member's reaction, detached once the result is decided. */ - readonly attached: Array -} - -/** - * A settled promise's reaction, as data. It runs as a job (see `react` in promises.ts) and either finishes at once or - * starts a task. The kinds: - * - `await`: resume the async body parked on the promise. - * - `settle`: pass the outcome on to `target`. - * - `handler`: `then`/`catch` handlers; the handler's result settles `derived`. - * - `combinator`: one member of a `Promise` combinator. - * - `finally`, `finallyCleaned`, `finallyOutcome`: the steps of `Promise.prototype.finally` with a cleanup. - * - `awaitReturn`: an async generator's `return(value)` once `value` settles. - * - `callback`: a host closure, still used by the async-from-sync iterator adapter. - */ -export type Reaction = - | { readonly kind: "await"; readonly task: Task } - | { readonly kind: "settle"; readonly target: Resolving } - | { - readonly kind: "handler" - readonly onFulfilled: Callable | undefined - readonly onRejected: Callable | undefined - readonly derived: Resolving - } - | { readonly kind: "combinator"; readonly combination: Combination; readonly index: number; called: boolean } - | { - readonly kind: "finally" - readonly cleanup: Callable - readonly target: Resolving - readonly site: AstNode | undefined - } - | { readonly kind: "finallyCleaned"; readonly target: Resolving } - | { readonly kind: "finallyOutcome"; readonly target: Resolving; readonly outcome: Completion } - | { readonly kind: "awaitReturn"; readonly generator: GeneratorObj } - | { readonly kind: "callback"; readonly run: (settled: Completion) => Start | void } - -/** One queued job, as data: run a settled promise's reaction, resume a task, or adopt a thenable. */ -export type Job = - | { readonly kind: "reaction"; readonly reaction: Reaction; readonly settled: Completion } - | { readonly kind: "resume"; readonly task: Task; readonly input: Completion } - | { readonly kind: "thenable"; readonly promise: PromiseObj; readonly thenable: Obj; readonly then: Value } - -/** - * Host work the program waits on: a tool or extension call, running as a fiber the runner keeps by `id`. Its outcome - * arrives as an inbox record and settles the call's promise in a later job. - */ -export class HostCall { - status: "inFlight" | "completed" = "inFlight" - constructor( - readonly id: number, - /** What runs: a tool path, or an extension function's label. */ - readonly label: string, - /** A tool's arguments, as they crossed. */ - readonly args: ReadonlyArray | undefined, - /** How the result becomes a program value: a tool's crosses as JSON, an extension's is already one. */ - readonly result: "json" | "value", - readonly resolving: Resolving, - /** The call that started the work, where a defect it raises is located. */ - readonly site: AstNode | undefined, - ) {} -} - -/** A host call's outcome, waiting to be delivered. */ -type Delivery = { readonly call: HostCall; readonly completion: Completion } - -/** - * CreateResolvingFunctions as data: settles one promise at most once, however many `Resolver`s share it. `reject` takes - * the failure as thrown; one without a location is located at `site`, the call that created the promise. - */ -export class Resolving { - done = false - constructor( - private readonly runner: Pick, "resolveWith" | "reject">, - readonly promise: PromiseObj, - readonly site: AstNode | undefined, - ) {} - - resolve(resolution: Value): void { - if (this.done) return - this.done = true - this.runner.resolveWith(this.promise, resolution, this.site) - } - - reject(reason: unknown): void { - if (this.done) return - this.done = true - this.runner.reject(this.promise, locate(reason, this.site)) - } -} - -type Pause = - | { readonly kind: "slice"; readonly input: Completion } - | { readonly kind: "host"; readonly effect: Effect.Effect } - -/** Interpreter steps between clock checks, and how long one slice may run before the host gets a turn. */ -const FUEL = 64 -const SLICE_MS = 1 - -/** - * One execution's job queue. Program code runs only inside a job, one job at a time and each to completion, as in a - * JavaScript engine: promise reactions and resumed awaits queue in order, host work runs concurrently and settles its - * promise as a later job, and the program is stuck exactly when no job is queued and no host work is running. - */ -export class Runner { - /** The intrinsic `Promise.prototype.then`, so adopting a native promise skips exposing its rejection first. */ - intrinsicThen: Callable | undefined - private readonly queue: Array = [] - private head = 0 - private current: Task | undefined - private fuel = FUEL - private sliceStart = 0 - private pending = 0 - private nextID = 0 - private nextHostID = 0 - /** Rejections nothing handled yet, described when they happened so reporting them at the end is cheap. */ - private readonly unhandled = new Map() - /** Bodies suspended at an `await` and the promise each waits on, so a stall can point at the one that is stuck. */ - private readonly parked = new Map() - /** Running host work by `HostCall` id; fibers stay here and never become part of the program's state. */ - private readonly fibers = new Map>() - private readonly inbox: Array = [] - private wake: (() => void) | undefined - private stopping = false - - constructor( - private readonly ctx: Interpreter, - /** The services host work runs with: the caller's, so tools see its context, tracing, and clock. */ - private readonly services: Context.Context, - ) {} - - /** Activations on the running task's stack. */ - get depth(): number { - return this.current?.stack.length ?? 0 - } - - /** - * Runs the program as the first job until its body completes, then returns how it ended. Host work it started may - * still be running; `interrupt` stops it. - */ - run(main: Body): Effect.Effect { - const self = this - return Effect.gen(function* () { - let outcome: Completion | undefined - let start: Start | undefined = { - task: new Task([ - { - body: main, - boundary: new AsyncBoundary((completion) => { - outcome = completion - }, undefined), - }, - ]), - input: normal(undefined), - } - self.sliceStart = performance.now() - while (outcome === undefined) { - if (start === undefined) { - const job = self.dequeue() - if (job !== undefined) { - start = self.runJob(job) ?? undefined - if (start === undefined && self.tick()) yield* self.yieldToHost - continue - } - const delivery = self.inbox.shift() - if (delivery !== undefined) { - self.deliver(delivery) - continue - } - if (self.fibers.size > 0) { - yield* self.hostSettled - continue - } - return abrupt( - new PendingThrow( - "Error", - "The program is waiting on a promise that can never settle: nothing else is running, so nothing can resolve it. Code Mode has no timers or sleep, so don't wait or poll inside a program.", - self.stuckAwait(), - ), - ) - } - const pause = self.drive(start.task, start.input) - if (pause === undefined) { - start = undefined - if (self.tick()) yield* self.yieldToHost - continue - } - if (pause.kind === "slice") { - start = { task: start.task, input: pause.input } - yield* self.yieldToHost - continue - } - const exit = yield* Effect.exit(pause.effect) - if (Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) return yield* Effect.failCause(exit.cause) - start = { - task: start.task, - input: Exit.isSuccess(exit) ? normal(exit.value) : abrupt(Cause.squash(exit.cause)), - } - } - if (!outcome.ok && outcome.error instanceof Interruption) return yield* Effect.failCause(outcome.error.cause) - return outcome - }) - } - - /** Stops host work still running, waiting for its cleanup. */ - interrupt(): Effect.Effect { - const self = this - return Effect.gen(function* () { - self.stopping = true - while (self.fibers.size > 0) yield* Fiber.interruptAll([...self.fibers.values()]) - }) - } - - /** - * Rejections no handler observed, in the order their promises were created. Host work that finished before the - * program did settles first, so its failures count even though no later job ran. - */ - diagnostics(): Array { - for (let delivery = this.inbox.shift(); delivery !== undefined; delivery = this.inbox.shift()) - this.deliver(delivery) - return [...this.unhandled].sort(([left], [right]) => left.id - right.id).map(([, diagnostic]) => diagnostic) - } - - /** Counts one interpreter step; true when the slice is over and the caller should yield `Slice`. */ - tick(): boolean { - if (--this.fuel > 0) return false - this.fuel = FUEL - return performance.now() - this.sliceStart >= SLICE_MS - } - - /** A blocking host call, with the caller's services provided so the runner can run it as is. */ - blocking(effect: Effect.Effect): Host { - return new Host(Effect.provideContext(effect, this.services)) - } - - enqueue(job: Job): void { - this.queue.push(job) - } - - /** - * Starts host work now, settling `resolving`'s promise in a later job, in completion order. A defect is located at - * `site`, the call that started the work. - */ - host( - effect: Effect.Effect, - call: { - readonly label: string - readonly args?: ReadonlyArray - readonly result: "json" | "value" - readonly resolving: Resolving - readonly site: AstNode | undefined - }, - ): HostCall { - const record = new HostCall(this.nextHostID++, call.label, call.args, call.result, call.resolving, call.site) - // Starts synchronously, so admission and hooks happen at the call site. - const fiber = Effect.runForkWith(this.services)(effect) - this.fibers.set(record.id, fiber) - fiber.addObserver((exit) => { - this.fibers.delete(record.id) - // Interrupted by the program ending: nothing is left to settle. - if (this.stopping) return - this.inbox.push({ - call: record, - completion: Exit.isSuccess(exit) - ? normal(exit.value) - : abrupt( - Cause.hasInterruptsOnly(exit.cause) - ? new Interruption(exit.cause) - : Cause.hasDies(exit.cause) - ? locate(Cause.squash(exit.cause), record.site) - : Cause.squash(exit.cause), - ), - }) - this.wake?.() - }) - return record - } - - private deliver(delivery: Delivery): void { - const call = delivery.call - const completion = delivery.completion - call.status = "completed" - if (!completion.ok) return call.resolving.reject(completion.error) - call.resolving.resolve( - call.result === "json" ? fromJson(this.ctx, completion.value as Json | undefined) : (completion.value as Value), - ) - } - - // --- Promise abstract operations --- - - newPromise(): PromiseObj { - if (this.pending >= MAX_PENDING_PROMISES) { - throw rangeError( - `Too many pending promises (limit ${MAX_PENDING_PROMISES}); await promises before creating more.`, - ) - } - this.pending++ - return new PromiseObj(this.ctx.builtins.Promise, this.nextID++) - } - - /** CreateResolvingFunctions for `promise`; a failure without a location is located at `site`. */ - resolving(promise: PromiseObj, site?: AstNode): Resolving { - return new Resolving(this, promise, site) - } - - /** PerformPromiseThen with one reaction for either outcome; marks the promise handled. */ - then(promise: PromiseObj, reaction: Reaction): void { - if (promise.state === "pending") promise.reactions.push(reaction) - else { - if (promise.state === "rejected") this.unhandled.delete(promise) - const settled = promise.state === "fulfilled" ? normal(promise.result) : abrupt(promise.result) - this.enqueue({ kind: "reaction", reaction, settled }) - } - promise.handled = true - } - - /** Removes a reaction an internal consumer no longer needs, so a long-pending promise does not keep it. */ - detach(promise: PromiseObj, reaction: Reaction): void { - const index = promise.reactions.indexOf(reaction) - if (index >= 0) promise.reactions.splice(index, 1) - } - - /** PromiseResolve(%Promise%, value). */ - promiseResolve(value: Value): PromiseObj { - if (value instanceof PromiseObj) return value - const promise = this.newPromise() - this.resolveWith(promise, value) - return promise - } - - /** Rejects `promise` with a failure as thrown. */ - reject(promise: PromiseObj, reason: unknown): void { - this.settle(promise, abrupt(reason)) - } - - private settle(promise: PromiseObj, completion: Completion): void { - if (promise.state !== "pending") return - this.pending-- - promise.state = completion.ok ? "fulfilled" : "rejected" - promise.result = completion.ok ? completion.value : completion.error - const reactions = promise.reactions - promise.reactions = [] - if (!completion.ok && !promise.handled && !(completion.error instanceof Interruption)) { - const failure = normalizeError(completion.error) - this.unhandled.set(promise, { - ...failure, - message: `Unhandled rejection from an un-awaited promise: ${failure.message}`, - }) - } - for (const reaction of reactions) this.enqueue({ kind: "reaction", reaction, settled: completion }) - } - - /** ResolvePromise: adopts a thenable, otherwise fulfills. */ - resolveWith(promise: PromiseObj, resolution: Value, site?: AstNode): void { - if (resolution === promise) { - const cycle = typeError("Chaining cycle detected: a promise cannot resolve with itself.", site) - return this.settle(promise, abrupt(cycle)) - } - if (!(resolution instanceof Obj)) return this.settle(promise, normal(resolution)) - const then = get(resolution, "then") - if (typeofValue(then) !== "function") return this.settle(promise, normal(resolution)) - this.enqueue({ kind: "thenable", promise, thenable: resolution, then }) - } - - // NewPromiseResolveThenableJob. Adopting a native promise through the intrinsic `then` takes the same jobs as - // calling it, and keeps the rejection as thrown so its diagnostic survives. - private thenableJob(promise: PromiseObj, thenable: Obj, then: Value): Start | void { - const resolving = this.resolving(promise) - if (then === this.intrinsicThen && thenable instanceof PromiseObj) { - this.then(thenable, { kind: "settle", target: resolving }) - return - } - const proto = this.ctx.builtins.Function - const resolve = new Resolver(proto, "", resolving, "resolve") - const reject = new Resolver(proto, "", resolving, "reject") - const ctx = this.ctx - const body = function* (): Eval { - try { - yield* ctx.call(then, thenable, [resolve, reject]) - } catch (error) { - resolving.reject(error) - } - } - return { task: new Task([{ body: body(), boundary: undefined }]), input: normal(undefined) } - } - - private runJob(job: Job): Start | void { - if (job.kind === "resume") return { task: job.task, input: job.input } - if (job.kind === "thenable") return this.thenableJob(job.promise, job.thenable, job.then) - const reaction = job.reaction - if (reaction.kind !== "await") return react(this.ctx, reaction, job.settled) - this.parked.delete(reaction.task) - return { task: reaction.task, input: job.settled } - } - - // --- Driver --- - - // Runs a task until its stack empties or it pauses for a slice or host work, which resumes it before any other job. - private drive(task: Task, input: Completion): Pause | undefined { - const stack = task.stack - this.current = task - let completion = input - while (stack.length > 0) { - const entry = stack[stack.length - 1]! - let step: IteratorResult - try { - step = completion.ok ? entry.body.next(completion.value) : entry.body.throw(completion.error) - } catch (error) { - stack.pop() - completion = entry.boundary === undefined ? abrupt(error) : entry.boundary.finish(abrupt(error)) - continue - } - if (step.done) { - stack.pop() - completion = entry.boundary === undefined ? normal(step.value) : entry.boundary.finish(normal(step.value)) - continue - } - const signal = step.value - if (signal instanceof Call) { - stack.push({ body: signal.body, boundary: signal.boundary }) - completion = normal(undefined) - if (this.tick()) return { kind: "slice", input: completion } - continue - } - if (signal instanceof Invoke) { - let call: Call - try { - call = this.ctx.enter(signal) - } catch (error) { - completion = abrupt(error) - continue - } - stack.push({ body: call.body, boundary: call.boundary }) - completion = normal(undefined) - if (this.tick()) return { kind: "slice", input: completion } - continue - } - if (signal instanceof Enter) { - for (const saved of signal.entries) stack.push(saved) - completion = signal.input - if (this.tick()) return { kind: "slice", input: completion } - continue - } - if (signal instanceof Await) { - const index = stack.findLastIndex((item) => item.boundary?.async === true) - if (index < 0) { - completion = abrupt(typeError("await is only valid in async functions.")) - continue - } - let promise: PromiseObj - try { - promise = this.promiseResolve(signal.value) - } catch (error) { - completion = abrupt(error) - continue - } - const boundary = stack[index]!.boundary! - const continuation = new Task(stack.splice(index)) - this.parked.set(continuation, { promise, node: signal.node }) - this.then(promise, { kind: "await", task: continuation }) - completion = boundary.suspended() - continue - } - if (signal instanceof Yield) { - const index = stack.findLastIndex((item) => item.boundary?.generator === true) - if (index < 0) { - completion = abrupt(typeError("yield is only valid inside a generator.")) - continue - } - completion = stack[index]!.boundary!.yielded(signal.value, () => stack.splice(index)) - continue - } - if (signal instanceof Host) return { kind: "host", effect: signal.effect } - return { kind: "slice", input: normal(undefined) } - } - this.current = undefined - if (!completion.ok) throw completion.error - return undefined - } - - // The innermost stuck `await`: an outer one waits on the result of a parked async body, so prefer one that does not. - // Bodies that wait on each other in a cycle leave no such await; any located one is then as good as another. - private stuckAwait(): AstNode | undefined { - const results = new Set() - for (const task of this.parked.keys()) { - const boundary = task.stack[0]?.boundary - if (boundary instanceof AsyncBoundary) results.add(boundary.result) - } - const located = [...this.parked.values()].filter((parked) => parked.node !== undefined) - return (located.find((parked) => !results.has(parked.promise)) ?? located[0])?.node - } - - private dequeue(): Job | undefined { - if (this.head === this.queue.length) { - if (this.head > 0) { - this.queue.length = 0 - this.head = 0 - } - return undefined - } - const job = this.queue[this.head++] - if (this.head > 4096 && this.head * 2 > this.queue.length) { - this.queue.splice(0, this.head) - this.head = 0 - } - return job - } - - private get yieldToHost(): Effect.Effect { - return Effect.map(Effect.yieldNow, () => { - this.sliceStart = performance.now() - }) - } - - private get hostSettled(): Effect.Effect { - return Effect.callback((resume) => { - if (this.inbox.length > 0) return resume(Effect.void) - this.wake = () => { - this.wake = undefined - resume(Effect.void) - } - return Effect.sync(() => { - this.wake = undefined - }) - }) - } -} diff --git a/packages/codemode/src/interpreter/signals.ts b/packages/codemode/src/interpreter/signals.ts index 60956262123..9050b598791 100644 --- a/packages/codemode/src/interpreter/signals.ts +++ b/packages/codemode/src/interpreter/signals.ts @@ -1,14 +1,15 @@ -import type { Cause, Effect } from "effect" +import type { Cause } from "effect" +import type { Frame } from "../machine/frame.js" +import type { HostRequest, Resolving } from "../machine/scheduler.js" import { type AstNode, GeneratorReturn, type GeneratorRequestKind, typeError } from "./model.js" import type { Value } from "./objects.js" -// The signals interpreter work sends its driver, and the activation records the driver keeps. Nothing here needs -// Effect at run time, so the machine can use them too. +// The signals interpreter work (a built-in's host generator) sends the machine. Nothing here needs Effect at run time. /** - * Interpreter work: a generator that yields signals to the driver and returns its result. Work inside one activation - * composes with `yield*`; a program call is a `Call`, so program recursion lives on the explicit stack below and never - * grows the host stack. + * Interpreter work: a generator that yields signals to the machine and returns its result. It runs in a native frame; + * a program call is an `Invoke`, which the machine runs as frames on its own stack, so program recursion never grows + * the host stack. */ export type Eval = Generator @@ -19,16 +20,13 @@ export type Completion = export const normal = (value: unknown): Completion => ({ ok: true, value }) export const abrupt = (error: unknown): Completion => ({ ok: false, error }) -/** - * What an activation's body is: interpreter work, or a machine task (`src/machine`), which answers the same `next` and - * `throw` as a generator. - */ +/** What a native frame runs: interpreter work, or a machine task (a generator's body), which answers the same calls. */ export interface Body { next(input?: unknown): IteratorResult throw(error: unknown): IteratorResult } -/** Run `body` as a new activation; a boundary marks where a suspended body splits off the stack. */ +/** Run `body` in a native frame of its own; a boundary marks where the stack splits when the body above suspends. */ export class Call { constructor( readonly body: Body, @@ -36,10 +34,10 @@ export class Call { ) {} } -/** Push saved activations back on the stack and continue the top one with `input`. */ +/** Push saved frames back on the stack and continue the top one with `input`. */ export class Enter { constructor( - readonly entries: Array, + readonly frames: Array, readonly input: Completion, ) {} } @@ -59,7 +57,7 @@ export class Yield { /** Block the current job on host work: no other job runs until it ends, as with a synchronous host call. */ export class Host { - constructor(readonly effect: Effect.Effect) {} + constructor(readonly request: HostRequest) {} } /** @@ -74,10 +72,7 @@ export class Interruption { /** Let the host run; the current job continues before any other job. */ export const Slice: unique symbol = Symbol("codemode.slice") -/** - * Call a program function from inside interpreter work. The machine evaluator answers it on its own stack; the - * runner, when it reaches it, starts the call as a new activation. - */ +/** Call a program function from inside interpreter work; the machine runs the call as frames on its own stack. */ export class Invoke { constructor( readonly callable: Value, @@ -100,40 +95,36 @@ export interface Boundary { readonly generator: boolean /** The body above returned or threw. */ finish(completion: Completion): Completion - /** The body above suspended at an `await`; its activations are saved in the continuation. */ + /** The body above suspended at an `await`; its frames are parked on the awaited promise. */ suspended(): Completion - /** The body above yielded; `split` takes its activations off the stack when it suspends. */ - yielded(value: Value, split: () => Array): Completion + /** The body above yielded; `split` takes its frames off the stack when it suspends. */ + yielded(value: Value, split: () => Array): Completion } -/** The boundary of an async body: the body's completion settles it, and the caller receives `result` at once. */ +/** + * The boundary of host work that crosses its arguments first (a tool call): the caller receives the call's promise + * at once, and a failure while crossing rejects it. Once the work starts, its outcome settles the promise. + */ export class AsyncBoundary implements Boundary { readonly async = true readonly generator = false - constructor( - private readonly settle: (completion: Completion) => void, - readonly result: Value, - ) {} + constructor(readonly resolving: Resolving) {} + get result(): Value { + return this.resolving.promise + } finish(completion: Completion): Completion { - this.settle(completion) - return normal(this.result) + if (!completion.ok) this.resolving.reject(completion.error) + return normal(this.resolving.promise) } suspended(): Completion { - return normal(this.result) + return normal(this.resolving.promise) } yielded(): Completion { return abrupt(typeError("yield is only valid inside a generator.")) } } -export type Entry = { readonly body: Body; readonly boundary: Boundary | undefined } - -/** Paused or running program code: activations on an explicit stack, innermost last. */ -export class Task { - constructor(readonly stack: Array) {} -} - -/** Most activations one task may hold, as the call-stack limit. */ +/** Most activations one stack may hold, as the call-stack limit. */ export const MAX_CALL_DEPTH = 10_000 // Program values never have a `next` method of their own: an object's properties live in its property map. diff --git a/packages/codemode/src/machine/calls.ts b/packages/codemode/src/machine/calls.ts index 97ab17e144a..367fa6a543c 100644 --- a/packages/codemode/src/machine/calls.ts +++ b/packages/codemode/src/machine/calls.ts @@ -30,19 +30,22 @@ import { import { isOpaque } from "../interpreter/references.js" import { ScopeStack } from "../interpreter/scope.js" import { - AsyncBoundary, - Call, + abrupt, + type Body, + type Boundary, + type Completion, type Eval, generatorBody, Invoke, isEval, MAX_CALL_DEPTH, + normal, type Signal, } from "../interpreter/signals.js" +import type { Resolving } from "./scheduler.js" import type { StatementResult } from "../interpreter/model.js" import { Frame, kind, type Out, PENDING, type Pending } from "./frame.js" import { ev } from "./expressions.js" -import type { Machine } from "./machine.js" import { bind, declaring } from "./patterns.js" import { blockStatement } from "./statements.js" import { MachineTask } from "./task.js" @@ -59,7 +62,7 @@ export function dispatch( if (callable instanceof Fn) return invoke(task, callable, thisValue, args, node) if (callable instanceof Builtin) { // A loop made only of built-in calls has no program call or loop back-edge to count, so count the call. - if (task.ctx.runner.tick()) task.slice = true + if (task.machine.tick()) task.slice = true // A bound function calls its target directly, so a program target runs as a frame rather than inside a native. if (callable instanceof BoundFn) { return dispatch(task, callable.target, callable.boundThis, [...callable.boundArgs, ...args], node) @@ -79,67 +82,36 @@ export function dispatch( throw notAFunction(callee, node) } -// A program call: a synchronous body runs as a counted `Call` frame on this stack. An async body runs as a task of its -// own behind an async boundary, as the runner keeps it today, and the caller receives its promise; a generator call -// binds its parameters on this stack and returns the generator. +// A program call runs as a counted `Call` frame on this stack. An async function's frame is an async boundary: an +// `await` in its body cuts the stack there and the caller receives its promise, which the body's completion settles. +// A generator call binds its parameters on this stack and returns the generator. function invoke(task: MachineTask, fn: Fn, thisValue: Value, args: Array, node: AstNode | undefined) { const machine = task.machine - const runner = task.ctx.runner - if (runner.depth + machine.calls > MAX_CALL_DEPTH) throw rangeError("Maximum call stack size exceeded", node) - if (runner.tick()) task.slice = true - if (fn.async && !fn.generator) return task.request(asyncCall(machine, fn, thisValue, args, node), fn.body) + if (machine.depth > MAX_CALL_DEPTH) throw rangeError("Maximum call stack size exceeded", node) + if (machine.tick()) task.slice = true + const resolving = fn.async && !fn.generator ? machine.resolving(machine.newPromise()) : undefined const counted = !fn.generator - if (counted) machine.calls++ - return task.push(CallKind, fn.body, [fn, thisValue, args, node, counted, undefined, 0]) -} - -/** A program call the runner starts as its own activation, so its frame is not counted again. */ -export function enter(machine: Machine, invocation: Invoke): Call { - const fn = invocation.callable as Fn - const runner = machine.ctx.runner - if (runner.depth + machine.calls > MAX_CALL_DEPTH) { - throw rangeError("Maximum call stack size exceeded", invocation.site) - } - if (fn.async && !fn.generator) return asyncCall(machine, fn, invocation.thisValue, invocation.args, invocation.site) - return new Call(callTask(machine, fn, invocation.thisValue, invocation.args, invocation.site)) -} - -const callTask = (machine: Machine, fn: Fn, thisValue: Value, args: Array, site: AstNode | undefined) => { - const task = new MachineTask(machine, new ScopeStack([], fn)) - task.stack.push(new Frame(CallKind, fn.body, 0, task.scopes, [fn, thisValue, args, site, false, undefined, 0])) - return task -} - -// An async function's body is a task behind an async boundary: its completion settles the promise the caller gets. -const asyncCall = (machine: Machine, fn: Fn, thisValue: Value, args: Array, site: AstNode | undefined) => { - const body = callTask(machine, fn, thisValue, args, site) - const runner = machine.ctx.runner - const promise = runner.newPromise() - const resolving = runner.resolving(promise) - return new Call( - body, - new AsyncBoundary( - (completion) => - completion.ok ? resolving.resolve(completion.value as Value) : resolving.reject(completion.error), - promise, - ), - ) + if (counted) machine.depth++ + return task.push(CallKind, fn.body, [fn, thisValue, args, node, counted, undefined, 0, resolving]) } // The phases of a `Call` frame: bind the parameters (resuming at the slot index after a pattern finishes), make the -// generator or start the body, and receive the body's result. A generator's own body task starts at `RUN`. +// generator or start the body, and receive the body's result. A generator's own body task starts at `RUN`. An async +// function's frame that took a throw has `SETTLED` its promise with it. const ENTER = 0 const BIND = 1 const BODY = 2 const RUN = 3 const RESULT = 4 +const SETTLED = 5 -// Slots: fn, this, args, call site, whether the frame counts toward the depth limit, the parameter scope, and the -// next parameter to bind. +// Slots: fn, this, args, call site, whether the frame counts toward the depth limit, the parameter scope, the next +// parameter to bind, and for an async function what settles its promise. function callStep(task: MachineTask, frame: Frame, _node: AstNode, input: unknown): Out { const slots = frame.slots const fn = slots[0] as Fn let phase = frame.phase + if (phase === SETTLED) return (slots[7] as Resolving).promise if (phase === ENTER) { const scopes = activate(task, fn, slots[1] as Value, slots[2] as Array) frame.scope = scopes @@ -168,10 +140,16 @@ function callStep(task: MachineTask, frame: Frame, _node: AstNode, input: unknow input = result } } - if (slots[4]) task.machine.calls-- - if (fn.body.type !== "BlockStatement") return input as Value - const result = input as StatementResult - return result.kind === "return" ? result.value : undefined + const result = + fn.body.type !== "BlockStatement" + ? (input as Value) + : (input as StatementResult).kind === "return" + ? (input as Extract).value + : undefined + const resolving = slots[7] as Resolving | undefined + if (resolving === undefined) return result + resolving.resolve(result) + return resolving.promise } // The activation's scope chain: the closure's scopes and a parameter scope holding `this`, `arguments`, and the @@ -249,10 +227,19 @@ const generator = (task: MachineTask, fn: Fn, frame: Frame): GeneratorObj => { } export const CallKind = kind("Call", callStep, { - unwind: (task, frame) => { - if (frame.slots[4]) task.machine.calls-- - return false + // An async function's body that throws rejects its promise; the caller still receives the promise. + unwind: (_task, frame, error) => { + const resolving = frame.slots[7] as Resolving | undefined + if (resolving === undefined || frame.phase === SETTLED) return false + resolving.reject(error) + frame.phase = SETTLED + return true }, + cut: (frame) => { + const resolving = frame.slots[7] as Resolving | undefined + return resolving === undefined ? undefined : normal(resolving.promise) + }, + counts: (frame) => frame.slots[4] === true, }) // The phases of a native frame: running (or receiving the answer it waits for), throwing a failure into the work, done. @@ -262,10 +249,10 @@ const FINISHED = 2 /** * Runs a built-in's interpreter work as a native frame: program calls it makes (`Invoke`) run as frames on this stack, - * and anything else it asks for goes to the driver. The frame holds a host generator, so it is opaque. + * and anything else it asks for is answered by the step loop. The frame holds a host generator, so it is opaque. */ export function native(task: MachineTask, work: Eval, site: AstNode | undefined, located: boolean): Out { - const frame = new Frame(Native, site, RUNNING, task.scopes, [work, located, undefined]) + const frame = new Frame(Native, site, RUNNING, task.scopes, [work, located, undefined, undefined, false]) task.stack.push(frame) let out: Out try { @@ -278,25 +265,41 @@ export function native(task: MachineTask, work: Eval, site: AstNode | unde return out } -// Slots: the work, whether its failures are located at the call site, and a failure to throw into it. +/** + * A native frame for a body that runs as an activation of its own: a job's work, or a body a built-in starts behind a + * boundary (`Call`). It counts toward the depth limit; the boundary decides what its completion gives the frame below. + */ +export const nativeFrame = (scopes: ScopeStack, body: Body, boundary: Boundary | undefined, counted: boolean): Frame => + new Frame(Native, undefined, RUNNING, scopes, [body, false, undefined, boundary, counted]) + +/** The boundary a native frame's body runs behind, if any. */ +export const boundaryOf = (frame: Frame): Boundary | undefined => + frame.kind === Native ? (frame.slots[3] as Boundary | undefined) : undefined + +// Slots: the work, whether its failures are located at the call site, a failure to throw into it, the boundary it +// runs behind, and whether it counts toward the depth limit. function nativeStep(task: MachineTask, frame: Frame, site: AstNode | undefined, input: unknown): Out { const slots = frame.slots - const work = slots[0] as Eval + const work = slots[0] as Body let ok = frame.phase !== THROWING let value = ok ? input : slots[2] slots[2] = undefined frame.phase = RUNNING while (true) { - let result: IteratorResult + let result: IteratorResult try { result = ok ? work.next(value) : work.throw(value) } catch (error) { frame.phase = FINISHED + const boundary = slots[3] as Boundary | undefined + if (boundary !== undefined) return outcome(boundary.finish(abrupt(error))) throw slots[1] ? locate(error, site) : error } if (result.done) { frame.phase = FINISHED - return result.value + const boundary = slots[3] as Boundary | undefined + if (boundary !== undefined) return outcome(boundary.finish(normal(result.value))) + return result.value as Value } const signal = result.value if (!(signal instanceof Invoke)) { @@ -317,6 +320,11 @@ function nativeStep(task: MachineTask, frame: Frame, site: AstNode | undefined, } } +const outcome = (completion: Completion): Value => { + if (!completion.ok) throw completion.error + return completion.value as Value +} + export const Native = kind("Native", nativeStep, { opaque: true, unwind: (_task, frame, error) => { @@ -325,6 +333,11 @@ export const Native = kind("Native", nativeStep, { frame.slots[2] = error return true }, + cut: (frame) => { + const boundary = frame.slots[3] as Boundary | undefined + return boundary?.async ? boundary.suspended() : undefined + }, + counts: (frame) => frame.slots[4] === true, }) export type Hint = "number" | "string" | "default" diff --git a/packages/codemode/src/machine/frame.ts b/packages/codemode/src/machine/frame.ts index 829f91d553d..9b827f12449 100644 --- a/packages/codemode/src/machine/frame.ts +++ b/packages/codemode/src/machine/frame.ts @@ -1,5 +1,6 @@ import type { AstNode } from "../interpreter/model.js" import type { ScopeStack } from "../interpreter/scope.js" +import type { Completion } from "../interpreter/signals.js" import type { MachineTask } from "./task.js" /** @@ -26,6 +27,13 @@ export type Kind = { // Method syntax, so each kind's step can name the node type it evaluates and the input it receives. step(task: MachineTask, frame: Frame | undefined, node: AstNode | undefined, input: unknown): unknown unwind?(task: MachineTask, frame: Frame, error: unknown): boolean + /** + * Whether the frame is an async boundary, where an `await` above cuts the stack: what the frame below receives once + * the frames from this one up are parked on the awaited promise, or undefined when the frame is no boundary. + */ + cut?(frame: Frame): Completion | undefined + /** Whether the frame counts toward the call-depth limit while it is on the stack. */ + counts?(frame: Frame): boolean /** A frame of this kind holds a host generator (a built-in that calls program code), so it cannot be saved. */ readonly opaque?: boolean } @@ -47,7 +55,11 @@ export class Frame { ) {} } -export const kind = (name: string, step: Kind["step"], options?: Pick): Kind => ({ +export const kind = ( + name: string, + step: Kind["step"], + options?: Pick, +): Kind => ({ name, step, ...options, diff --git a/packages/codemode/src/machine/machine.ts b/packages/codemode/src/machine/machine.ts index e51f97a2c53..08d4604c4f7 100644 --- a/packages/codemode/src/machine/machine.ts +++ b/packages/codemode/src/machine/machine.ts @@ -1,34 +1,364 @@ +import type { Diagnostic } from "../codemode.js" +import { fromJson, type Json } from "../data.js" +import { normalizeError } from "../interpreter/errors.js" import type { Interpreter } from "../interpreter/interpreter.js" +import { MAX_PENDING_PROMISES } from "../interpreter/limits.js" +import { type AstNode, rangeError, typeError } from "../interpreter/model.js" +import { type Callable, get, Obj, PromiseObj, type Value } from "../interpreter/objects.js" +import { adoptThenable, react } from "../interpreter/promises.js" +import { typeofValue } from "../interpreter/references.js" +import { ScopeStack } from "../interpreter/scope.js" +import { + abrupt, + Await, + type Completion, + type Eval, + Host, + Interruption, + normal, + Slice, + Yield, +} from "../interpreter/signals.js" import type { Source } from "../interpreter/source.js" -import type { Call, Invoke } from "../interpreter/signals.js" +import { nativeFrame } from "./calls.js" import { Frame } from "./frame.js" -import { enter } from "./calls.js" +import { HostCall, type HostRequest, type Job, type Reaction, Resolving, type Stop } from "./scheduler.js" import { ProgramKind } from "./statements.js" import { MachineTask } from "./task.js" -/** Which evaluator runs program code: the machine (`src/machine`) or the generator evaluator in `interpreter.ts`. */ -export type Evaluator = "machine" | "legacy" +/** Interpreter steps between clock checks, and how long one slice may run before the host gets a turn. */ +const FUEL = 64 +const SLICE_MS = 1 + +/** A host call's outcome, waiting to be delivered. */ +type Delivery = { readonly call: HostCall; readonly completion: Completion } + +/** Frames parked on a promise at an `await`, and what the frame below their boundary received when they were cut. */ +type Parked = { readonly promise: PromiseObj; readonly node: AstNode | undefined; readonly result: Value } /** - * The machine evaluator for one execution. Program code runs as explicit frames on `MachineTask` stacks, stepped by a - * loop, instead of host generators. Until the scheduler moves onto the machine too, each task is an activation body - * the runner drives: the program, an async function's body, a generator's body, or a call the runner starts. + * One execution's machine: program code runs as explicit frames on a stack, stepped by a loop, and the job queue, + * promises, and host calls it waits on are data it keeps. Program code runs only inside a job, one job at a time and + * each to completion, as in a JavaScript engine: promise reactions and resumed awaits queue in order, host work runs + * outside and settles its promise as a later job, and the program is stuck exactly when no job is queued and no host + * work is running. + * + * `run` steps until the machine has to stop, and says why (`Stop`); the driver (`driver.ts`) does what the stop asks + * and calls `run` again. Nothing here runs host work or knows about Effect. */ export class Machine { - /** Program calls on the running task's machine stacks, counted toward the call-depth limit with the runner's. */ - calls = 0 + /** Activations on the stack that count toward the call-depth limit. */ + depth = 0 + /** The intrinsic `Promise.prototype.then`, so adopting a native promise skips exposing its rejection first. */ + intrinsicThen: Callable | undefined + /** The stack every job runs on. */ + readonly root: MachineTask + /** How the program's body ended, once it has. */ + outcome: Completion | undefined + /** Frames suspended at an `await`, by the slice cut off the stack, so a stall can point at the one that is stuck. */ + readonly parked = new Map, Parked>() - constructor(readonly ctx: Interpreter) {} + private readonly queue: Array = [] + private head = 0 + private fuel = FUEL + private sliceStart = 0 + private pending = 0 + private nextID = 0 + private nextHostID = 0 + /** Rejections nothing handled yet, described when they happened so reporting them at the end is cheap. */ + private readonly unhandled = new Map() + /** Host calls queued since the last stop, for the driver to start. */ + private queued: Array = [] + /** Host calls the driver is running, by id. */ + private readonly running = new Map() + private readonly inbox: Array = [] + /** What the root stack continues with: the job in progress was paused, or has just started. */ + private input: Completion | undefined + /** A slice is due once the queued host calls are handed over. */ + private sliced = false - /** The program's body as a task: it runs the program, adopts a returned thenable, and copies the result out. */ - program(source: Source): MachineTask { - const task = new MachineTask(this, this.ctx.programScopes()) - task.stack.push(new Frame(ProgramKind, source.program, 0, task.scopes, [])) - return task + constructor(readonly ctx: Interpreter) { + this.root = new MachineTask(this, new ScopeStack([])) } - /** A call the runner reached (a built-in called a program function outside any task), as its own activation. */ - enter(invoke: Invoke): Call { - return enter(this, invoke) + /** Queues the program's body as the first job. */ + start(source: Source): void { + this.root.enter([new Frame(ProgramKind, source.program, 0, this.ctx.programScopes(), [])]) + this.input = normal(undefined) + } + + /** Steps until the machine has to stop: see `Stop`. A throw no frame handles escapes as a defect. */ + run(): Stop { + this.sliceStart = performance.now() + while (true) { + if (this.queued.length > 0) { + const calls = this.queued + this.queued = [] + for (const call of calls) { + call.status = "inFlight" + this.running.set(call.id, call) + } + return { kind: "host", calls } + } + if (this.sliced) { + this.sliced = false + return { kind: "slice" } + } + const input = this.input + if (input !== undefined) { + this.input = undefined + this.continue(input) + continue + } + // A blocking call is running: the job that made it continues once it completes. + if (this.root.stack.length > 0) return { kind: "host", calls: [] } + if (this.outcome !== undefined) return { kind: "done", completion: this.outcome } + const job = this.dequeue() + if (job !== undefined) { + this.begin(job) + if (this.input === undefined && this.tick()) this.sliced = true + continue + } + const delivery = this.inbox.shift() + if (delivery !== undefined) { + this.deliver(delivery) + continue + } + if (this.running.size > 0) return { kind: "host", calls: [] } + return { kind: "stuck", node: this.stuckAwait() } + } + } + + /** + * Reports how host work handed over by a stop ended. A blocking call's outcome continues the job that made it; a + * concurrent call's waits in the inbox and settles its promise in a later job, in completion order. + */ + complete(id: number, completion: Completion): void { + const call = this.running.get(id) + if (call === undefined) return + this.running.delete(id) + if (call.blocking) { + call.status = "completed" + this.input = completion + return + } + this.inbox.push({ call, completion }) + } + + /** Whether a host call's outcome is waiting to be delivered. */ + get ready(): boolean { + return this.inbox.length > 0 || this.input !== undefined + } + + /** + * Rejections no handler observed, in the order their promises were created. Host work that finished before the + * program did settles first, so its failures count even though no later job ran. + */ + diagnostics(): Array { + for (let delivery = this.inbox.shift(); delivery !== undefined; delivery = this.inbox.shift()) + this.deliver(delivery) + return [...this.unhandled].sort(([left], [right]) => left.id - right.id).map(([, diagnostic]) => diagnostic) + } + + /** Counts one interpreter step; true when the slice is over and the host should get a turn. */ + tick(): boolean { + if (--this.fuel > 0) return false + this.fuel = FUEL + return performance.now() - this.sliceStart >= SLICE_MS + } + + enqueue(job: Job): void { + this.queue.push(job) + } + + /** + * Queues concurrent host work, settling `resolving`'s promise in a later job, in completion order. The driver starts + * it at the next stop, before any other job runs. A defect is located at `site`, the call that started the work. + */ + host( + request: HostRequest, + call: { + readonly label: string + readonly result: "json" | "value" + readonly resolving: Resolving + readonly site: AstNode | undefined + }, + ): HostCall { + const record = new HostCall(this.nextHostID++, request, call.label, false, call.result, call.resolving, call.site) + this.queued.push(record) + return record + } + + /** A native frame that runs `body` as a job of its own. */ + job(body: Eval): Frame { + return nativeFrame(this.root.scopes, body, undefined, true) + } + + /** Parks frames cut off the stack at an `await` on the awaited promise; they resume as a job once it settles. */ + park(frames: Array, promise: PromiseObj, node: AstNode | undefined, result: Value): void { + this.parked.set(frames, { promise, node, result }) + this.then(promise, { kind: "await", frames }) + } + + // --- Promise abstract operations --- + + newPromise(): PromiseObj { + if (this.pending >= MAX_PENDING_PROMISES) { + throw rangeError( + `Too many pending promises (limit ${MAX_PENDING_PROMISES}); await promises before creating more.`, + ) + } + this.pending++ + return new PromiseObj(this.ctx.builtins.Promise, this.nextID++) + } + + /** CreateResolvingFunctions for `promise`; a failure without a location is located at `site`. */ + resolving(promise: PromiseObj, site?: AstNode): Resolving { + return new Resolving(this, promise, site) + } + + /** PerformPromiseThen with one reaction for either outcome; marks the promise handled. */ + then(promise: PromiseObj, reaction: Reaction): void { + if (promise.state === "pending") promise.reactions.push(reaction) + else { + if (promise.state === "rejected") this.unhandled.delete(promise) + const settled = promise.state === "fulfilled" ? normal(promise.result) : abrupt(promise.result) + this.enqueue({ kind: "reaction", reaction, settled }) + } + promise.handled = true + } + + /** Removes a reaction an internal consumer no longer needs, so a long-pending promise does not keep it. */ + detach(promise: PromiseObj, reaction: Reaction): void { + const index = promise.reactions.indexOf(reaction) + if (index >= 0) promise.reactions.splice(index, 1) + } + + /** PromiseResolve(%Promise%, value). */ + promiseResolve(value: Value): PromiseObj { + if (value instanceof PromiseObj) return value + const promise = this.newPromise() + this.resolveWith(promise, value) + return promise + } + + /** Rejects `promise` with a failure as thrown. */ + reject(promise: PromiseObj, reason: unknown): void { + this.settle(promise, abrupt(reason)) + } + + /** ResolvePromise: adopts a thenable, otherwise fulfills. */ + resolveWith(promise: PromiseObj, resolution: Value, site?: AstNode): void { + if (resolution === promise) { + const cycle = typeError("Chaining cycle detected: a promise cannot resolve with itself.", site) + return this.settle(promise, abrupt(cycle)) + } + if (!(resolution instanceof Obj)) return this.settle(promise, normal(resolution)) + const then = get(resolution, "then") + if (typeofValue(then) !== "function") return this.settle(promise, normal(resolution)) + this.enqueue({ kind: "thenable", promise, thenable: resolution, then }) + } + + private settle(promise: PromiseObj, completion: Completion): void { + if (promise.state !== "pending") return + this.pending-- + promise.state = completion.ok ? "fulfilled" : "rejected" + promise.result = completion.ok ? completion.value : completion.error + const reactions = promise.reactions + promise.reactions = [] + if (!completion.ok && !promise.handled && !(completion.error instanceof Interruption)) { + const failure = normalizeError(completion.error) + this.unhandled.set(promise, { + ...failure, + message: `Unhandled rejection from an un-awaited promise: ${failure.message}`, + }) + } + for (const reaction of reactions) this.enqueue({ kind: "reaction", reaction, settled: completion }) + } + + // --- Jobs --- + + // Starts a job on the root stack: parked frames resume with how their promise settled, and a reaction or thenable + // that calls program code runs its work in a native frame. One that finishes at once leaves the stack empty. + private begin(job: Job): void { + if (job.kind === "resume") return this.resume(job.frames, job.input) + if (job.kind === "thenable") return this.work(adoptThenable(this.ctx, job.promise, job.thenable, job.then)) + const reaction = job.reaction + if (reaction.kind !== "await") return this.work(react(this.ctx, reaction, job.settled)) + this.parked.delete(reaction.frames) + this.resume(reaction.frames, job.settled) + } + + private resume(frames: Array, input: Completion): void { + this.root.enter(frames) + this.input = input + } + + private work(body: Eval | void): void { + if (body === undefined) return + this.root.enter([this.job(body)]) + this.input = normal(undefined) + } + + // Continues the job on the root stack. It ends when the stack empties, or pauses for a slice or a blocking host call; + // either way it continues before any other job. + private continue(input: Completion): void { + const root = this.root + const result = root.drive(input.ok, input.ok ? input.value : input.error) + if (result.done) { + if (this.tick()) this.sliced = true + return + } + const signal = result.value + if (signal === Slice) { + this.input = normal(undefined) + this.sliced = true + return + } + if (signal instanceof Host) { + const call = new HostCall(this.nextHostID++, signal.request, "", true, "value", undefined, undefined) + this.queued.push(call) + return + } + // Nothing on the stack can take an `await` or `yield`: the frame that asked receives the failure. + if (signal instanceof Await) this.input = abrupt(typeError("await is only valid in async functions.")) + else if (signal instanceof Yield) this.input = abrupt(typeError("yield is only valid inside a generator.")) + else this.input = abrupt(new TypeError("The machine cannot answer this signal here.")) + } + + private deliver(delivery: Delivery): void { + const call = delivery.call + const completion = delivery.completion + call.status = "completed" + const resolving = call.resolving! + if (!completion.ok) return resolving.reject(completion.error) + resolving.resolve( + call.result === "json" ? fromJson(this.ctx, completion.value as Json | undefined) : (completion.value as Value), + ) + } + + // The innermost stuck `await`: an outer one waits on the result of a parked async body, so prefer one that does not. + // Bodies that wait on each other in a cycle leave no such await; any located one is then as good as another. + private stuckAwait(): AstNode | undefined { + const results = new Set() + for (const parked of this.parked.values()) if (parked.result instanceof PromiseObj) results.add(parked.result) + const located = [...this.parked.values()].filter((parked) => parked.node !== undefined) + return (located.find((parked) => !results.has(parked.promise)) ?? located[0])?.node + } + + private dequeue(): Job | undefined { + if (this.head === this.queue.length) { + if (this.head > 0) { + this.queue.length = 0 + this.head = 0 + } + return undefined + } + const job = this.queue[this.head++] + if (this.head > 4096 && this.head * 2 > this.queue.length) { + this.queue.splice(0, this.head) + this.head = 0 + } + return job } } diff --git a/packages/codemode/src/machine/scheduler.ts b/packages/codemode/src/machine/scheduler.ts new file mode 100644 index 00000000000..6293d904f5e --- /dev/null +++ b/packages/codemode/src/machine/scheduler.ts @@ -0,0 +1,136 @@ +import type { Json } from "../data.js" +import { locate } from "../interpreter/errors.js" +import type { AstNode } from "../interpreter/model.js" +import type { Callable, GeneratorObj, Obj, PromiseObj, Value } from "../interpreter/objects.js" +import type { Completion, Eval } from "../interpreter/signals.js" +import type { Frame } from "./frame.js" + +// The scheduler's records: jobs, reactions, resolvers, and host calls. All of them are data the machine keeps, so a +// run's queue and the work it waits on can be read (and later saved) at any step boundary. + +/** What the member reactions of one `Promise.all`/`allSettled`/`race`/`any` share. */ +export type Combination = { + readonly name: "all" | "allSettled" | "race" | "any" + /** Settles the combinator's result. */ + readonly made: Resolving + readonly values: Array + readonly reasons: Array + /** Members still to settle, plus one until iteration ends. */ + remaining: number + /** Every member's reaction, detached once the result is decided. */ + readonly attached: Array +} + +/** + * A settled promise's reaction, as data. It runs as a job (see `react` in promises.ts) and either finishes at once or + * starts interpreter work. The kinds: + * - `await`: resume the frames parked on the promise, cut off the stack at an `await`. + * - `settle`: pass the outcome on to `target`. + * - `handler`: `then`/`catch` handlers; the handler's result settles `derived`. + * - `combinator`: one member of a `Promise` combinator. + * - `finally`, `finallyCleaned`, `finallyOutcome`: the steps of `Promise.prototype.finally` with a cleanup. + * - `awaitReturn`: an async generator's `return(value)` once `value` settles. + * - `callback`: a host closure, still used by the async-from-sync iterator adapter. + */ +export type Reaction = + | { readonly kind: "await"; readonly frames: Array } + | { readonly kind: "settle"; readonly target: Resolving } + | { + readonly kind: "handler" + readonly onFulfilled: Callable | undefined + readonly onRejected: Callable | undefined + readonly derived: Resolving + } + | { readonly kind: "combinator"; readonly combination: Combination; readonly index: number; called: boolean } + | { + readonly kind: "finally" + readonly cleanup: Callable + readonly target: Resolving + readonly site: AstNode | undefined + } + | { readonly kind: "finallyCleaned"; readonly target: Resolving } + | { readonly kind: "finallyOutcome"; readonly target: Resolving; readonly outcome: Completion } + | { readonly kind: "awaitReturn"; readonly generator: GeneratorObj } + | { readonly kind: "callback"; readonly run: (settled: Completion) => Eval | void } + +/** One queued job, as data: run a settled promise's reaction, resume parked frames, or adopt a thenable. */ +export type Job = + | { readonly kind: "reaction"; readonly reaction: Reaction; readonly settled: Completion } + | { readonly kind: "resume"; readonly frames: Array; readonly input: Completion } + | { readonly kind: "thenable"; readonly promise: PromiseObj; readonly thenable: Obj; readonly then: Value } + +/** + * What host work a call asks for. A tool call or a search is data; an extension's work is a live host value the driver + * runs as it is. + */ +export type HostRequest = + | { readonly kind: "tool"; readonly path: ReadonlyArray; readonly args: Array } + | { readonly kind: "search"; readonly args: Array } + | { readonly kind: "effect"; readonly effect: unknown } + +/** + * Host work the program waits on. The machine queues it and hands it to the driver at its next stop; the driver runs + * it and reports how it ended with `Machine.complete`. A concurrent call's outcome waits in the inbox and settles its + * promise in a later job; a blocking call stops the machine, and no other job runs until it ends. + */ +export class HostCall { + status: "queued" | "inFlight" | "completed" = "queued" + constructor( + readonly id: number, + readonly request: HostRequest, + /** What runs: a tool path, or an extension function's label. */ + readonly label: string, + /** The running job waits for it: its outcome continues the frame that asked, before any other job. */ + readonly blocking: boolean, + /** How a concurrent call's result becomes a program value: a tool's crosses as JSON, an extension's is one. */ + readonly result: "json" | "value", + /** Settles a concurrent call's promise. */ + readonly resolving: Resolving | undefined, + /** The call that started the work, where a defect it raises is located. */ + readonly site: AstNode | undefined, + ) {} +} + +/** + * CreateResolvingFunctions as data: settles one promise at most once, however many `Resolver`s share it. `reject` takes + * the failure as thrown; one without a location is located at `site`, the call that created the promise. + */ +export class Resolving { + done = false + constructor( + private readonly scheduler: Settles, + readonly promise: PromiseObj, + readonly site: AstNode | undefined, + ) {} + + resolve(resolution: Value): void { + if (this.done) return + this.done = true + this.scheduler.resolveWith(this.promise, resolution, this.site) + } + + reject(reason: unknown): void { + if (this.done) return + this.done = true + this.scheduler.reject(this.promise, locate(reason, this.site)) + } +} + +type Settles = { + resolveWith(promise: PromiseObj, resolution: Value, site?: AstNode): void + reject(promise: PromiseObj, reason: unknown): void +} + +/** + * Why `Machine.run` stopped: + * - `done`: the program's body ended, with its completion. + * - `host`: start `calls`, newly queued host work, in order (a blocking call is last, and the machine continues only + * once it completes). With no calls, the machine waits for host work already running to complete. + * - `slice`: the step budget is used up; let the host run, then continue. + * - `stuck`: nothing can run or settle any more; `node` is the `await` that waits forever, when one is known. + */ +export type Stop = + | { readonly kind: "done"; readonly completion: Completion } + | { readonly kind: "host"; readonly calls: ReadonlyArray } + | { readonly kind: "slice" } + | { readonly kind: "stuck"; readonly node: AstNode | undefined } diff --git a/packages/codemode/src/machine/statements.ts b/packages/codemode/src/machine/statements.ts index cd65b9cbb26..39980b67d6e 100644 --- a/packages/codemode/src/machine/statements.ts +++ b/packages/codemode/src/machine/statements.ts @@ -49,7 +49,7 @@ import { predeclarePattern, } from "../interpreter/operations.js" import { describeValue } from "../interpreter/references.js" -import { Await, Interruption } from "../interpreter/signals.js" +import { abrupt, Await, Interruption, normal } from "../interpreter/signals.js" import { type Frame, kind, type Out, PENDING, type Pending } from "./frame.js" import { ev, named } from "./expressions.js" import { type Iter, iteratorClose, iteratorStep, openIterator } from "./iteration.js" @@ -357,7 +357,7 @@ const Labeled = kind("LabeledStatement", labeledStatement) // The machine checks for a slice at each loop back-edge, so a long loop lets the host run. const backEdge = (task: MachineTask, frame: Frame, phase: number): boolean => { - if (!task.ctx.runner.tick() && !task.slice) return false + if (!task.machine.tick() && !task.slice) return false task.slice = true frame.phase = phase return true @@ -866,10 +866,15 @@ const Try = kind("TryStatement", tryStatement, { // Slots: the statement running and the value of the last expression statement. Phases: 1 has a statement's result, // 2 has the last expression's value, 3 has a returned thenable settled, 4 has the result copied out. +// The phases of the program's frame: 0 starts, 1 has a statement's result, 2 has the body's value, 3 has it adopted, +// 4 has it copied out, and `ENDED` has ended the program, its outcome recorded on the machine. +const ENDED = 5 + function programStep(task: MachineTask, frame: Frame, node: Program, input: unknown) { const slots = frame.slots const body = node.body let phase = frame.phase + if (phase === ENDED) return undefined let value: Value = undefined let index = 0 if (phase === 0) { @@ -930,9 +935,12 @@ function programStep(task: MachineTask, frame: Frame, node: Program, input: unkn } if (phase === 3) { frame.phase = 4 - return task.native(toBoundary(task.ctx, input as Value), node, false) + const copied = task.native(toBoundary(task.ctx, input as Value), node, false) + if (copied === PENDING) return PENDING + input = copied } - return input + task.machine.outcome = normal(input) + return undefined } const outsideLoop = (result: StatementResult, statement: Statement | ModuleDeclaration) => { @@ -941,4 +949,17 @@ const outsideLoop = (result: StatementResult, statement: Statement | ModuleDecla } } -export const ProgramKind = kind("Program", programStep) +/** + * The program's body, the first job. It is an async boundary: an `await` in it parks the whole body, and how the body + * ends, normally or by a throw, is the program's outcome. + */ +export const ProgramKind = kind("Program", programStep, { + unwind: (task, frame, error) => { + if (frame.phase === ENDED) return false + task.machine.outcome = abrupt(error) + frame.phase = ENDED + return true + }, + cut: () => normal(undefined), + counts: () => true, +}) diff --git a/packages/codemode/src/machine/task.ts b/packages/codemode/src/machine/task.ts index cf09fec71cf..4150264c685 100644 --- a/packages/codemode/src/machine/task.ts +++ b/packages/codemode/src/machine/task.ts @@ -3,24 +3,40 @@ import type { Interpreter } from "../interpreter/interpreter.js" import type { AstNode } from "../interpreter/model.js" import type { Value } from "../interpreter/objects.js" import type { ScopeStack } from "../interpreter/scope.js" -import { type Eval, type Signal, Slice } from "../interpreter/signals.js" +import { + abrupt, + Await, + Call, + type Completion, + Enter, + type Eval, + normal, + type Signal, + Slice, + Yield, +} from "../interpreter/signals.js" import { Frame, type Kind, PASS, PENDING, type Pending } from "./frame.js" -import { dispatch, native } from "./calls.js" +import { boundaryOf, dispatch, native, nativeFrame } from "./calls.js" import type { Machine } from "./machine.js" /** Steps run by every machine task in this process, so tests can see that a program ran on the machine. */ export const stats = { steps: 0 } /** - * A stack of machine frames and the loop that steps it. To the runner it is an activation body, like interpreter - * work: `next` and `throw` run it until it finishes or needs the driver (`Await`, `Yield`, a host call, a slice, or a - * call it cannot make on its own stack), and the driver's answer resumes the frame that asked. + * A stack of machine frames and the loop that steps it. The machine runs every job on its root task. A generator's + * body is a task of its own for now, run by a native frame on the stack that resumes it; to that frame it answers + * `next` and `throw` like interpreter work. + * + * The loop answers what frames ask for on the stack itself: an `await` cuts the stack at the innermost async boundary + * and parks the frames it cut on the promise, a `yield` splits it at the innermost generator body, and a body a + * built-in starts (`Call`) or a generator resumes (`Enter`) is pushed. What it cannot answer (host work, a slice, or an + * `await` or `yield` with no boundary on this stack) it hands to whoever runs it. */ export class MachineTask { readonly stack: Array = [] /** The scope chain of the frame running now. */ scopes: ScopeStack - /** What the running frame asked the driver for; the step loop hands it over once the frame parks. */ + /** What the running frame asked for; the step loop answers it once the frame parks. */ signal: Signal | undefined = undefined /** The host should get a turn at the next step boundary. */ slice = false @@ -103,15 +119,28 @@ export class MachineTask { return native(this, work, site, locate) } - /** Asks the driver for `signal`; its answer (or failure) is the result of the frame this pushes. */ + /** Asks for `signal`; its answer (or failure) is the result of the frame this pushes. */ request(signal: Signal, node: AstNode | undefined): Pending { this.signal = signal this.stack.push(new Frame(passKind, node, PASS, this.scopes, [])) return PENDING } - private drive(ok: boolean, input: unknown): IteratorResult { + /** Puts frames taken off a stack back on this one, counting their calls again. */ + enter(frames: ReadonlyArray): void { + for (const frame of frames) { + this.stack.push(frame) + if (frame.kind.counts?.(frame)) this.machine.depth++ + } + } + + /** + * Runs the stack from the top frame with `input` until it empties (its last result) or a frame asks for something + * this stack cannot answer (the signal). A throw that unwinds past the bottom frame is thrown. + */ + drive(ok: boolean, input: unknown): IteratorResult { const stack = this.stack + const machine = this.machine let value = input while (true) { stats.steps++ @@ -130,6 +159,7 @@ export class MachineTask { } } stack.pop() + if (frame.kind.counts?.(frame)) machine.depth-- } ok = true value = undefined @@ -151,6 +181,7 @@ export class MachineTask { } if (out !== PENDING) { stack.pop() + if (frame.kind.counts?.(frame)) machine.depth-- value = out continue } @@ -158,7 +189,11 @@ export class MachineTask { const signal = this.signal if (signal !== undefined) { this.signal = undefined - return { done: false, value: signal } + const answer = this.answer(signal) + if (answer === undefined) return { done: false, value: signal } + ok = answer.ok + value = answer.ok ? answer.value : answer.error + continue } if (this.slice) { this.slice = false @@ -166,7 +201,59 @@ export class MachineTask { } } } + + // Answers a signal on this stack, giving what the top frame continues with; undefined hands the signal on. + private answer(signal: Signal): Completion | undefined { + const machine = this.machine + if (signal instanceof Call) { + this.stack.push(nativeFrame(this.scopes, signal.body, signal.boundary, true)) + machine.depth++ + if (machine.tick()) this.slice = true + return normal(undefined) + } + if (signal instanceof Enter) { + this.enter(signal.frames) + if (machine.tick()) this.slice = true + return signal.input + } + if (signal instanceof Await) return this.await(signal) + if (signal instanceof Yield) { + const stack = this.stack + for (let index = stack.length - 1; index >= 0; index--) { + const boundary = boundaryOf(stack[index]!) + if (boundary?.generator) return boundary.yielded(signal.value, () => this.split(index)) + } + } + return undefined + } + + // Cuts the stack at the innermost async boundary and parks the frames cut off on the awaited promise; the frame + // below the boundary continues with what the boundary gives it (an async function's promise, say). + private await(signal: Await): Completion | undefined { + const stack = this.stack + for (let index = stack.length - 1; index >= 0; index--) { + const frame = stack[index]! + const cut = frame.kind.cut?.(frame) + if (cut === undefined) continue + let promise + try { + promise = this.machine.promiseResolve(signal.value) + } catch (error) { + return abrupt(error) + } + this.machine.park(this.split(index), promise, signal.node, cut.ok ? (cut.value as Value) : undefined) + return cut + } + return undefined + } + + // Takes the frames from `index` up off the stack; their calls no longer count toward its depth. + private split(index: number): Array { + const frames = this.stack.splice(index) + for (const frame of frames) if (frame.kind.counts?.(frame)) this.machine.depth-- + return frames + } } -// A frame waiting on the driver's answer to a request; the answer passes straight to the frame below. +// A frame waiting on the answer to a request; the answer passes straight to the frame below. const passKind: Kind = { name: "Request", step: (_task, _frame, _node, input) => input } diff --git a/packages/codemode/src/stdlib/array.ts b/packages/codemode/src/stdlib/array.ts index a600cf8db5d..98f0992db91 100644 --- a/packages/codemode/src/stdlib/array.ts +++ b/packages/codemode/src/stdlib/array.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, type Method, methods, prototypeFrom, receiver } from "../interpreter/native.js" import { checkArrayLength, checkStringLength, MAX_ARRAY_LENGTH } from "../interpreter/limits.js" import { invalidData, IteratorSymbol, rangeError, typeError } from "../interpreter/model.js" diff --git a/packages/codemode/src/stdlib/bytes.ts b/packages/codemode/src/stdlib/bytes.ts index 6e6ff0e1f77..7f37b2bca22 100644 --- a/packages/codemode/src/stdlib/bytes.ts +++ b/packages/codemode/src/stdlib/bytes.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { checkArrayLength, checkStringLength } from "../interpreter/limits.js" import { constructor, methods, prototypeFrom, receiver, requiresNew } from "../interpreter/native.js" import { IteratorSymbol, rangeError, syntaxError, typeError } from "../interpreter/model.js" diff --git a/packages/codemode/src/stdlib/collections.ts b/packages/codemode/src/stdlib/collections.ts index 3bec536dc40..37f5eca798b 100644 --- a/packages/codemode/src/stdlib/collections.ts +++ b/packages/codemode/src/stdlib/collections.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, fn, type Method, methods, prototypeFrom, receiver, requiresNew } from "../interpreter/native.js" import { invalidData, IteratorSymbol, typeError } from "../interpreter/model.js" import { diff --git a/packages/codemode/src/stdlib/date.ts b/packages/codemode/src/stdlib/date.ts index 5fd2b12d565..ebf5b4c5193 100644 --- a/packages/codemode/src/stdlib/date.ts +++ b/packages/codemode/src/stdlib/date.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, type Method, methods, prototypeFrom, receiver } from "../interpreter/native.js" import { rangeError } from "../interpreter/model.js" import { DateObj, Obj, coerceToNumber, coerceToString, type Value } from "../interpreter/objects.js" diff --git a/packages/codemode/src/stdlib/headers.ts b/packages/codemode/src/stdlib/headers.ts index 0e052733077..67c156df799 100644 --- a/packages/codemode/src/stdlib/headers.ts +++ b/packages/codemode/src/stdlib/headers.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, methods, prototypeFrom, receiver, requiresNew } from "../interpreter/native.js" import { IteratorSymbol, typeError } from "../interpreter/model.js" import { diff --git a/packages/codemode/src/stdlib/iterator.ts b/packages/codemode/src/stdlib/iterator.ts index 99210a8553e..bdb05bb8f0b 100644 --- a/packages/codemode/src/stdlib/iterator.ts +++ b/packages/codemode/src/stdlib/iterator.ts @@ -16,7 +16,7 @@ import { type Value, } from "../interpreter/objects.js" import { describeValue } from "../interpreter/references.js" -import { type Eval, Interruption } from "../interpreter/runner.js" +import { type Eval, Interruption } from "../interpreter/signals.js" const finished: Step = { done: true, value: undefined } diff --git a/packages/codemode/src/stdlib/json.ts b/packages/codemode/src/stdlib/json.ts index c90c123fcdf..a1aef194525 100644 --- a/packages/codemode/src/stdlib/json.ts +++ b/packages/codemode/src/stdlib/json.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { methods } from "../interpreter/native.js" import { applyCollectionCallback } from "../interpreter/callback.js" import type { Interpreter } from "../interpreter/interpreter.js" diff --git a/packages/codemode/src/stdlib/math.ts b/packages/codemode/src/stdlib/math.ts index 5cb9e11a542..9f979bab658 100644 --- a/packages/codemode/src/stdlib/math.ts +++ b/packages/codemode/src/stdlib/math.ts @@ -3,7 +3,7 @@ import { typeError } from "../interpreter/model.js" import { Obj, coerceToNumber } from "../interpreter/objects.js" import { preserveConsumerError, withPrimitives } from "../interpreter/callback.js" import type { Interpreter } from "../interpreter/interpreter.js" -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" // Bun exposes ES2026 Math.sumPrecise before TypeScript's standard library types. declare global { diff --git a/packages/codemode/src/stdlib/object.ts b/packages/codemode/src/stdlib/object.ts index 825e48c29c7..9ccfcf3486e 100644 --- a/packages/codemode/src/stdlib/object.ts +++ b/packages/codemode/src/stdlib/object.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, methods, receiver } from "../interpreter/native.js" import { type AstNode, AsyncIteratorSymbol, invalidData, IteratorSymbol, typeError } from "../interpreter/model.js" import { diff --git a/packages/codemode/src/stdlib/string.ts b/packages/codemode/src/stdlib/string.ts index c532a890d43..26e506cc7be 100644 --- a/packages/codemode/src/stdlib/string.ts +++ b/packages/codemode/src/stdlib/string.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, fn, type Impl, type Method, methods } from "../interpreter/native.js" import { checkArrayLength, checkStringLength } from "../interpreter/limits.js" import { invalidData, IteratorSymbol, rangeError, typeError } from "../interpreter/model.js" diff --git a/packages/codemode/src/stdlib/url.ts b/packages/codemode/src/stdlib/url.ts index 1957d358116..930d7b1a664 100644 --- a/packages/codemode/src/stdlib/url.ts +++ b/packages/codemode/src/stdlib/url.ts @@ -1,4 +1,4 @@ -import type { Eval } from "../interpreter/runner.js" +import type { Eval } from "../interpreter/signals.js" import { constructor, fn, type Method, methods, prototypeFrom, receiver, requiresNew } from "../interpreter/native.js" import { IteratorSymbol, PendingThrow, typeError, uriError } from "../interpreter/model.js" import { diff --git a/packages/codemode/test/limits.test.ts b/packages/codemode/test/limits.test.ts index db5175379d5..3dbe6071d80 100644 --- a/packages/codemode/test/limits.test.ts +++ b/packages/codemode/test/limits.test.ts @@ -91,7 +91,7 @@ describe("the host stays in control", () => { }), ) expect(result).toMatchObject({ ok: true, value: 999_999 }) - // The loop takes well over 10 ms anywhere, and the runner yields to the host every millisecond; a job that kept + // The loop takes well over 10 ms anywhere, and the machine yields to the host every millisecond; a job that kept // the thread for its whole run would let the counter tick at most a couple of times around it. expect(ticks).toBeGreaterThan(10) }) diff --git a/packages/codemode/test/machine.test.ts b/packages/codemode/test/machine.test.ts index 18f117b362c..48a9c8fd375 100644 --- a/packages/codemode/test/machine.test.ts +++ b/packages/codemode/test/machine.test.ts @@ -2,59 +2,65 @@ import { describe, expect, test } from "bun:test" import { Glob } from "bun" import { Effect, Schema } from "effect" import { CodeMode, Tool } from "../src/index.js" +import { parseProgram } from "../src/interpreter/execute.js" +import { Interpreter } from "../src/interpreter/interpreter.js" +import { createBuiltins } from "../src/interpreter/intrinsics.js" +import { normal } from "../src/interpreter/signals.js" +import { Frame } from "../src/machine/frame.js" +import type { Machine } from "../src/machine/machine.js" import { stats } from "../src/machine/task.js" +import { ToolRuntime } from "../src/tool-runtime.js" -// The machine evaluator (`src/machine`) runs programs as explicit frames. These run each program on both evaluators, -// check the machine really ran it, and check both give the same result. +// The machine (`src/machine`) runs programs as explicit frames, and keeps the jobs, promises, and host calls they wait +// on as data. The Effect driver only does what the machine's stops ask. -const runtime = CodeMode.make({ - tools: { - echo: Tool.make({ - description: "Echo a value", - input: Schema.Struct({ value: Schema.Number }), - output: Schema.Number, - execute: ({ value }) => Effect.succeed(value), - }), - }, -}) - -const run = async (code: string, evaluator: "machine" | "legacy") => { - const previous = process.env.CODEMODE_EVALUATOR - process.env.CODEMODE_EVALUATOR = evaluator - try { - return await Effect.runPromise(runtime.execute(code)) - } finally { - if (previous === undefined) delete process.env.CODEMODE_EVALUATOR - else process.env.CODEMODE_EVALUATOR = previous - } +const runtimeTools = { + echo: Tool.make({ + description: "Echo a value", + input: Schema.Struct({ value: Schema.Number }), + output: Schema.Number, + execute: ({ value }) => Effect.succeed(value), + }), } -const both = async (code: string) => { - const legacy = await run(code, "legacy") - const before = stats.steps - const machine = await run(code, "machine") - expect(stats.steps).toBeGreaterThan(before) - expect(machine).toEqual(legacy) - return machine -} +const runtime = CodeMode.make({ tools: runtimeTools }) + +const run = (code: string) => Effect.runPromise(runtime.execute(code)) const value = async (code: string) => { - const result = await both(code) + const before = stats.steps + const result = await run(code) + expect(stats.steps).toBeGreaterThan(before) if (!result.ok) throw new Error(`expected success, got ${result.error.kind}: ${result.error.message}`) return result.value } const error = async (code: string) => { - const result = await both(code) + const result = await run(code) if (result.ok) throw new Error(`expected failure, got value ${JSON.stringify(result.value)}`) return result.error } -describe("machine evaluator", () => { - test("runs programs as frames, and the generator evaluator runs none", async () => { - const before = stats.steps - await run(`return 1 + 2`, "legacy") - expect(stats.steps).toBe(before) +// A machine for `code`, stepped by hand instead of by the Effect driver. +const machineFor = (code: string) => { + const parsed = parseProgram(code) + if (!parsed.ok) throw new Error(parsed.error.message) + const tools = ToolRuntime.make(ToolRuntime.prepare(runtimeTools), undefined, {}) + const ctx = new Interpreter({ tools, builtins: createBuiltins() }) + ctx.start(parsed.source) + return ctx.machine +} + +// The next stop that is not a slice. +const stop = (machine: Machine) => { + for (;;) { + const next = machine.run() + if (next.kind !== "slice") return next + } +} + +describe("machine", () => { + test("runs programs as frames", async () => { expect(await value(`const f = (n) => (n < 2 ? n : f(n - 1) + f(n - 2)); return f(12)`)).toBe(144) }) @@ -151,12 +157,66 @@ describe("machine evaluator", () => { ]) }) - test("runaway recursion stops at the same depth with the same location", async () => { + test("runaway recursion stops with the location of the call that overflowed", async () => { expect((await error(`const f = (n) => f(n + 1); return f(0)`)).message).toBe( "RangeError: Maximum call stack size exceeded (line 1, col 18)", ) }) + test("concurrent tool calls stop the machine once, with every call in one batch", () => { + const machine = machineFor(`return await Promise.all([1, 2, 3].map((value) => tools.echo({ value })))`) + const started = stop(machine) + if (started.kind !== "host") throw new Error(`expected a host stop, got ${started.kind}`) + expect( + started.calls.map((call) => ({ request: call.request, blocking: call.blocking, status: call.status })), + ).toEqual( + [1, 2, 3].map((value) => ({ + request: { kind: "tool", path: ["echo"], args: [{ value }] }, + blocking: false, + status: "inFlight", + })), + ) + // Nothing else can run until host work completes. + expect(stop(machine)).toEqual({ kind: "host", calls: [] }) + for (const [index, call] of [...started.calls.entries()].reverse()) + machine.complete(call.id, normal((index + 1) * 10)) + expect(stop(machine)).toEqual({ kind: "done", completion: normal([10, 20, 30]) }) + expect(started.calls.map((call) => call.status)).toEqual(["completed", "completed", "completed"]) + }) + + test("a blocking host call stops the job that made it until it completes", () => { + const machine = machineFor(`const found = search({ query: "echo" }); return found.items.length`) + const started = stop(machine) + if (started.kind !== "host") throw new Error(`expected a host stop, got ${started.kind}`) + expect(started.calls.map((call) => [call.request.kind, call.blocking])).toEqual([["search", true]]) + expect(stop(machine)).toEqual({ kind: "host", calls: [] }) + machine.complete(started.calls[0]!.id, normal({ items: [1, 2], remaining: 0, next: null })) + expect(stop(machine)).toEqual({ kind: "done", completion: normal(2) }) + }) + + test("await parks the frames it cuts off the stack as data on the promise", () => { + const machine = machineFor(` + const inner = async (x) => { const y = x + 1; return await new Promise(() => {}) } + const outer = async () => [await inner(1)] + return await outer() + `) + const stuck = stop(machine) + expect(stuck.kind).toBe("stuck") + const slices = [...machine.parked.keys()] + expect(slices.length).toBe(3) + for (const frames of slices) { + expect(Array.isArray(frames)).toBe(true) + for (const frame of frames) { + expect(frame).toBeInstanceOf(Frame) + expect(frame.kind.opaque).toBeUndefined() + } + } + // Each slice starts at its async boundary: the program's body, or an async function's call. + expect(slices.map((frames) => frames[0]!.kind.name).sort()).toEqual(["Call", "Call", "Program"]) + expect(machine.depth).toBe(0) + expect(machine.root.stack).toEqual([]) + }) + test("the machine does not import Effect", async () => { const files = await Array.fromAsync(new Glob("src/machine/*.ts").scan({ cwd: `${import.meta.dir}/..` })) expect(files.length).toBeGreaterThan(0) diff --git a/packages/codemode/test/promise.test.ts b/packages/codemode/test/promise.test.ts index 947e3a50d69..c9514c09b17 100644 --- a/packages/codemode/test/promise.test.ts +++ b/packages/codemode/test/promise.test.ts @@ -1344,7 +1344,7 @@ describe("promise construction", () => { }) describe("jobs run to completion", () => { - // The runner pauses a long job to let the host run; the paused job still finishes before any other job starts. + // The machine pauses a long job to let the host run; the paused job still finishes before any other job starts. test("updates to shared state between awaits are not lost", async () => { expect( await value(` diff --git a/packages/codemode/test/resumable.test.ts b/packages/codemode/test/resumable.test.ts index cf863c987ff..7e9b022b2bc 100644 --- a/packages/codemode/test/resumable.test.ts +++ b/packages/codemode/test/resumable.test.ts @@ -1,5 +1,4 @@ import { describe, expect, test } from "bun:test" -import { Context } from "effect" import { parseProgram } from "../src/interpreter/execute.js" import { Interpreter } from "../src/interpreter/interpreter.js" import { createBuiltins } from "../src/interpreter/intrinsics.js" @@ -14,7 +13,6 @@ import { nodeId, sourceOf } from "../src/interpreter/source.js" const interpreter = () => new Interpreter({ tools: ToolRuntime.make(ToolRuntime.prepare({}), undefined, {}), - services: Context.empty(), builtins: createBuiltins(), })