fix(core): honor plugin tool repairs in Code Mode (#63444)

This commit is contained in:
Aiden Cline
2026-11-12 17:44:07 -06:00
committed by GitHub
Unverified
parent 9070bff302
commit c80548e800
17 changed files with 1214 additions and 278 deletions
+14 -1
View File
@@ -94,13 +94,26 @@ runtime.execute(source) // Effect<CodeMode.Result, never, ToolServices>
```
The Effect environment is inferred from the supplied tools. `hooks` surround every call the program makes into the
host: `tool.before`/`tool.after` receive `{ name, input, key }` with decoded input and the call's `<runId>:<callId>`
host: `tool.before`/`tool.after` receive `{ name, input, key }` with the call's `<runId>:<callId>`
idempotency key, and `extension.before`/`extension.after` receive `{ extension, name, args }`. An `after` hook also
receives how the call ended (`success` with its value, `failure` with its error, or `interrupted`). A `before` hook
runs before the call reaches the host, and a failing one denies it: the host function is never called, and the program
catches the failure as a thrown error. A denied call fails the way the call itself would: a tool call or asynchronous
global rejects its promise, and a synchronous global throws.
The runtime counts a tool attempt toward `maxToolCalls` under its requested name before lookup or validation.
The optional `tool.resolve` hook can rewrite the name and input, including an unknown path. Return the original
name to keep the catalog's lookup and suggestions. The built-in `search` skips resolution. `tool.before` sees the
decoded input. `tool.after` sees the same call object even if lookup, validation, or a hook failed; in that case
it carries the last name and input reached. A veto from `tool.resolve` or `tool.before` rejects the script's call
with `ToolFailure`.
A redirect must preserve the execution mode (plain, idempotent, or durable) because the driver chooses how to
suspend the call from its requested path. Unknown paths can redirect to plain tools. For resumable calls the
runtime saves the resolved name and JSON input before schema decoding. Resume reuses a completed resolution;
if suspension interrupted an idempotent call during resolution, the hook runs again with the same key.
Detaching to suspend leaves `tool.after` pending until the call settles.
### Suspending and resuming a run
`CodeMode.make` returns a `CodeMode.Runtime`, which can also run a program step by step:
+12 -9
View File
@@ -164,8 +164,18 @@ export class Driver<R> {
call.status = "running"
const request = call.request
if (request.kind === "tool") {
const options = {
key: call.key,
admit: (name: string) => this.admit(call.id, name),
detaching: () => this.suspending,
resolved: request.resolved,
save: (name: string, input: Json | undefined) => {
const current = this.vm.call(call.id)
if (current) current.request = { kind: "tool", path: name.split("."), args: [input], resolved: true }
},
}
if (this.tools.mode(request.path)?._tag === "CodeModeDurable") {
const started = this.tools.start(request.path, request.args, call.key, (name) => this.admit(call.id, name))
const started = this.tools.start(request.path, request.args, options)
return this.fork(call.id, started, (data) => {
const current = this.vm.call(call.id)
if (current === undefined) return
@@ -175,14 +185,7 @@ export class Driver<R> {
this.wait(current)
})
}
return this.fork(
call.id,
this.tools.execute(request.path, request.args, {
key: call.key,
admit: (name) => this.admit(call.id, name),
detaching: () => this.suspending,
}),
)
return this.fork(call.id, this.tools.execute(request.path, request.args, options))
}
if (request.kind === "search") {
return this.fork(
+7 -2
View File
@@ -71,7 +71,12 @@ export type Job =
* extension function's call (see `ExtensionRequest`).
*/
export type HostRequest =
| { readonly kind: "tool"; readonly path: ReadonlyArray<string>; readonly args: Array<Json | undefined> }
| {
readonly kind: "tool"
readonly path: ReadonlyArray<string>
readonly args: Array<Json | undefined>
readonly resolved?: true
}
| { readonly kind: "search"; readonly args: Array<Json | undefined> }
| { readonly kind: "extension"; readonly call: ExtensionRequest }
@@ -110,7 +115,7 @@ export class HostCall {
readonly id: number,
/** The call's stable idempotency key, `<runId>:<callId>`. */
readonly key: string,
readonly request: HostRequest,
public request: HostRequest,
/** What runs: a tool path, or an extension function's label. */
readonly label: string,
/** The running job waits for it: its outcome continues the frame that asked, before any other job. */
+120 -86
View File
@@ -34,7 +34,7 @@ export type ToolCall = {
readonly name: string
}
/** A tool call the program is making, with its decoded input. */
/** A tool call the program is making. Host resolution updates its path and input; `tool.before` sees decoded input. */
export type ToolInvocation = {
readonly name: string
readonly input: unknown
@@ -58,8 +58,16 @@ export type CallResult =
| { readonly status: "failure"; readonly error: unknown }
| { readonly status: "interrupted" }
function callResult(exit: Exit.Exit<unknown, unknown>): CallResult {
if (Exit.isSuccess(exit)) return { status: "success", value: exit.value }
if (Cause.hasInterruptsOnly(exit.cause)) return { status: "interrupted" }
return { status: "failure", error: Cause.squash(exit.cause) }
}
/** Hooks around every call the program makes into the host. A failing `before` denies the call. */
export type Hooks<R = never> = {
/** Resolve a registered or unknown tool path before validation. Redirects must preserve the call's execution mode. */
readonly "tool.resolve"?: (call: ToolInvocation) => Effect.Effect<Pick<ToolInvocation, "name" | "input">, unknown, R>
readonly "tool.before"?: ((call: ToolInvocation) => Effect.Effect<void, unknown, R>) | undefined
readonly "tool.after"?: ((call: ToolInvocation, result: CallResult) => Effect.Effect<void, never, R>) | undefined
readonly "extension.before"?: ((call: ExtensionInvocation) => Effect.Effect<void, unknown, R>) | undefined
@@ -81,9 +89,7 @@ export const hooked = <Call, A, R>(
? run
: Effect.onExit(run, (exit) => {
if (detaching?.()) return Effect.void
if (Exit.isSuccess(exit)) return after(call, { status: "success", value: exit.value })
if (Cause.hasInterruptsOnly(exit.cause)) return after(call, { status: "interrupted" })
return after(call, { status: "failure", error: Cause.squash(exit.cause) })
return after(call, callResult(exit))
})
return before === undefined ? observed : Effect.andThen(before(call), observed)
}
@@ -525,10 +531,12 @@ const namespaceKeys = <R>(root: ToolNode<R>, path: ReadonlyArray<string>): Reado
return Array.from(node.children.keys())
}
const resolve = <R>(root: ToolNode<R>, path: ReadonlyArray<string>, index: ReadonlyArray<SearchEntry>): Tool<R> => {
const resolve = <R>(prepared: Prepared<R>, path: ReadonlyArray<string>): Tool<R> => {
const root = prepared.root
const segments = canonicalSegments(path)
const node = lookup(root, segments)
if (node === undefined) {
const index = prepared.searchIndex
const name = segments.join(".")
const ns = segments.length > 1 && root.children.has(segments[0]) ? segments[0] : undefined
const closest = rank(
@@ -568,6 +576,9 @@ export type ExecuteCallOptions = {
readonly key: string
readonly admit: (name: string) => void
readonly detaching?: (() => boolean) | undefined
/** A completed resolution saved for replay, before schema decoding. */
readonly resolved?: boolean | undefined
readonly save?: (name: string, input: Json | undefined) => void
}
/** The tool bridge of one execution. Arguments arrive and results leave as JSON; program values never enter. */
@@ -594,8 +605,7 @@ export type ToolRuntime<R = never> = {
readonly start: (
path: ReadonlyArray<string>,
args: Array<Json | undefined>,
key: string,
admit: (name: string) => void,
options: ExecuteCallOptions,
) => Effect.Effect<Json, unknown, R>
/**
* Waits for a durable tool's running work: its output, as `execute` gives it. `detaching` says whether an
@@ -630,22 +640,22 @@ export const make = <R>(
calls.push(call)
}
const decode = (name: string, tool: Tool<R>, args: Array<Json | undefined>) =>
Effect.gen(function* () {
const normalized = args.length === 0 ? [{}] : args
if (normalized.length !== 1)
throw new ToolRuntimeError("InvalidToolInput", `Tool '${name}' expects at most one input object.`)
const input = normalized[0]
const field = typeof input === "string" ? stringParameter(tool)?.name : undefined
return yield* Effect.try({
try: () => decodeToolInput(tool, field === undefined ? input : { [field]: input }),
catch: (cause) =>
new ToolRuntimeError(
"InvalidToolInput",
`Invalid input for tool '${name}': ${String(cause)}`,
name === "search" ? [] : ["The signature may have changed. Use search to get the current signature."],
),
})
const argument = (args: Array<Json | undefined>) => (args.length === 0 ? {} : args[0])
const wrap = (tool: Tool<R> | undefined, input: unknown) => {
const field = tool && typeof input === "string" ? stringParameter(tool)?.name : undefined
return field === undefined ? input : { [field]: input }
}
const decode = (name: string, tool: Tool<R>, input: unknown) =>
Effect.try({
try: () => decodeToolInput(tool, wrap(tool, input)),
catch: (cause) =>
new ToolRuntimeError(
"InvalidToolInput",
`Invalid input for tool '${name}': ${String(cause)}`,
name === "search" ? [] : ["The signature may have changed. Use search to get the current signature."],
),
})
// A tool's own failure becomes a ToolError; interruption passes through.
@@ -673,27 +683,77 @@ export const make = <R>(
catch: (cause) => new ToolRuntimeError("InvalidToolOutput", `Invalid output from tool '${name}': ${cause}`),
})
const executeTool = (name: string, tool: Tool<R>, args: Array<Json | undefined>, options: ExecuteCallOptions) =>
const invoke = (tool: Tool<R>, call: ToolInvocation) =>
Effect.gen(function* () {
if ("_tag" in tool.execute && tool.execute._tag === "CodeModeDurable") {
throw new TypeError(`Tool '${name}' is a durable tool; start it with ToolRuntime.start.`)
throw new TypeError(`Tool '${call.name}' is a durable tool; start it with ToolRuntime.start.`)
}
const run = "_tag" in tool.execute ? tool.execute.execute : tool.execute
const key = options.key
const input = yield* decode(name, tool, args)
yield* Effect.sync(() => options.admit(name))
return yield* hooked(
{ name, input, key },
hooks["tool.before"],
hooks["tool.after"],
Effect.gen(function* () {
const raw = yield* failure(Effect.suspend(() => run(input, { key })))
return yield* output(name, tool, raw)
}),
options.detaching,
)
const raw = yield* failure(Effect.suspend(() => run(call.input, { key: call.key })))
return yield* output(call.name, tool, raw)
})
const mode = (tool: Tool<R> | undefined) => (tool && "_tag" in tool.execute ? tool.execute : undefined)
const attempt = <A>(
path: ReadonlyArray<string>,
args: Array<Json | undefined>,
options: ExecuteCallOptions,
run: (tool: Tool<R>, call: ToolInvocation) => Effect.Effect<A, unknown, R>,
search = false,
) => {
const name = path.join(".")
const original = search ? makeSearchTool(prepared.searchIndex) : lookup(root, path)?.tool
const execution = mode(original)
const rewrite = search || options.resolved ? undefined : hooks["tool.resolve"]
return Effect.acquireUseRelease(
Effect.sync(() => {
options.admit(name)
const input: unknown = args.length > 1 ? args : argument(args)
return { name, key: options.key, input }
}),
(call) =>
Effect.gen(function* () {
if (args.length > 1)
throw new ToolRuntimeError("InvalidToolInput", `Tool '${name}' expects at most one input object.`)
// A replay must read the saved JSON even if a hook or the executor edits its input in place.
if (execution) call.input = structuredClone(call.input)
if (rewrite) {
const target = yield* failure(Effect.suspend(() => rewrite(call)))
call.name = target.name
call.input = target.input
}
const tool = (call.name === name ? original : undefined) ?? resolve(prepared, [call.name])
if (execution?._tag !== mode(tool)?._tag)
throw new ToolRuntimeError(
"InvalidToolInput",
`Cannot redirect '${name}' to '${call.name}' across execution modes.`,
)
if (rewrite && execution) {
if (call.input !== undefined && !Schema.is(Schema.Json)(call.input))
throw new ToolRuntimeError(
"InvalidToolInput",
`Resolved input for resumable tool '${call.name}' must be JSON.`,
)
options.save?.(call.name, structuredClone(call.input))
}
call.input = yield* decode(call.name, tool, call.input)
const before = hooks["tool.before"]
if (before) yield* failure(Effect.suspend(() => before(call)))
return yield* run(tool, call)
}),
(call, exit) => {
if (options.detaching?.()) return Effect.void
// A durable start hands settlement to wait, including after resume.
if (execution?._tag === "CodeModeDurable" && Exit.isSuccess(exit)) {
if (hooks["tool.after"]) invocations.set(call.key, call)
return Effect.void
}
return hooks["tool.after"]?.(call, callResult(exit)) ?? Effect.void
},
)
}
const durableTool = (
path: ReadonlyArray<string>,
): {
@@ -716,55 +776,31 @@ export const make = <R>(
invocations,
record: (name) => recordCall({ name }),
keys: (path) => namespaceKeys(root, path),
search: (args, options) =>
Effect.suspend(() => executeTool("search", makeSearchTool(prepared.searchIndex), args, options)),
search: (args, options) => attempt(["search"], args, options, invoke, true),
execute: (path, args, options) =>
Effect.suspend(() => {
const segments = canonicalSegments(path)
// Models often write `tools.search(...)` for the bare `search(...)`; honor it unless a tool owns that path.
if (segments.length === 1 && segments[0] === "search" && lookup(root, segments) === undefined)
return executeTool("search", makeSearchTool(prepared.searchIndex), args, options)
return executeTool(segments.join("."), resolve(root, path, prepared.searchIndex), args, options)
}),
mode: (path) => {
const segments = canonicalSegments(path)
if (segments.length === 1 && segments[0] === "search" && lookup(root, segments) === undefined) return undefined
const execute = lookup(root, segments)?.tool?.execute
return execute !== undefined && "_tag" in execute ? execute : undefined
},
start: (path, args, key, admit) =>
Effect.suspend(() => {
const found = durableTool(path)
return Effect.gen(function* () {
const input = yield* decode(found.name, found.tool, args)
yield* Effect.sync(() => admit(found.name))
const invocation: ToolInvocation = { name: found.name, input, key }
invocations.set(key, invocation)
const before = hooks["tool.before"]
const after = hooks["tool.after"]
const run = failure(Effect.suspend(() => found.durable.start(input, { key }))).pipe(
Effect.flatMap((raw) =>
Effect.try({
try: (): Json => {
const text = JSON.stringify(raw)
return text === undefined ? null : JSON.parse(text)
},
catch: (cause) =>
new ToolRuntimeError("InvalidToolOutput", `Invalid start data from tool '${found.name}': ${cause}`),
}),
),
)
const observed =
after === undefined
? run
: Effect.onError(run, (cause) => {
invocations.delete(key)
if (Cause.hasInterruptsOnly(cause)) return after(invocation, { status: "interrupted" })
return after(invocation, { status: "failure", error: Cause.squash(cause) })
})
return yield* before === undefined ? observed : Effect.andThen(before(invocation), observed)
})
const search = segments.length === 1 && segments[0] === "search" && lookup(root, segments) === undefined
return attempt(segments, args, options, invoke, search)
}),
mode: (path) => mode(lookup(root, canonicalSegments(path))?.tool),
start: (path, args, options) =>
attempt(canonicalSegments(path), args, options, (tool, call) =>
Effect.gen(function* () {
const durable = mode(tool)
if (durable?._tag !== "CodeModeDurable") throw new TypeError(`Tool '${call.name}' is not a durable tool.`)
const raw = yield* failure(Effect.suspend(() => durable.start(call.input, { key: call.key })))
return yield* Effect.try({
try: (): Json => {
const text = JSON.stringify(raw)
return text === undefined ? null : JSON.parse(text)
},
catch: (cause) =>
new ToolRuntimeError("InvalidToolOutput", `Invalid start data from tool '${call.name}': ${cause}`),
})
}),
),
wait: (path, args, key, data, detaching) =>
Effect.suspend(() => {
const found = durableTool(path)
@@ -780,7 +816,7 @@ export const make = <R>(
known !== undefined
? Effect.succeed(known)
: Effect.map(
decode(found.name, found.tool, args),
decode(found.name, found.tool, argument(args)),
(input): ToolInvocation => ({
name: found.name,
input,
@@ -791,9 +827,7 @@ export const make = <R>(
Effect.onExit(run, (exit) => {
if (detaching()) return Effect.void
invocations.delete(key)
if (Exit.isSuccess(exit)) return after(call, { status: "success", value: exit.value })
if (Cause.hasInterruptsOnly(exit.cause)) return after(call, { status: "interrupted" })
return after(call, { status: "failure", error: Cause.squash(exit.cause) })
return after(call, callResult(exit))
}),
)
}),
+77
View File
@@ -714,6 +714,83 @@ describe("resume", () => {
})
describe("idempotent and durable tools", () => {
test("suspend during idempotent resolution retries the hook under its original key", async () => {
const entered = latch()
const keys: string[] = []
const after: string[] = []
const runtime = CodeMode.make({
tools: {
read: Tool.make({ description: "Read", input: Value, output: Value, execute: Tool.idempotent(Effect.succeed) }),
},
limits: { maxToolCalls: 1 },
hooks: {
"tool.resolve": (call) => Effect.gen(function* () {
keys.push(call.key)
if (keys.length > 1) return call
entered.resolve()
return yield* Effect.never
}),
"tool.after": (_, result) => Effect.sync(() => void after.push(result.status)),
},
})
const run = background(runtime.program(`return await tools.read({ value: 7 })`, undefined, { runId: "policy" }))
await entered.promise
const saved = suspendedOf(await Effect.runPromise(run.suspend))
expect(after).toEqual([])
expect(await Effect.runPromise(background(runtime.resume(CodeMode.toBytes(saved.state))).result)).toEqual({
ok: true, value: { value: 7 }, toolCalls: [{ name: "read" }],
})
expect(keys).toEqual(["policy:0", "policy:0"])
expect(after).toEqual(["success"])
})
test("resume uses the resolved tools and inputs for replay and durable wait", async () => {
const read = idempotentManual()
const jobs = durableJobs()
const resolved: string[] = []
const after: Array<[string, unknown, string]> = []
const runtime = () =>
CodeMode.make({
tools: { oldRead: idempotentManual().tool, oldJob: durableJobs().tool, read: read.tool, job: jobs.tool },
limits: { maxToolCalls: 2 },
hooks: {
"tool.resolve": (call) =>
Effect.sync(() => {
resolved.push(call.name)
return { name: call.name === "oldRead" ? "read" : "job", input: { value: 21 } }
}),
"tool.after": (call, result) => Effect.sync(() => void after.push([call.name, call.input, result.status])),
},
})
const run = background(
runtime().program(`return await Promise.all([tools.oldRead(), tools.oldJob()])`, undefined, { runId: "rewrite" }),
)
await Promise.all([read.called(), jobs.waiting()])
const saved = suspendedOf(await Effect.runPromise(run.suspend))
expect(after).toEqual([])
read.reset()
jobs.reset()
const resumed = background(runtime().resume(CodeMode.toBytes(saved.state)))
await Promise.all([read.called(), jobs.waiting()])
expect(resolved).toEqual(["oldRead", "oldJob"])
expect(read.attempts.map(({ key, input }) => [key, input])).toEqual([
["rewrite:0", 21],
["rewrite:0", 21],
])
expect(jobs.log).toEqual(["start rewrite:1", "wait rewrite:1 rewrite:1", "wait rewrite:1 rewrite:1"])
read.attempts[1]!.answer(21)
jobs.finish("rewrite:1", 21)
expect(await Effect.runPromise(resumed.result)).toEqual({
ok: true,
value: [{ value: 21 }, { value: 21 }],
toolCalls: [{ name: "oldRead" }, { name: "oldJob" }],
})
expect(after.toSorted(([a], [b]) => a.localeCompare(b))).toEqual([
["job", { value: 21 }, "success"],
["read", { value: 21 }, "success"],
])
})
test("an idempotent tool in flight can be suspended, is interrupted without firing after, and reruns on resume with the same key", async () => {
const read = idempotentManual()
const events: Array<string> = []
+96
View File
@@ -77,6 +77,102 @@ describe("dotted tool names", () => {
})
})
describe("host resolution", () => {
test.each([false, true])(
"settles each admitted lookup, validation and veto failure (resolver: %s)",
async (resolve) => {
const settled: Array<[string, unknown, string]> = []
const runtime = CodeMode.make({
tools: {
"issues.list": Tool.make({
description: "List issues",
input: Schema.Struct({ page: Schema.NumberFromString }),
execute: () => Effect.die("denied call reached the tool"),
}),
},
hooks: {
...(resolve && { "tool.resolve": (call: CodeMode.ToolInvocation) => Effect.succeed(call) }),
"tool.before": () => Effect.fail(new Error("policy denied this call")),
"tool.after": (call, result) => Effect.sync(() => void settled.push([call.name, call.input, result.status])),
},
})
const cases = [
["issues.get_list", "{}", "UnknownTool", "Did you mean tools.issues.list?"],
["issues.list", "{ page: 1 }", "InvalidToolInput", "Expected string"],
["issues.list", "{ page: '2' }", "ToolFailure", "policy denied this call"],
] as const
for (const [name, args, kind, message] of cases) {
const result = await Effect.runPromise(runtime.execute(`await tools.${name}(${args})`))
expect(result).toMatchObject({
ok: false,
toolCalls: [{ name }],
error: { kind, message: expect.stringContaining(message) },
})
}
expect(settled).toEqual([
["issues.get_list", {}, "failure"],
["issues.list", { page: 1 }, "failure"],
["issues.list", { page: 2 }, "failure"],
])
},
)
test("a resolver veto rejects the script promise as a tool failure", async () => {
const runtime = CodeMode.make({ hooks: { "tool.resolve": () => Effect.fail(new Error("unavailable here")) } })
expect(await value(runtime, `return tools.missing().catch(e => e.message)`)).toBe("unavailable here")
expect(await failure(runtime, `await tools.missing()`)).toMatchObject({
kind: "ToolFailure",
message: "unavailable here",
})
})
test("rejects a redirect that would change how the call suspends", async () => {
const runtime = CodeMode.make({
tools: {
read: Tool.make({ description: "Read", input: Schema.Struct({}), execute: Tool.idempotent(() => Effect.void) }),
},
hooks: { "tool.resolve": () => Effect.succeed({ name: "read", input: {} }) },
})
expect(await failure(runtime, `await tools.missing()`)).toMatchObject({
kind: "InvalidToolInput",
message: "Cannot redirect 'missing' to 'read' across execution modes.",
})
})
test("reserves the last slot before an async rewrite and counts the attempted call once", async () => {
const attempts: string[] = []
const calls = new Set<CodeMode.ToolInvocation>()
const settled: string[] = []
const runtime = CodeMode.make({
tools: { available: echo("Available", "ok") },
limits: { maxToolCalls: 1 },
hooks: {
"tool.resolve": (call) =>
Effect.gen(function* () {
attempts.push(call.name)
calls.add(call)
yield* Effect.yieldNow
return { name: "available", input: call.input }
}),
"tool.before": (call) => Effect.sync(() => void calls.add(call)),
"tool.after": (call, result) =>
Effect.sync(() => {
calls.add(call)
settled.push(`${call.name}:${result.status}`)
}),
},
})
const result = await Effect.runPromise(
runtime.execute(`return await Promise.allSettled([tools.first(), tools.second()])`),
)
expect(attempts).toEqual(["first"])
expect(calls.size).toBe(1)
expect(settled).toEqual(["available:success"])
expect(result.toolCalls).toEqual([{ name: "first" }])
expect(result.ok && result.value).toMatchObject([{ status: "fulfilled", value: "ok" }, { status: "rejected" }])
})
})
describe("callable namespaces", () => {
const runtime = CodeMode.make({
tools: { issues: echo("All issues", "all"), "issues.list": echo("List issues", "list") },