diff --git a/packages/core/src/codemode/catalog.ts b/packages/core/src/codemode/catalog.ts index 07dfe59c5ad..1fd72bc5c8b 100644 --- a/packages/core/src/codemode/catalog.ts +++ b/packages/core/src/codemode/catalog.ts @@ -6,6 +6,7 @@ export type Tool = { readonly type: "tool" readonly name: string readonly description: string + /** Read only for the tools the catalog shows. */ readonly signature: string readonly pinned?: boolean } @@ -25,17 +26,19 @@ const Listing = Schema.Struct({ path: Schema.String, line: Schema.String, }) +export type Listing = typeof Listing.Type const NamespaceSummary = Schema.Struct({ name: Schema.String, description: Schema.optionalKey(Schema.String), - count: Schema.Number, + /** Every tool's path, so an update can tell a new tool from one that was not shown. */ + tools: Schema.Array(Schema.String), entries: Schema.Array(Listing), }) +export type NamespaceSummary = typeof NamespaceSummary.Type export const Summary = Schema.Struct({ total: Schema.Number, - shown: Schema.Number, namespaces: Schema.Array(NamespaceSummary), }) export type Summary = typeof Summary.Type @@ -48,8 +51,8 @@ const DESCRIPTION_LIMIT = 120 const CHARACTERS_PER_TOKEN = 4 const INLINE_BUDGET = 2_000 -// Keep every namespace visible, then select full listings one per namespace per round, -// considering shorter listings first until the inline budget is exhausted. +// Keep every namespace visible, then select full listings one per namespace per round, in path order, until the +// inline budget is exhausted. The selection depends only on the tools, and only selected signatures are rendered. export function summarize(inventory: Inventory, options: Options = {}): Summary { const budget = options.budget ?? INLINE_BUDGET const flattened = flatten(inventory.tools) @@ -61,32 +64,17 @@ export function summarize(inventory: Inventory, options: Options = {}): Summary }) .map(([name, namespaceEntries]) => { const description = flattened.namespaces.get(name)?.description - const listings = namespaceEntries - .map((entry) => { - const firstLine = entry.description.split("\n", 1)[0]?.trim() ?? "" - const description = - firstLine.length > DESCRIPTION_LIMIT ? firstLine.slice(0, DESCRIPTION_LIMIT - 3) + "..." : firstLine - const suffix = description.length === 0 ? "" : ` // ${description}` - return { path: entry.path, line: ` - ${entry.signature}${suffix}` } - }) - .toSorted((left, right) => { - if (left.path < right.path) return -1 - if (left.path > right.path) return 1 - return 0 - }) - const ranked = rankListings(listings) - const pinned = new Set( - namespaceEntries - .filter((entry) => entry.pinned) - .map((entry) => listings.find((listing) => listing.path === entry.path)) - .filter((listing) => listing !== undefined), - ) + const sorted = namespaceEntries.toSorted((left, right) => { + if (left.path < right.path) return -1 + if (left.path > right.path) return 1 + return 0 + }) return { name, ...(description === undefined ? {} : { description }), - listings, - selectionOrder: ranked.filter((candidate) => !pinned.has(candidate.listing)), - selectedListings: pinned, + tools: sorted.map((entry) => entry.path), + selectionOrder: sorted.filter((entry) => !entry.tool.pinned), + selectedListings: sorted.filter((entry) => entry.tool.pinned).map(listing), selectionIndex: 0, } }) @@ -102,54 +90,64 @@ export function summarize(inventory: Inventory, options: Options = {}): Summary namespaceLine({ name: namespace.name, ...(namespace.description === undefined ? {} : { description: namespace.description }), - count: namespace.listings.length, + tools: namespace.tools, entries: [], }), ), 0, ) - namespaces - .flatMap((namespace) => namespace.listings.filter((listing) => namespace.selectedListings.has(listing))) + .flatMap((namespace) => namespace.selectedListings) .reduce((total, listing) => total + cost(listing.line), 0) while (active.size > 0) { for (const namespace of active) { - const candidate = namespace.selectionOrder[namespace.selectionIndex] - if (!candidate || candidate.cost > remaining) { + const entry = namespace.selectionOrder[namespace.selectionIndex] + const candidate = entry && listing(entry) + if (!candidate || cost(candidate.line) > remaining) { active.delete(namespace) continue } - namespace.selectedListings.add(candidate.listing) + namespace.selectedListings.push(candidate) namespace.selectionIndex += 1 - remaining -= candidate.cost + remaining -= cost(candidate.line) if (namespace.selectionIndex === namespace.selectionOrder.length) active.delete(namespace) } } - const namespaceSummaries = namespaces.map((namespace) => ({ - name: namespace.name, - ...(namespace.description === undefined ? {} : { description: namespace.description }), - count: namespace.listings.length, - entries: namespace.listings.filter((listing) => namespace.selectedListings.has(listing)), - })) return { total: flattened.tools.length, - shown: namespaceSummaries.reduce((total, namespace) => total + namespace.entries.length, 0), - namespaces: namespaceSummaries, + namespaces: namespaces.map((namespace) => ({ + name: namespace.name, + ...(namespace.description === undefined ? {} : { description: namespace.description }), + tools: namespace.tools, + entries: namespace.selectedListings, + })), } } +/** Looks up any tool's listing as the catalog would show it. */ +export function lookup(inventory: Inventory) { + const tools = new Map(flatten(inventory.tools).tools.map((entry) => [entry.path, entry])) + return (path: string) => { + const entry = tools.get(path) + return entry === undefined ? undefined : listing(entry) + } +} + +function listing(entry: { readonly path: string; readonly tool: Tool }) { + const firstLine = entry.tool.description.split("\n", 1)[0]?.trim() ?? "" + const description = + firstLine.length > DESCRIPTION_LIMIT ? firstLine.slice(0, DESCRIPTION_LIMIT - 3) + "..." : firstLine + const suffix = description.length === 0 ? "" : ` // ${description}` + return { path: entry.path, line: ` - ${entry.tool.signature}${suffix}` } +} + function flatten(entries: ReadonlyArray, path: ReadonlyArray = []) { - const tools: Array & { readonly path: string }> = [] + const tools: Array<{ readonly path: string; readonly tool: Tool }> = [] const namespaces = new Map() for (const entry of entries) { if (entry.type === "tool") { - tools.push({ - type: "tool", - path: [...path, entry.name].join("."), - description: entry.description, - signature: entry.signature, - ...(entry.pinned === undefined ? {} : { pinned: entry.pinned }), - }) + tools.push({ path: [...path, entry.name].join("."), tool: entry }) continue } const next = [...path, entry.name] @@ -161,28 +159,13 @@ function flatten(entries: ReadonlyArray, path: ReadonlyArray) { - return listings - .map((listing) => ({ listing, cost: cost(listing.line) })) - .toSorted((left, right) => { - if (left.cost !== right.cost) return left.cost - right.cost - if (left.listing.path < right.listing.path) return -1 - if (left.listing.path > right.listing.path) return 1 - return 0 - }) -} - -function cost(text: string) { +export function cost(text: string) { return Math.round(text.length / CHARACTERS_PER_TOKEN) } diff --git a/packages/core/src/codemode/instructions.ts b/packages/core/src/codemode/instructions.ts index 2f93d261325..d368e12ac51 100644 --- a/packages/core/src/codemode/instructions.ts +++ b/packages/core/src/codemode/instructions.ts @@ -16,6 +16,14 @@ ${hasMoreTools ? `The catalog is partial. Before calling a tool that is not list ## Code Mode tools (call only inside \`execute\`)` +// Sent once, when tools arrive that the catalog cannot list in full, so the model learns about `search`. +// prettier-ignore +const searchNote = `You can also look up tools that aren't listed with the global \`search(...)\` function inside \`execute\`. Before calling a tool that is not listed, find it this way. Do not guess tool names. Each result's \`path\` is the complete call expression: write it out as code, for example \`await tools.ns.tool_name({ ... })\`. Do not use it as a key such as \`tools[path]\`. + +- ${searchSignature}` + +const UPDATE_BUDGET = 1_000 + export function render(catalog: CodeModeCatalog.Summary) { if (catalog.total === 0) return "No Code Mode tools are currently available. Later Code Mode catalog updates may add or remove tools. Do not call `execute` unless there is at least one available Code Mode tool." @@ -24,105 +32,118 @@ export function render(catalog: CodeModeCatalog.Summary) { return [CodeModeCatalog.namespaceLine(namespace), ...namespace.entries.map((entry) => entry.line)] }) - return `${prompt(catalog.shown < catalog.total)} + return `${prompt(!complete(catalog))} ${tools.join("\n")}` } -export function update(previous: CodeModeCatalog.Summary, current: CodeModeCatalog.Summary) { +// An update names what changed for the model: namespaces and tools that came or went, and signatures it was shown +// that changed, even if they are no longer shown. Shown tools shifting because other namespaces came or went is left +// out. New signatures share a fixed budget while the catalog is partial, and the rest are counted. +export function update( + previous: CodeModeCatalog.Summary, + current: CodeModeCatalog.Summary, + lookup: (path: string) => CodeModeCatalog.Listing | undefined, +) { const replacement = `The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog. ${render(current)}` - if (current.total === 0) return replacement - const previousComplete = previous.shown === previous.total - const currentComplete = current.shown === current.total - if (previousComplete !== currentComplete) return replacement + if (current.total === 0 || previous.total === 0) return replacement - const descriptions = Instructions.diffByKey( - previous.namespaces.filter((namespace) => namespace.description !== undefined), - current.namespaces.filter((namespace) => namespace.description !== undefined), + const namespaces = Instructions.diffByKey( + previous.namespaces, + current.namespaces, (namespace) => namespace.name, (before, after) => before.description !== after.description, ) - if (descriptions.added.length > 0 || descriptions.removed.length > 0 || descriptions.changed.length > 0) - return replacement - - const diff = Instructions.diffByKey( - previous.namespaces.flatMap((namespace) => namespace.entries), - current.namespaces.flatMap((namespace) => namespace.entries), - (entry) => entry.path, - (before, after) => before.line !== after.line, + // A namespace whose description changed is restated as if new. + const restated = [...namespaces.added, ...namespaces.changed.map((change) => change.current)] + const unchanged = new Set( + current.namespaces + .map((namespace) => namespace.name) + .filter((name) => !restated.some((item) => item.name === name)), ) - const entriesChanged = diff.added.length > 0 || diff.removed.length > 0 || diff.changed.length > 0 + const kept = (path: string) => unchanged.has(path.split(".", 1)[0] ?? path) + const before = new Set(previous.namespaces.flatMap((namespace) => namespace.tools)) + const after = new Set(current.namespaces.flatMap((namespace) => namespace.tools)) + const added = [...after].filter((path) => kept(path) && !before.has(path)) + const removed = [...before].filter((path) => kept(path) && !after.has(path)) + const changed = previous.namespaces + .flatMap((namespace) => namespace.entries) + .filter((entry) => kept(entry.path) && after.has(entry.path)) + .flatMap((entry) => { + const listing = lookup(entry.path) + return listing === undefined || listing.line === entry.line ? [] : [listing] + }) + const budget = complete(current) ? Infinity : UPDATE_BUDGET / (restated.length + (added.length > 0 ? 1 : 0) || 1) - if (!currentComplete) { - if (entriesChanged) return replacement - const namespaces = Instructions.diffByKey( - previous.namespaces, - current.namespaces, - (namespace) => namespace.name, - (before, after) => before.count !== after.count, - ) - const changed = namespaces.added.length > 0 || namespaces.removed.length > 0 || namespaces.changed.length > 0 - if (!changed) return replacement - - const parts = ["The Code Mode tool catalog has changed."] - if (namespaces.added.length > 0) { - parts.push( - `New tool namespaces are available: ${namespaces.added - .map((namespace) => `\`${namespace.name}\` (${namespace.count} tools)`) - .join(", ")}.`, - ) - } - if (namespaces.changed.length > 0) { - parts.push( - `The following namespace inventories changed; search them again before relying on previous results: ${namespaces.changed - .map((change) => `\`${change.current.name}\` now has ${change.current.count} tools`) - .join(", ")}.`, - ) - } - if (namespaces.removed.length > 0) { - parts.push( - `The following tool namespaces are no longer available and must not be used: ${namespaces.removed - .map((namespace) => `\`${namespace.name}\``) - .join(", ")}.`, - ) - } - const delta = parts.join("\n\n") - if (delta.length < replacement.length) return delta - return replacement - } - - if (!entriesChanged) return replacement const parts = ["The Code Mode tool catalog has changed."] - if (diff.added.length > 0) { + if (restated.length > 0) { + parts.push( + [ + "New tools are available:", + ...restated.map((namespace) => { + const shown = fit( + namespace.tools.flatMap((path) => lookup(path) ?? []), + budget, + ) + return [CodeModeCatalog.namespaceLine(namespace, shown.length), ...shown.map((entry) => entry.line)].join( + "\n", + ) + }), + ].join("\n"), + ) + } + if (added.length > 0) { + const shown = fit( + added.flatMap((path) => lookup(path) ?? []), + budget, + ) + const rest = added.length - shown.length parts.push( [ "New tools are available in addition to those previously listed:", - ...diff.added.map((entry) => entry.line), + ...shown.map((entry) => entry.line), + ...(rest === 0 ? [] : [` - ${rest} more; search for them`]), ].join("\n"), ) } - if (diff.changed.length > 0) { + if (complete(previous) && !complete(current)) parts.push(searchNote) + if (changed.length > 0) { parts.push( - [ - "Changed tool listings supersede the previously listed ones:", - ...diff.changed.map((change) => change.current.line), - ].join("\n"), + ["Changed tool listings supersede the previously listed ones:", ...changed.map((entry) => entry.line)].join("\n"), ) } - if (diff.removed.length > 0) { + if (removed.length > 0) { parts.push( - `The following tools are no longer available and must not be called: ${diff.removed - .map((entry) => toolExpression(entry.path)) + `The following tools are no longer available and must not be called: ${removed.map(toolExpression).join(", ")}.`, + ) + } + if (namespaces.removed.length > 0) { + parts.push( + `The following tool namespaces are no longer available and must not be used: ${namespaces.removed + .map((namespace) => `\`${namespace.name}\``) .join(", ")}.`, ) } + if (parts.length === 1) return undefined const delta = parts.join("\n\n") if (delta.length < replacement.length) return delta return replacement } +function complete(catalog: CodeModeCatalog.Summary) { + return catalog.namespaces.every((namespace) => namespace.entries.length === namespace.tools.length) +} + +/** The leading listings whose cost fits `budget`. */ +function fit(listings: ReadonlyArray, budget: number) { + const totals: number[] = [] + for (const entry of listings) totals.push((totals.at(-1) ?? 0) + CodeModeCatalog.cost(entry.line)) + const end = totals.findIndex((total) => total > budget) + return end === -1 ? listings : listings.slice(0, end) +} + const key = Instructions.Key.make("core/codemode") const codec = Schema.toCodecJson(CodeModeCatalog.Summary) @@ -134,7 +155,9 @@ export const make = (inventory?: CodeModeCatalog.Inventory): Instructions.List = read: Effect.succeed(catalog), render: { initial: render, - changed: update, + // Rendered once, against the inventory that produced `current`, and stored with the update. + changed: (previous, current) => + update(previous, current, inventory === undefined ? () => undefined : CodeModeCatalog.lookup(inventory)), removed: () => "Code Mode tools are no longer available. Do not use any previously listed Code Mode tools.", }, }) diff --git a/packages/core/src/codemode/tool.ts b/packages/core/src/codemode/tool.ts index c63142d07ef..1030ab1d95c 100644 --- a/packages/core/src/codemode/tool.ts +++ b/packages/core/src/codemode/tool.ts @@ -232,7 +232,10 @@ export const catalog = (inventory: Inventory) => { type: "tool", name: tool.path.split(".").at(-1) ?? tool.path, description: tool.description, - signature: tool.signature, + // Rendered only for the tools the catalog shows. + get signature() { + return tool.signature + }, pinned: pinned.has(tool.path), } return { diff --git a/packages/core/test/codemode/catalog.test.ts b/packages/core/test/codemode/catalog.test.ts index fe483102a60..97d17e47eed 100644 --- a/packages/core/test/codemode/catalog.test.ts +++ b/packages/core/test/codemode/catalog.test.ts @@ -27,19 +27,22 @@ const update = ( CodeModeInstructions.update( CodeModeCatalog.summarize({ tools: previous }, budget === undefined ? {} : { budget }), CodeModeCatalog.summarize({ tools: current }, budget === undefined ? {} : { budget }), + CodeModeCatalog.lookup({ tools: current }), ) +const shown = (catalog: CodeModeCatalog.Summary) => + catalog.namespaces.reduce((total, namespace) => total + namespace.entries.length, 0) + describe("CodeModeCatalog.summarize", () => { test("retains namespace inventory without retaining tools outside the inline budget", () => { const catalog = CodeModeCatalog.summarize( { tools: Array.from({ length: 10_000 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`)) }, { budget: 0 }, ) - expect(catalog).toEqual({ - total: 10_000, - shown: 0, - namespaces: [{ name: "bulk", count: 10_000, entries: [] }], - }) + expect(catalog.total).toBe(10_000) + expect(catalog.namespaces).toHaveLength(1) + expect(catalog.namespaces[0]?.tools).toHaveLength(10_000) + expect(catalog.namespaces[0]?.entries).toEqual([]) }) test("retains every namespace when no full tool listing fits", () => { @@ -58,7 +61,7 @@ describe("CodeModeCatalog.summarize", () => { { budget: 0 }, ) - expect(catalog.shown).toBe(2) + expect(shown(catalog)).toBe(2) expect(catalog.namespaces.flatMap((namespace) => namespace.entries.map((item) => item.path))).toEqual([ "alpha.first", "beta.second", @@ -71,17 +74,22 @@ describe("CodeModeCatalog.summarize", () => { const pinCost = Math.round(` - ${pinned.signature} // Pinned`.length / 4) const unpinnedCost = Math.round(` - ${unpinned.signature} // Unpinned`.length / 4) const namespaceCost = [ - { name: "alpha", count: 1, entries: [] }, - { name: "beta", count: 1, entries: [] }, + { name: "alpha", tools: ["alpha.pinned"], entries: [] }, + { name: "beta", tools: ["beta.unpinned"], entries: [] }, ].reduce((total, namespace) => total + Math.round(CodeModeCatalog.namespaceLine(namespace).length / 4), 0) expect( - CodeModeCatalog.summarize({ tools: [pinned, unpinned] }, { budget: namespaceCost + pinCost + unpinnedCost }) - .shown, + shown( + CodeModeCatalog.summarize({ tools: [pinned, unpinned] }, { budget: namespaceCost + pinCost + unpinnedCost }), + ), ).toBe(2) expect( - CodeModeCatalog.summarize({ tools: [pinned, unpinned] }, { budget: namespaceCost + pinCost + unpinnedCost - 1 }) - .shown, + shown( + CodeModeCatalog.summarize( + { tools: [pinned, unpinned] }, + { budget: namespaceCost + pinCost + unpinnedCost - 1 }, + ), + ), ).toBe(1) }) @@ -102,16 +110,42 @@ describe("CodeModeCatalog.summarize", () => { test("always retains namespace descriptions and charges them before tool listings", () => { const tool = entry("alpha.one", "One") const listingCost = Math.round(` - ${tool.signature} // One`.length / 4) - const namespaceCost = Math.round(CodeModeCatalog.namespaceLine({ name: "alpha", count: 1, entries: [] }).length / 4) + const namespaceCost = Math.round( + CodeModeCatalog.namespaceLine({ name: "alpha", tools: ["alpha.one"], entries: [] }).length / 4, + ) const description = "A namespace description that stays visible beyond the available tool budget" const namespace = { type: "namespace" as const, name: "alpha", description, tools: [tool] } - expect(CodeModeCatalog.summarize({ tools: [tool] }, { budget: namespaceCost + listingCost }).shown).toBe(1) + expect(shown(CodeModeCatalog.summarize({ tools: [tool] }, { budget: namespaceCost + listingCost }))).toBe(1) const catalog = CodeModeCatalog.summarize({ tools: [namespace] }, { budget: namespaceCost + listingCost }) - expect(catalog.shown).toBe(0) + expect(shown(catalog)).toBe(0) expect(catalog.namespaces[0]?.description).toBe(description) expect(CodeModeInstructions.render(catalog)).toContain(`- alpha (1 tool, none shown) // ${description}`) }) + + test("selects tools in path order, whatever their cost", () => { + const costly = entry("alpha.a", "A", `tools.alpha.a({\n ${"long".repeat(40)}: string,\n}): Promise`) + const catalog = CodeModeCatalog.summarize({ tools: [entry("alpha.b", "B"), costly] }, { budget: 70 }) + expect(catalog.namespaces[0]?.entries.map((item) => item.path)).toEqual(["alpha.a"]) + }) + + test("renders only the signatures it selects, and the next one it tries", () => { + const rendered = new Set() + const tools = Array.from({ length: 500 }, (_, index): CodeModeCatalog.Tool => { + const path = `bulk.tool${String(index).padStart(3, "0")}` + return { + type: "tool", + name: path, + description: path, + get signature() { + rendered.add(path) + return `tools.${path}({ q: string }): Promise` + }, + } + }) + const catalog = CodeModeCatalog.summarize({ tools }, { budget: 100 }) + expect(rendered.size).toBe(shown(catalog) + 1) + }) }) describe("CodeModeInstructions.render", () => { @@ -154,8 +188,8 @@ describe("CodeModeInstructions.render", () => { // Round 1 places alpha.cheap and beta.cheap; in round 2 alpha.expensive does not fit, // which marks only alpha done - it must NOT prevent other namespaces from inlining. const namespaceCost = [ - { name: "alpha", count: 2, entries: [] }, - { name: "beta", count: 1, entries: [] }, + { name: "alpha", tools: ["alpha.cheap", "alpha.expensive"], entries: [] }, + { name: "beta", tools: ["beta.cheap"], entries: [] }, ].reduce((total, namespace) => total + Math.round(CodeModeCatalog.namespaceLine(namespace).length / 4), 0) const instructions = render([cheapAlpha, expensive, cheapBeta], 40 + namespaceCost) expect(instructions).toContain("The catalog is partial.") @@ -192,61 +226,93 @@ describe("CodeModeInstructions.update", () => { const added = entry("notes.list", "List notes") const unchanged = Array.from({ length: 5 }, (_, index) => entry(`stable.tool${index}`, `Stable ${index}`)) const text = update([echo, lookup, ...unchanged], [changed, added, ...unchanged]) - expect(text).toContain("The Code Mode tool catalog has changed.") - expect(text).toContain(`New tools are available in addition to those previously listed:\n - ${added.signature}`) - expect(text).toContain( - `Changed tool listings supersede the previously listed ones:\n - ${changed.signature} // Echo text`, + expect(text).toBe( + [ + "The Code Mode tool catalog has changed.", + `New tools are available in addition to those previously listed:\n - ${added.signature} // List notes`, + `Changed tool listings supersede the previously listed ones:\n - ${changed.signature} // Echo text`, + "The following tool namespaces are no longer available and must not be used: `orders`.", + ].join("\n\n"), ) - expect(text).toContain("The following tools are no longer available and must not be called: tools.orders.lookup.") - expect(text).not.toContain("## Code Mode tools (call only inside `execute`)") }) test("names removed tools with exact callable expressions including bracket notation", () => { const dashed = entry("context7.resolve-library-id", "Resolve a library ID") - const text = update([echo, dashed], [echo]) - expect(text).toContain( + const kept = entry("context7.query-docs", "Query docs") + expect(update([echo, dashed, kept], [echo, kept])).toContain( 'The following tools are no longer available and must not be called: tools.context7["resolve-library-id"].', ) }) - test("restates namespace descriptions when they change", () => { - const previous = CodeModeCatalog.summarize({ - tools: [{ type: "namespace", name: "notes", description: "Old description", tools: [echo] }], - }) - const current = CodeModeCatalog.summarize({ - tools: [{ type: "namespace", name: "notes", description: "New description", tools: [echo] }], - }) - const text = CodeModeInstructions.update(previous, current) - expect(text).toContain("This catalog supersedes the previous Code Mode tool catalog.") - expect(text).toContain("- notes (1 tool) // New description") + test("restates a namespace whose description changed", () => { + const text = update( + [{ type: "namespace", name: "notes", description: "Old description", tools: [echo] }], + [{ type: "namespace", name: "notes", description: "New description", tools: [echo] }], + ) + expect(text).toContain("New tools are available:\n- notes (1 tool) // New description") expect(text).not.toContain("Old description") }) - test("restates the full catalog when the rendering mode crosses full and compact", () => { - const wide = Array.from({ length: 40 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`)) - const text = update([echo], [echo, ...wide], 30) - expect(text).toContain( - "The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.", + test("says nothing when only which tools are shown changes", () => { + const alpha = Array.from({ length: 20 }, (_, index) => entry(`alpha.tool${index}`, `Tool ${index}`)) + const text = update(alpha, [...alpha, entry("beta.one", "One")], 120) + expect(text).toStartWith("The Code Mode tool catalog has changed.\n\nNew tools are available:\n- beta (1 tool") + expect(text).not.toContain("alpha") + expect(update([echo], [{ ...echo, pinned: true }])).toBeUndefined() + }) + + test("shows a new tool's signature even when the catalog would not", () => { + const alpha = Array.from({ length: 10 }, (_, index) => entry(`alpha.tool${index}`, `Tool ${index}`)) + expect(update(alpha, [...alpha, entry("alpha.tool10", "Tool 10")], 0)).toContain( + "New tools are available in addition to those previously listed:\n - tools.alpha.tool10({", ) - expect(text).toContain("The catalog is partial.") - expect(text).toContain("## Code Mode tools (call only inside `execute`)") + }) + + test("counts new tools past the update budget", () => { + const bulk = Array.from({ length: 300 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`)) + expect(update([bulk[0]!], bulk)).toMatch(/ {2}- \d+ more; search for them\n/) + }) + + test("shows a changed signature the model was shown even when the tool is no longer shown", () => { + const before = Array.from({ length: 200 }, (_, index) => entry(`ns.tool${String(index).padStart(3, "0")}`, "Tool")) + const last = CodeModeCatalog.summarize({ tools: before }).namespaces[0]!.entries.at(-1)!.path + const signature = `tools.${last}({\n key: string,\n}): Promise` + // An arriving namespace takes the room the tool was shown in, in the same step its signature changes. + const arriving = Array.from({ length: 40 }, (_, index) => entry(`aa.tool${index}`, "Tool")) + const after = [...arriving, ...before.map((tool) => (tool.name === last ? { ...tool, signature } : tool))] + const current = CodeModeCatalog.summarize({ tools: after }).namespaces.find((namespace) => namespace.name === "ns") + expect(current?.entries.map((item) => item.path)).not.toContain(last) + expect(update(before, after)).toContain( + `Changed tool listings supersede the previously listed ones:\n - ${signature} // Tool`, + ) + }) + + test("shows every new tool, without pointing to search, while the catalog lists every tool", () => { + const signature = (name: string) => + `tools.wide.${name}({\n /** ${"A long parameter description. ".repeat(4)} */\n q: string,\n}): Promise` + const wide = Array.from({ length: 30 }, (_, index) => + entry(`wide.tool${index}`, `Tool ${index}`, signature(`tool${index}`)), + ) + const text = update([echo], [echo, ...wide])! + expect(CodeModeCatalog.cost(text)).toBeGreaterThan(1_000) + expect(text).toContain("- wide (30 tools)\n") + expect(text).not.toContain("search") + }) + + test("introduces search once, when the catalog stops listing every tool", () => { + const wide = Array.from({ length: 200 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`)) + const text = update([echo], [echo, ...wide]) + expect(text).toStartWith("The Code Mode tool catalog has changed.\n\nNew tools are available:\n- bulk (200 tools") + expect(text).toContain("You can also look up tools that aren't listed with the global `search(...)` function") + expect(text).toContain("- search({") + expect(update([echo, ...wide], [echo, ...wide, entry("bulk.more", "More")])).not.toContain("search(") + expect(update([echo, ...wide], [echo])).not.toContain("search(") }) test("falls back to full replacement when the delta is larger than the catalog", () => { - const previous = Array.from({ length: 200 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`)) - const text = update([...previous, echo], [echo]) + const removed = Array.from({ length: 200 }, (_, index) => entry(`a_long_removed_namespace_${index}.tool`, "Tool")) + const text = update([...removed, echo], [echo]) expect(text).toContain("This catalog supersedes the previous Code Mode tool catalog.") expect(text).toContain("## Code Mode tools (call only inside `execute`)") - expect(text).toContain("The catalog is complete.") - expect(text).not.toContain("The following tools are no longer available") - }) - - test("renders namespace-only deltas without persisting hidden tool entries", () => { - const alpha = Array.from({ length: 10 }, (_, index) => entry(`alpha.tool${index}`, `Tool ${index}`)) - const text = update(alpha, [...alpha, entry("alpha.tool10", "Tool 10")], 0) - expect(text).toContain("`alpha` now has 11 tools") - expect(text).toContain("search them again before relying on previous results") - expect(text).not.toContain("tools.alpha.tool10(input:") - expect(text).not.toContain("## Code Mode tools (call only inside `execute`)") }) }) diff --git a/packages/core/test/codemode/instructions.test.ts b/packages/core/test/codemode/instructions.test.ts index f128b8ac0d9..58c62af6054 100644 --- a/packages/core/test/codemode/instructions.test.ts +++ b/packages/core/test/codemode/instructions.test.ts @@ -31,8 +31,9 @@ describe("CodeModeInstructions", () => { "No Code Mode tools are currently available. Later Code Mode catalog updates may add or remove tools. Do not call `execute` unless there is at least one available Code Mode tool.", ) + // The empty notice never explained Code Mode, so the first tools restate the whole catalog. const added = yield* readUpdate(CodeModeInstructions.make({ tools: [echo] }), initialized) - expect(added.text).toContain("New tools are available in addition to those previously listed:") + expect(added.text).toContain("This catalog supersedes the previous Code Mode tool catalog.") expect(added.text).toContain(echo.signature) expect(yield* readUpdate(CodeModeInstructions.make({ tools: [] }), { values: added.values })).toMatchObject({ @@ -54,15 +55,14 @@ describe("CodeModeInstructions", () => { expect(initialized.text).toContain(` - ${echo.signature} // Echo text`) const added = yield* readUpdate(CodeModeInstructions.make({ tools: [echo, lookup] }), initialized) - expect(added.text).toContain("The Code Mode tool catalog has changed.") - expect(added.text).toContain("New tools are available in addition to those previously listed:") - expect(added.text).toContain(` - ${lookup.signature} // Look up an order`) - expect(added.text).not.toContain("## Code Mode tools (call only inside `execute`)") + expect(added.text).toBe( + `The Code Mode tool catalog has changed.\n\nNew tools are available:\n- orders (1 tool)\n - ${lookup.signature} // Look up an order`, + ) const removed = yield* readUpdate(CodeModeInstructions.make({ tools: [echo] }), { values: added.values }) expect(removed.text).toBe( "The Code Mode tool catalog has changed.\n\n" + - "The following tools are no longer available and must not be called: tools.orders.lookup.", + "The following tool namespaces are no longer available and must not be used: `orders`.", ) expect(yield* readUpdate(CodeModeInstructions.make(), initialized)).toMatchObject({ diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index e3499f5dfda..ac585ac8f6d 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -88,7 +88,7 @@ import { permissionLayer } from "./lib/permission" import { agentHost, modelHost, host, noProviders } from "./plugin/host" import { CodeModeInstructions } from "@opencode/core/codemode/instructions" -const emptyCodeMode = `${CodeModeInstructions.render({ total: 0, shown: 0, namespaces: [] })}\n\n` +const emptyCodeMode = `${CodeModeInstructions.render({ total: 0, namespaces: [] })}\n\n` type ToolBarrier = { readonly count: number readonly started: Deferred.Deferred @@ -2382,6 +2382,75 @@ describe("SessionRunnerLLM", () => { ]) }) + scenario("narrates only the Code Mode tools that change, through compaction and fork", function* (s) { + const registry = yield* Tool.Service + const register = (name: string, count: number, from = 0) => + Effect.gen(function* () { + const scope = yield* Scope.make() + const tools = Object.fromEntries( + Array.from({ length: count }, (_, offset) => [ + `${name}_${from + offset}`, + { + name: `${name}_${from + offset}`, + description: `${name} tool ${from + offset}`, + input: Schema.Struct({ id: Schema.String }), + output: Schema.String, + execute: () => Effect.succeed({ output: "ok" }), + } satisfies ToolInfo, + ]), + ) + yield* transformTools(registry, tools, { namespace: name }).pipe(Scope.provide(scope)) + return scope + }) + const catalogOf = (request: LLMRequest | undefined) => + request?.system.map((part) => part.text).find((text) => text.includes("# Code Mode")) ?? "" + const updates = (request: LLMRequest | undefined) => + request ? systemTexts(request).filter((text) => text.startsWith("The Code Mode tool catalog has changed")) : [] + const signature = (name: string, index: number) => + ` - tools.${name}.${name}_${index}({\n id: string,\n}): Promise // ${name} tool ${index}` + + yield* register("alpha", 12) + yield* s.runPrompt("First") + expect(catalogOf(s.requests.at(-1))).toContain(`- alpha (12 tools)\n${signature("alpha", 0)}`) + + const beta = yield* register("beta", 1) + yield* s.runPrompt("Second") + expect(updates(s.requests.at(-1))).toEqual([ + `The Code Mode tool catalog has changed.\n\nNew tools are available:\n- beta (1 tool)\n${signature("beta", 0)}`, + ]) + // The next step rebuilds the same catalog and has nothing to say. + yield* s.runPrompt("Again") + expect(updates(s.requests.at(-1))).toHaveLength(1) + + yield* Scope.close(beta, Exit.void) + yield* s.runPrompt("Third") + expect(updates(s.requests.at(-1)).at(-1)).toBe( + "The Code Mode tool catalog has changed.\n\nThe following tool namespaces are no longer available and must not be used: `beta`.", + ) + + yield* register("alpha", 1, 12) + yield* s.runPrompt("Fourth") + expect(updates(s.requests.at(-1)).at(-1)).toBe( + `The Code Mode tool catalog has changed.\n\nNew tools are available in addition to those previously listed:\n${signature("alpha", 12)}`, + ) + + // Compaction restates the current catalog as the new baseline and has no update to send. + yield* s.llm.push(TestLLM.text("## Objective\n- summary", "catalog-summary")) + yield* s.session.compact({ sessionID }) + yield* s.resume + const latest = yield* s.runPrompt("Fifth") + expect(catalogOf(s.requests.at(-1))).toContain(signature("alpha", 12)) + expect(catalogOf(s.requests.at(-1))).not.toContain("- beta") + expect(updates(s.requests.at(-1))).toEqual([]) + + // A fork starts from the same catalog and has nothing to narrate. + const forked = yield* s.session.fork({ sessionID, before: latest.id }) + yield* s.session.prompt({ sessionID: forked.id, text: "Forked", resume: false }) + yield* s.session.resume(forked.id) + expect(catalogOf(s.requests.at(-1))).toContain(signature("alpha", 12)) + expect(updates(s.requests.at(-1))).toEqual([]) + }) + scenario("moves the epoch at compaction and narrates later changes", function* (s) { yield* s.runPrompt("First") s.systemBaseline = "Changed before compaction"