mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
feat(codemode): rank tool search with name-weighted BM25 (#62984)
This commit is contained in:
committed by
GitHub
Unverified
parent
be6f5a367e
commit
59ed43f0b8
@@ -204,44 +204,227 @@ export type Prepared<R = never> = {
|
||||
|
||||
export type SearchEntry = {
|
||||
readonly description: ToolDescription
|
||||
/** The path split into words, so `zones` matches `get_zones` as a word rather than as a substring of `timezones`. */
|
||||
readonly pathWords: ReadonlyArray<string>
|
||||
readonly searchText: string
|
||||
/** Position in the full search index, for scoring through the word index. */
|
||||
readonly position: number
|
||||
/** The tool's searchable text by field, read when the word index is built. */
|
||||
readonly text: () => Readonly<Record<SearchField, ReadonlyArray<string>>>
|
||||
/** Words in the tool's own name (the last path segment). */
|
||||
readonly nameLength: number
|
||||
}
|
||||
|
||||
const tokenize = (query: string): Array<string> =>
|
||||
query
|
||||
type SearchField = "namespace" | "name" | "description" | "parameters" | "namespaceDescription"
|
||||
|
||||
// Standard field-weighted BM25 (k1 1.2, b 0.75) plus a preference for short names. Compared against other rankers on
|
||||
// model-written and real search queries over an ~11k-tool catalog (anomalyco/evals, dumber/evals/tool-search).
|
||||
const fieldWeights: Record<SearchField, number> = {
|
||||
name: 3,
|
||||
namespace: 2,
|
||||
description: 1,
|
||||
namespaceDescription: 0.4,
|
||||
parameters: 0.3,
|
||||
}
|
||||
const searchFields = Object.keys(fieldWeights) as ReadonlyArray<SearchField>
|
||||
const k1 = 1.2
|
||||
const lengthNormalization = 0.75
|
||||
/** score × (1 + brevityBoost / words in the tool name): "users" prefers `list_users` over `list_user_groups`. */
|
||||
const brevityBoost = 0.5
|
||||
/** A query word of 4+ letters also matches longer words it starts with, at this weight: `term` → `terminal`. */
|
||||
const prefixWeight = 0.3
|
||||
|
||||
/** Words split at case changes and non-alphanumerics: `listUserGroups` → list, user, groups. */
|
||||
const splitWords = (text: string): Array<string> =>
|
||||
text
|
||||
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/)
|
||||
.filter((term) => term.length > 0 && term !== "*")
|
||||
.filter((word) => word.length > 0)
|
||||
|
||||
const termForms = (term: string): Array<string> => {
|
||||
const forms = [term]
|
||||
if (term.endsWith("es") && term.length > 3) forms.push(term.slice(0, -2))
|
||||
if (term.endsWith("s") && term.length > 2) forms.push(term.slice(0, -1))
|
||||
return forms
|
||||
/** Plural endings only, applied the same everywhere: `settings` → `setting`, `statuses` → `status`, `vms` → `vm`. */
|
||||
const stem = (word: string) => {
|
||||
if (word.length > 4 && word.endsWith("ies")) return word.slice(0, -3) + "y"
|
||||
if (word.length > 4 && /(?:ss|sh|ch|x|z|us)es$/.test(word)) return word.slice(0, -2)
|
||||
if (word.length > 2 && word.endsWith("s") && !/(?:ss|us|is)$/.test(word)) return word.slice(0, -1)
|
||||
return word
|
||||
}
|
||||
|
||||
const rank = (entries: ReadonlyArray<SearchEntry>, query: string): Array<SearchEntry> => {
|
||||
const terms = tokenize(query).map(termForms)
|
||||
return entries
|
||||
.map((entry) => {
|
||||
const path = entry.description.path.toLowerCase()
|
||||
const description = entry.description.description.toLowerCase()
|
||||
const score = terms.reduce(
|
||||
(total, forms) =>
|
||||
total +
|
||||
(forms.some((form) => path === form || path.endsWith(`.${form}`)) ? 20 : 0) +
|
||||
(forms.some((form) => entry.pathWords.includes(form)) ? 12 : 0) +
|
||||
(forms.some((form) => path.includes(form)) ? 8 : 0) +
|
||||
(forms.some((form) => description.includes(form)) ? 4 : 0) +
|
||||
(forms.some((form) => entry.searchText.includes(form)) ? 2 : 0),
|
||||
0,
|
||||
)
|
||||
return { entry, score }
|
||||
})
|
||||
.filter(({ score }) => terms.length === 0 || score > 0)
|
||||
/** Searchable words of a text: its split words, plus each camelCase word whole (`CronJob` → cron, job, cronjob). */
|
||||
const searchWords = (text: string) =>
|
||||
splitWords(text)
|
||||
.concat((text.match(/[A-Za-z0-9]*[a-z0-9][A-Z][A-Za-z0-9]*/g) ?? []).map((word) => word.toLowerCase()))
|
||||
.map(stem)
|
||||
|
||||
const fieldWords = (text: string, cache: Map<string, ReadonlyArray<string>>) => {
|
||||
const cached = cache.get(text)
|
||||
if (cached) return cached
|
||||
const words = searchWords(text)
|
||||
cache.set(text, words)
|
||||
return words
|
||||
}
|
||||
|
||||
/** For each word, the tools containing it and their field-weighted, length-normalized term frequency. */
|
||||
type WordIndex = {
|
||||
readonly postings: ReadonlyMap<string, { readonly positions: Int32Array; readonly weights: Float32Array }>
|
||||
readonly documents: number
|
||||
/** Sorted indexed words, for partial-word matching. */
|
||||
readonly words: ReadonlyArray<string>
|
||||
}
|
||||
|
||||
// Built once per search index on the first query.
|
||||
const wordIndexes = new WeakMap<ReadonlyArray<SearchEntry>, WordIndex>()
|
||||
|
||||
// Locations with the same tools (global MCP servers) share one word index, keyed by a hash of the indexed text. An
|
||||
// index lives while some search index uses it; once none does it can be collected and its key is removed. A hash
|
||||
// collision (about 1 in 37 trillion with 1,000 distinct catalogs alive) would only misrank results: they still come
|
||||
// from the caller's own tools, and the tool count in the key keeps positions in range.
|
||||
const shared =
|
||||
typeof WeakRef === "undefined" || typeof FinalizationRegistry === "undefined"
|
||||
? undefined
|
||||
: {
|
||||
indexes: new Map<string, WeakRef<WordIndex>>(),
|
||||
cleanup: new FinalizationRegistry<{ key: string; ref: WeakRef<WordIndex> }>(({ key, ref }) => {
|
||||
// A newer index may have been stored under the same key since this one was dropped.
|
||||
if (shared?.indexes.get(key) === ref) shared.indexes.delete(key)
|
||||
}),
|
||||
}
|
||||
|
||||
const wordIndexFor = (index: ReadonlyArray<SearchEntry>): WordIndex => {
|
||||
const cached = wordIndexes.get(index)
|
||||
if (cached) return cached
|
||||
const texts = index.map((entry) => entry.text())
|
||||
const key = shared ? textKey(texts) : undefined
|
||||
const reused = key === undefined ? undefined : shared?.indexes.get(key)?.deref()
|
||||
if (reused) {
|
||||
wordIndexes.set(index, reused)
|
||||
return reused
|
||||
}
|
||||
const built = buildWordIndex(texts)
|
||||
wordIndexes.set(index, built)
|
||||
if (shared && key !== undefined) {
|
||||
const ref = new WeakRef(built)
|
||||
shared.indexes.set(key, ref)
|
||||
shared.cleanup.register(built, { key, ref })
|
||||
}
|
||||
return built
|
||||
}
|
||||
|
||||
/** Tool count plus two independent 32-bit FNV-1a hashes of every indexed field, in index order. */
|
||||
const textKey = (texts: ReadonlyArray<Readonly<Record<SearchField, ReadonlyArray<string>>>>) => {
|
||||
let first = 0x811c9dc5
|
||||
let second = 0x9e3779b9
|
||||
const add = (text: string) => {
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const code = text.charCodeAt(i)
|
||||
first = Math.imul(first ^ code, 0x01000193)
|
||||
second = Math.imul(second ^ code, 0x5bd1e995)
|
||||
second ^= second >>> 15
|
||||
}
|
||||
// Separators keep field boundaries distinct: ["ab", "c"] and ["a", "bc"] hash differently.
|
||||
first = Math.imul(first ^ 0x1f, 0x01000193)
|
||||
second = Math.imul(second ^ 0x1f, 0x5bd1e995)
|
||||
}
|
||||
for (const text of texts)
|
||||
for (const field of searchFields) {
|
||||
for (const part of text[field]) add(part)
|
||||
add("\u0000")
|
||||
}
|
||||
return `${texts.length}:${(first >>> 0).toString(36)}:${(second >>> 0).toString(36)}`
|
||||
}
|
||||
|
||||
const buildWordIndex = (texts: ReadonlyArray<Readonly<Record<SearchField, ReadonlyArray<string>>>>): WordIndex => {
|
||||
// Field texts are split once here and dropped afterwards; descriptions repeat across tools, so each text is split
|
||||
// once and shared while building.
|
||||
const cache = new Map<string, ReadonlyArray<string>>()
|
||||
const words = (texts: ReadonlyArray<string>) => texts.flatMap((text) => fieldWords(text, cache))
|
||||
const fields = texts.map((text) => {
|
||||
return {
|
||||
namespace: words(text.namespace),
|
||||
name: words(text.name),
|
||||
description: words(text.description),
|
||||
parameters: words(text.parameters),
|
||||
namespaceDescription: words(text.namespaceDescription),
|
||||
}
|
||||
})
|
||||
const averageLength = Object.fromEntries(
|
||||
searchFields.map((field) => [
|
||||
field,
|
||||
fields.reduce((sum, entry) => sum + entry[field].length, 0) / texts.length || 1,
|
||||
]),
|
||||
) as Record<SearchField, number>
|
||||
const building = new Map<string, { positions: Array<number>; weights: Array<number> }>()
|
||||
const weights = new Map<string, number>()
|
||||
fields.forEach((entry, position) => {
|
||||
weights.clear()
|
||||
for (const field of searchFields) {
|
||||
const weight =
|
||||
fieldWeights[field] /
|
||||
(1 - lengthNormalization + (lengthNormalization * entry[field].length) / averageLength[field])
|
||||
for (const word of entry[field]) weights.set(word, (weights.get(word) ?? 0) + weight)
|
||||
}
|
||||
for (const [word, weight] of weights) {
|
||||
let posting = building.get(word)
|
||||
if (!posting) building.set(word, (posting = { positions: [], weights: [] }))
|
||||
posting.positions.push(position)
|
||||
posting.weights.push(weight)
|
||||
}
|
||||
})
|
||||
const postings = new Map(
|
||||
[...building].map(([word, posting]) => [
|
||||
word,
|
||||
{ positions: Int32Array.from(posting.positions), weights: Float32Array.from(posting.weights) },
|
||||
]),
|
||||
)
|
||||
return { postings, documents: texts.length, words: [...postings.keys()].sort() }
|
||||
}
|
||||
|
||||
/** Indexed words longer than a 4+ letter query word that start with it. */
|
||||
const prefixMatches = (words: WordIndex, term: string): Array<readonly [string, number]> => {
|
||||
if (term.length < 4) return []
|
||||
let low = 0
|
||||
let high = words.words.length
|
||||
while (low < high) {
|
||||
const middle = (low + high) >> 1
|
||||
if (words.words[middle]! < term) low = middle + 1
|
||||
else high = middle
|
||||
}
|
||||
const matches: Array<readonly [string, number]> = []
|
||||
for (let i = low; i < words.words.length && words.words[i]!.startsWith(term); i++)
|
||||
if (words.words[i] !== term) matches.push([words.words[i]!, prefixWeight])
|
||||
return matches
|
||||
}
|
||||
|
||||
const rank = (
|
||||
entries: ReadonlyArray<SearchEntry>,
|
||||
query: string,
|
||||
index: ReadonlyArray<SearchEntry>,
|
||||
): Array<SearchEntry> => {
|
||||
const terms = searchWords(query)
|
||||
if (terms.length === 0)
|
||||
return [...entries].sort((left, right) => compareText(left.description.path, right.description.path))
|
||||
const words = wordIndexFor(index)
|
||||
const allowed = entries === index ? undefined : new Set(entries.map((entry) => entry.position))
|
||||
const scores = new Map<number, number>()
|
||||
for (const term of terms) {
|
||||
// Each tool counts a query word once, through its best-scoring match.
|
||||
const best = new Map<number, number>()
|
||||
for (const [word, factor] of [[term, 1] as const, ...prefixMatches(words, term)]) {
|
||||
const posting = words.postings.get(word)
|
||||
if (!posting) continue
|
||||
const idf = Math.log(1 + (words.documents - posting.positions.length + 0.5) / (posting.positions.length + 0.5))
|
||||
for (let i = 0; i < posting.positions.length; i++) {
|
||||
const position = posting.positions[i]!
|
||||
if (allowed && !allowed.has(position)) continue
|
||||
const weight = posting.weights[i]!
|
||||
const score = (factor * idf * weight * (k1 + 1)) / (weight + k1)
|
||||
if (score > (best.get(position) ?? 0)) best.set(position, score)
|
||||
}
|
||||
}
|
||||
for (const [position, score] of best) scores.set(position, (scores.get(position) ?? 0) + score)
|
||||
}
|
||||
return [...scores]
|
||||
.map(([position, score]) => ({
|
||||
entry: index[position]!,
|
||||
score: score * (1 + brevityBoost / Math.max(1, index[position]!.nameLength)),
|
||||
}))
|
||||
.sort(
|
||||
(left, right) =>
|
||||
right.score - left.score || compareText(left.entry.description.path, right.entry.description.path),
|
||||
@@ -276,7 +459,7 @@ const makeSearchTool = (searchIndex: ReadonlyArray<SearchEntry>): Tool => ({
|
||||
: scoped.find(
|
||||
(entry) => entry.description.path === pathQuery || toolExpression(entry.description.path) === trimmed,
|
||||
)
|
||||
const ranked = exact !== undefined ? [exact] : rank(scoped, query)
|
||||
const ranked = exact !== undefined ? [exact] : rank(scoped, pathQuery, searchIndex)
|
||||
const items = ranked.slice(offset, offset + (request.limit ?? defaultSearchLimit)).map(({ description }) => ({
|
||||
...description,
|
||||
path: toolExpression(description.path),
|
||||
@@ -296,20 +479,23 @@ export const searchSignature = (() => {
|
||||
return `search(${inputTypeScript(tool, true)}): ${outputTypeScript(tool, true)}`
|
||||
})()
|
||||
|
||||
const toSearchEntry = <R>(visible: VisibleTool<R>): SearchEntry => ({
|
||||
description: describeTool(visible),
|
||||
pathWords: tokenize(visible.path),
|
||||
searchText: [
|
||||
visible.path,
|
||||
visible.tool.description,
|
||||
...visible.namespaces.flatMap((namespace) => (namespace.description === undefined ? [] : [namespace.description])),
|
||||
...inputProperties(visible.tool).flatMap(({ name, description: property }) =>
|
||||
property === undefined ? [name] : [name, property],
|
||||
),
|
||||
]
|
||||
.join("\n")
|
||||
.toLowerCase(),
|
||||
})
|
||||
const toSearchEntry = <R>(visible: VisibleTool<R>, position: number): SearchEntry => {
|
||||
const segments = visible.path.split(".")
|
||||
const name = segments.at(-1) ?? ""
|
||||
return {
|
||||
description: describeTool(visible),
|
||||
position,
|
||||
text: () => ({
|
||||
namespace: segments.slice(0, -1),
|
||||
name: [name],
|
||||
description: [visible.tool.description],
|
||||
// Parameter names only: their descriptions are half of all words and didn't improve ranking.
|
||||
parameters: inputProperties(visible.tool).map((property) => property.name),
|
||||
namespaceDescription: visible.namespaces.map((namespace) => namespace.description ?? ""),
|
||||
}),
|
||||
nameLength: splitWords(name).length,
|
||||
}
|
||||
}
|
||||
|
||||
export const prepare = <R>(tools: Tools<R>): Prepared<R> => {
|
||||
const root = toolTrie(tools)
|
||||
@@ -351,6 +537,7 @@ const resolve = <R>(root: ToolNode<R>, path: ReadonlyArray<string>, index: Reado
|
||||
const closest = rank(
|
||||
ns ? index.filter((entry) => entry.description.path.startsWith(`${ns}.`)) : index,
|
||||
ns ? segments.slice(1).join(" ") : name,
|
||||
index,
|
||||
)[0]
|
||||
throw new ToolRuntimeError(
|
||||
"UnknownTool",
|
||||
|
||||
@@ -1101,7 +1101,7 @@ describe("CodeMode public contract", () => {
|
||||
}
|
||||
})
|
||||
|
||||
test("a query term that is a whole word of the path outranks a substring of it", async () => {
|
||||
test("a query term matches whole words, not substrings", async () => {
|
||||
const simple = (description: string) =>
|
||||
Tool.make({
|
||||
description,
|
||||
@@ -1120,13 +1120,80 @@ describe("CodeMode public contract", () => {
|
||||
expect(ranked.ok).toBe(true)
|
||||
if (ranked.ok) {
|
||||
const value = ranked.value as { items: Array<{ path: string }> }
|
||||
expect(value.items.map((item) => item.path)).toStrictEqual([
|
||||
"tools.cloudflare.get_zones",
|
||||
"tools.cloudflare.get_timezones",
|
||||
])
|
||||
expect(value.items.map((item) => item.path)).toStrictEqual(["tools.cloudflare.get_zones"])
|
||||
}
|
||||
})
|
||||
|
||||
test("search indexes are shared only between runtimes with the same tool text", async () => {
|
||||
const simple = (description: string) =>
|
||||
Tool.make({
|
||||
description,
|
||||
input: Schema.Struct({}),
|
||||
output: Schema.String,
|
||||
execute: () => Effect.succeed("ok"),
|
||||
})
|
||||
const top = async (runtime: ReturnType<typeof CodeMode.make>, query: string) => {
|
||||
const result = await Effect.runPromise(runtime.execute(`return search({ query: ${JSON.stringify(query)} })`))
|
||||
expect(result.ok).toBe(true)
|
||||
return result.ok ? (result.value as { items: Array<{ path: string }> }).items[0]?.path : undefined
|
||||
}
|
||||
// Same tools registered in a different order index identically.
|
||||
const first = CodeMode.make({ tools: { mail: { send: simple("Send an email"), archive: simple("Archive a thread") } } })
|
||||
const reordered = CodeMode.make({ tools: { mail: { archive: simple("Archive a thread"), send: simple("Send an email") } } })
|
||||
// Same paths and count, different descriptions: must not reuse the other index.
|
||||
const changed = CodeMode.make({ tools: { mail: { send: simple("Archive a thread"), archive: simple("Send an email") } } })
|
||||
|
||||
expect(await top(first, "email")).toBe("tools.mail.send")
|
||||
expect(await top(reordered, "email")).toBe("tools.mail.send")
|
||||
expect(await top(changed, "email")).toBe("tools.mail.archive")
|
||||
expect(await top(first, "archive thread")).toBe("tools.mail.archive")
|
||||
})
|
||||
|
||||
test("search ranks by rare words, prefers short names and matches partial words", async () => {
|
||||
const simple = (description: string) =>
|
||||
Tool.make({
|
||||
description,
|
||||
input: Schema.Struct({}),
|
||||
output: Schema.String,
|
||||
execute: () => Effect.succeed("ok"),
|
||||
})
|
||||
const runtime = CodeMode.make({
|
||||
tools: {
|
||||
gmail: { list_labels: simple("List the labels in the mailbox"), list_messages: simple("List messages") },
|
||||
slack: {
|
||||
list_channels: simple("List channels"),
|
||||
list_users: simple("List users"),
|
||||
list_user_groups: simple("List user groups"),
|
||||
get_schedules: simple("Get scheduled messages"),
|
||||
},
|
||||
},
|
||||
})
|
||||
const paths = async (query: Record<string, unknown>) => {
|
||||
const result = await Effect.runPromise(runtime.execute(`return search(${JSON.stringify(query)})`))
|
||||
expect(result.ok).toBe(true)
|
||||
return result.ok ? (result.value as { items: Array<{ path: string }> }).items.map((item) => item.path) : []
|
||||
}
|
||||
|
||||
// "list" is in every tool; "labels" decides.
|
||||
expect((await paths({ query: "list gmail labels" }))[0]).toBe("tools.gmail.list_labels")
|
||||
// Both match "users"; the shorter, more general name wins.
|
||||
expect((await paths({ query: "users", namespace: "slack" }))[0]).toBe("tools.slack.list_users")
|
||||
// A partial word of 4+ letters that matches nothing whole finds the words it starts with.
|
||||
expect(await paths({ query: "schedu" })).toStrictEqual(["tools.slack.get_schedules"])
|
||||
// Short plurals and stop-word-only queries still match.
|
||||
const short = CodeMode.make({
|
||||
tools: { cloud: { describe_vm: simple("Describe a virtual machine"), list_all_issues: simple("List issues") } },
|
||||
})
|
||||
const top = async (query: string) => {
|
||||
const result = await Effect.runPromise(short.execute(`return search(${JSON.stringify({ query })})`))
|
||||
return result.ok ? (result.value as { items: Array<{ path: string }> }).items.map((item) => item.path) : []
|
||||
}
|
||||
expect((await top("vms"))[0]).toBe("tools.cloud.describe_vm")
|
||||
expect(await top("all")).toStrictEqual(["tools.cloud.list_all_issues"])
|
||||
// A namespace filter is a hard filter.
|
||||
expect(await paths({ query: "labels", namespace: "slack" })).toStrictEqual([])
|
||||
})
|
||||
|
||||
test("a plural query term matches singular-only tool text", async () => {
|
||||
const simple = (description: string) =>
|
||||
Tool.make({
|
||||
|
||||
Reference in New Issue
Block a user