From e573115f17ddd362e189fd1b54dacd3128ee5726 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sat, 24 Oct 2026 14:34:07 -0700 Subject: [PATCH] perf(core): cache git discovery in Project.resolve by metadata fingerprint (#58616) --- packages/core/src/git.ts | 96 ++++++++++++++++++++--- packages/core/src/project.ts | 71 +++++++++++------ packages/core/src/repository-cache.ts | 4 +- packages/core/test/fixture/spawn.ts | 24 ++++++ packages/core/test/git.test.ts | 47 ++++++++++- packages/core/test/project.test.ts | 90 ++++++++++++++++++++- packages/core/test/session-create.test.ts | 35 +++++++++ 7 files changed, 330 insertions(+), 37 deletions(-) create mode 100644 packages/core/test/fixture/spawn.ts diff --git a/packages/core/src/git.ts b/packages/core/src/git.ts index 80c790eccb4..8a5a1b0d344 100644 --- a/packages/core/src/git.ts +++ b/packages/core/src/git.ts @@ -1,5 +1,6 @@ export * as Git from "./git.js" +import os from "os" import path from "path" import { Cause, Context, Effect, Exit, Layer, Option, Schedule, Schema } from "effect" import { ChildProcess } from "effect/process" @@ -80,6 +81,13 @@ export class OperationError extends Schema.TaggedError()("Git.Op cause: Schema.optional(Schema.Defect()), }) {} +/** Git could not answer: the spawn failed or it exited with an unexpected status. */ +export class CommandError extends Schema.TaggedError()("Git.CommandError", { + args: Schema.Array(Schema.String), + exitCode: Schema.optional(Schema.Number), + cause: Schema.optional(Schema.Defect()), +}) {} + export class Worktree extends Schema.Class("Git.Worktree")({ directory: AbsolutePath, kind: Schema.Literals(["main", "linked"]), @@ -96,6 +104,11 @@ export class WorktreeError extends Schema.TaggedError()("Git.Work export interface Interface { readonly repo: { readonly discover: (input: AbsolutePath) => Effect.Effect + /** + * Spawn-free stamp of the metadata that `discover`, `remote.get` and `history.rootCommits` + * read for the `.git` entry at `dotGit`: undefined when it isn't a readable repository. + */ + readonly fingerprint: (dotGit: AbsolutePath) => Effect.Effect readonly clone: (input: { remote: string directory: AbsolutePath @@ -109,13 +122,15 @@ export interface Interface { }) => Effect.Effect } readonly remote: { - readonly get: (repository: Repository, name?: string) => Effect.Effect + /** Undefined when the remote doesn't exist. */ + readonly get: (repository: Repository, name?: string) => Effect.Effect } readonly history: { readonly head: (repository: Repository) => Effect.Effect readonly branch: (repository: Repository) => Effect.Effect readonly defaultRemoteBranch: (repository: Repository, remote?: string) => Effect.Effect - readonly rootCommits: (repository: Repository) => Effect.Effect + /** Empty when HEAD is unborn. */ + readonly rootCommits: (repository: Repository) => Effect.Effect } readonly sync: { readonly fetchRemotes: (repository: Repository, input?: { prune?: boolean }) => Effect.Effect @@ -215,20 +230,77 @@ const layer = Layer.effect( }) }) + const stamp = (target: string) => + fs.stat(target).pipe( + Effect.map((info) => + [info.type, Option.getOrUndefined(info.ino), info.size, Option.getOrUndefined(info.mtime)?.getTime()].join( + ":", + ), + ), + Effect.orElseSucceed(() => "missing"), + ) + const readText = (target: string) => fs.readFileStringSafe(target).pipe(Effect.orElseSucceed(() => undefined)) + + // Covers linked worktrees (gitdir file + commondir), HEAD and the ref it points at, + // loose/packed/reftable refs, repository/worktree/global/XDG/system config, and the + // cached `opencode` project id, so external `git init`, a first commit or `git remote` + // edits change the stamp. + const fingerprint = Effect.fn("Git.repo.fingerprint")(function* (dotGit: AbsolutePath) { + const dot = yield* fs.stat(dotGit).pipe(Effect.orElseSucceed(() => undefined)) + if (!dot || (dot.type !== "File" && dot.type !== "Directory")) return undefined + const gitFile = dot.type === "File" ? (yield* readText(dotGit))?.match(/^gitdir:\s*(.+)$/m)?.[1]?.trim() : "" + if (gitFile === undefined) return undefined + const gitDir = gitFile ? yield* fs.resolve(path.resolve(path.dirname(dotGit), gitFile)) : dotGit + const commondir = (yield* readText(path.join(gitDir, "commondir")))?.trim() + const commonDir = commondir ? yield* fs.resolve(path.resolve(gitDir, commondir)) : gitDir + const head = (yield* readText(path.join(gitDir, "HEAD")))?.trim() + if (head === undefined) return undefined + const ref = head.startsWith("ref:") ? head.slice(4).trim() : undefined + + const home = process.env.HOME || os.homedir() + const nosystem = ["1", "true", "yes", "on"].includes((process.env.GIT_CONFIG_NOSYSTEM ?? "").toLowerCase()) + const files = [ + path.join(gitDir, "HEAD"), + path.join(commonDir, "config"), + path.join(commonDir, "config.worktree"), + path.join(gitDir, "config.worktree"), + path.join(commonDir, "packed-refs"), + path.join(commonDir, "reftable", "tables.list"), + path.join(commonDir, "opencode"), + ...(ref ? [path.join(commonDir, ref), path.join(gitDir, ref)] : []), + process.env.GIT_CONFIG_GLOBAL || path.join(home, ".gitconfig"), + path.join(process.env.XDG_CONFIG_HOME || path.join(home, ".config"), "git", "config"), + ...(nosystem ? [] : [process.env.GIT_CONFIG_SYSTEM || "/etc/gitconfig"]), + ] + const stamps = yield* Effect.forEach([dotGit, ...files], stamp, { concurrency: "unbounded" }) + return [gitDir, commonDir, head, ...files, ...stamps].join("\0") + }) + + const command = (cwd: string, args: string[]) => + execute(cwd, proc, args).pipe(Effect.mapError((cause) => new CommandError({ args, cause }))) + const remote = Effect.fn("Git.remote.get")(function* (repository: Repository, name = "origin") { - const result = yield* run(repository.worktree, proc, ["remote", "get-url", name]) - if (result.exitCode !== 0) return undefined + const args = ["remote", "get-url", name] + const result = yield* command(repository.worktree, args) + // Exit 2: no such remote. + if (result.exitCode === 2) return undefined + if (result.exitCode !== 0) return yield* new CommandError({ args, exitCode: result.exitCode }) return result.text.trim() || undefined }) const roots = Effect.fn("Git.history.rootCommits")(function* (repository: Repository) { - const result = yield* run(repository.worktree, proc, ["rev-list", "--max-parents=0", "HEAD"]) - if (result.exitCode !== 0) return [] - return result.text - .split("\n") - .map((item) => item.trim()) - .filter(Boolean) - .toSorted() + const args = ["rev-list", "--max-parents=0", "HEAD"] + const result = yield* command(repository.worktree, args) + if (result.exitCode === 0) + return result.text + .split("\n") + .map((item) => item.trim()) + .filter(Boolean) + .toSorted() + // `rev-parse -q --verify` exits 1 only when HEAD doesn't resolve, i.e. it is unborn. + const verify = yield* command(repository.worktree, ["rev-parse", "-q", "--verify", "HEAD"]) + if (verify.exitCode === 1) return [] + return yield* new CommandError({ args, exitCode: result.exitCode }) }) const head = Effect.fn("Git.history.head")(function* (repository: Repository) { @@ -1291,7 +1363,7 @@ const layer = Layer.effect( }) return Service.of({ - repo: { discover, clone, create }, + repo: { discover, fingerprint, clone, create }, remote: { get: remote }, history: { head, branch, defaultRemoteBranch: remoteHead, rootCommits: roots }, sync: { fetchRemotes: fetch, fetchBranch, checkoutRemoteBranch: checkout, resetHard: reset }, diff --git a/packages/core/src/project.ts b/packages/core/src/project.ts index 15533578772..9922166a89e 100644 --- a/packages/core/src/project.ts +++ b/packages/core/src/project.ts @@ -1,6 +1,6 @@ export * as Project from "./project.js" -import { Context, Duration, Effect, Layer, Schedule, Schema } from "effect" +import { Cache, Context, Duration, Effect, Layer, Schedule, Schema } from "effect" import { ChildProcess } from "effect/process" import { and, asc, desc, eq, gte, isNotNull, isNull, lt, lte, ne, sql } from "drizzle-orm" import path from "path" @@ -320,6 +320,44 @@ const layer = Layer.effect( return root ? ID.make(root) : undefined }) + // Failures resolve like dev (no remote / no root / own worktree) but aren't cached. + const gitDiscover = Effect.fnUntraced(function* (dotGit: AbsolutePath) { + const repo = yield* git.repo.discover(AbsolutePath.make(path.dirname(dotGit))) + if (!repo) return { resolved: undefined, cacheable: false } + let cacheable = true + const fallback = (value: A) => + Effect.orElseSucceed(() => { + cacheable = false + return value + }) + const previous = yield* cached(repo.commonDirectory) + const id = + (yield* remote(repo).pipe(fallback(undefined))) ?? + previous ?? + (yield* rootCommit(repo).pipe(fallback(undefined))) + const canonical = + repo.gitDirectory === repo.commonDirectory + ? repo.worktree + : yield* git.worktree.list(repo).pipe( + Effect.map((items) => items.find((item) => item.kind === "main")?.directory ?? repo.worktree), + fallback(repo.worktree), + ) + const resolved: Resolved = { + previous, + id: id ?? ID.global, + directory: repo.worktree, + canonical, + vcs: { type: "git" as const, store: repo.commonDirectory }, + } + return { resolved, cacheable } + }) + + // Keyed by `.git` path plus its metadata fingerprint, so repeat resolves spawn no git. + const gitCache = yield* Cache.make({ + capacity: 512, + lookup: (key: string) => gitDiscover(AbsolutePath.make(key.slice(0, key.indexOf("\u0000")))), + }) + // Mercurial identity uses the cached ID or the first root changeset; remote-derived // identity (the git `remote()` path) is a follow-up. const hgRoot = Effect.fnUntraced(function* (worktree: AbsolutePath) { @@ -362,27 +400,16 @@ const layer = Layer.effect( Effect.map((matches) => matches[0]), Effect.orElseSucceed(() => undefined), ) - const repo = - native && path.basename(native) === ".git" - ? yield* git.repo.discover(AbsolutePath.make(path.dirname(native))) - : undefined - if (repo) { - const previous = yield* cached(repo.commonDirectory) - const id = (yield* remote(repo)) ?? previous ?? (yield* rootCommit(repo)) - const canonical = - repo.gitDirectory === repo.commonDirectory - ? repo.worktree - : yield* git.worktree.list(repo).pipe( - Effect.map((items) => items.find((item) => item.kind === "main")?.directory ?? repo.worktree), - Effect.orElseSucceed(() => repo.worktree), - ) - return yield* save({ - previous, - id: id ?? ID.global, - directory: repo.worktree, - canonical, - vcs: { type: "git" as const, store: repo.commonDirectory }, - }) + if (native && path.basename(native) === ".git") { + const dotGit = AbsolutePath.make(native) + const fingerprint = yield* git.repo.fingerprint(dotGit) + const key = `${dotGit}\u0000${fingerprint}` + const discovered = fingerprint + ? yield* Cache.get(gitCache, key).pipe( + Effect.tap((entry) => (entry.cacheable ? Effect.void : Cache.invalidate(gitCache, key))), + ) + : yield* gitDiscover(dotGit) + if (discovered.resolved) return yield* save(discovered.resolved) } const hg = native && path.basename(native) === ".hg" ? yield* hgDiscover(AbsolutePath.make(native)) : undefined diff --git a/packages/core/src/repository-cache.ts b/packages/core/src/repository-cache.ts index c6d664a6af4..2106f48a20a 100644 --- a/packages/core/src/repository-cache.ts +++ b/packages/core/src/repository-cache.ts @@ -136,7 +136,9 @@ const layer = Layer.effect( yield* cacheOperation(fs.ensureDir(path.dirname(localPath)), "ensure cache directory", localPath) const existing = yield* git.repo.discover(AbsolutePath.make(localPath)) - const origin = existing ? yield* git.remote.get(existing) : undefined + const origin = existing + ? yield* git.remote.get(existing).pipe(Effect.orElseSucceed(() => undefined)) + : undefined const originReference = origin ? Repository.parse(origin) : undefined // Discovery walks upward, so an enclosing repository with a // matching origin could masquerade as the cache entry; reuse diff --git a/packages/core/test/fixture/spawn.ts b/packages/core/test/fixture/spawn.ts new file mode 100644 index 00000000000..0436f15a06f --- /dev/null +++ b/packages/core/test/fixture/spawn.ts @@ -0,0 +1,24 @@ +import { Effect, Layer, PlatformError } from "effect" +import { ChildProcessSpawner } from "effect/process" +import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner" + +/** Counts process spawns; `fail` rejects matching spawns with a transient error. */ +export function countingSpawner(fail?: (args: readonly string[]) => boolean) { + const state = { spawns: 0 } + const node = CrossSpawnSpawner.node.replace( + CrossSpawnSpawner.node.mapLayer((base) => + Layer.effect( + ChildProcessSpawner.ChildProcessSpawner, + Effect.map(ChildProcessSpawner.ChildProcessSpawner, (spawner) => + ChildProcessSpawner.make((command) => { + state.spawns++ + if (fail && command._tag === "StandardCommand" && fail(command.args)) + return Effect.fail(PlatformError.systemError({ _tag: "Busy", module: "ChildProcess", method: "spawn" })) + return spawner.spawn(command) + }), + ), + ).pipe(Layer.provide(base)), + ), + ) + return { node, spawns: () => state.spawns } +} diff --git a/packages/core/test/git.test.ts b/packages/core/test/git.test.ts index be7576907fe..4c5b3c40ed4 100644 --- a/packages/core/test/git.test.ts +++ b/packages/core/test/git.test.ts @@ -7,7 +7,7 @@ import { LayerNode } from "@opencode/util/effect/layer-node" import { Git } from "@opencode/core/git" import { AbsolutePath, RelativePath } from "@opencode/core/schema" import { VcsPatch } from "@opencode/core/vcs/patch" -import { branch, commit, initRepo, read, withRemote } from "./fixture/git" +import { branch, commit, initGit, initRepo, read, withRemote } from "./fixture/git" import { tmpdir } from "./fixture/tmpdir" import { testEffect } from "./lib/effect" @@ -103,6 +103,51 @@ describe("Git worktrees", () => { expect((yield* git.worktree.list(repo)).some((entry) => entry.directory.endsWith("-git-worktree"))).toBe(false) }), ) + + it.live("reads no remote and no root commits from an unborn repository", () => + Effect.gen(function* () { + const root = yield* Effect.acquireRelease( + Effect.promise(() => tmpdir()), + (dir) => Effect.promise(() => dir[Symbol.asyncDispose]()), + ) + yield* Effect.promise(() => initGit(root.path)) + const git = yield* Git.Service + const repo = yield* git.repo.discover(AbsolutePath.make(root.path)) + if (!repo) throw new Error("Repository not found") + expect(yield* git.remote.get(repo)).toBeUndefined() + expect(yield* git.history.rootCommits(repo)).toEqual([]) + }), + ) + + it.live("fingerprint changes with repository metadata, including from a linked worktree", () => + Effect.gen(function* () { + const root = yield* Effect.acquireRelease( + Effect.promise(() => tmpdir()), + (dir) => Effect.promise(() => dir[Symbol.asyncDispose]()), + ) + const main = path.join(root.path, "main") + const linked = path.join(root.path, "linked") + yield* Effect.promise(() => initGit(main)) + const git = yield* Git.Service + const fingerprint = (dir: string) => git.repo.fingerprint(AbsolutePath.make(path.join(dir, ".git"))) + + expect(yield* fingerprint(root.path)).toBeUndefined() + const unborn = yield* fingerprint(main) + expect(unborn).toBeDefined() + expect(yield* fingerprint(main)).toBe(unborn) + + yield* Effect.promise(() => $`git commit --allow-empty -qm root`.cwd(main).quiet()) + const committed = yield* fingerprint(main) + expect(committed).not.toBe(unborn) + + yield* Effect.promise(() => $`git worktree add -q ${linked}`.cwd(main).quiet()) + const before = yield* fingerprint(linked) + expect(before).toBeDefined() + yield* Effect.promise(() => $`git remote add origin git@github.com:owner/repo.git`.cwd(main).quiet()) + expect(yield* fingerprint(main)).not.toBe(committed) + expect(yield* fingerprint(linked)).not.toBe(before) + }), + ) }) describe("Git trees", () => { diff --git a/packages/core/test/project.test.ts b/packages/core/test/project.test.ts index 1c1e1e70d70..47536b0cac9 100644 --- a/packages/core/test/project.test.ts +++ b/packages/core/test/project.test.ts @@ -2,7 +2,7 @@ import { describe, expect } from "bun:test" import { $ } from "bun" import fs from "fs/promises" import path from "path" -import { Effect, Stream } from "effect" +import { Effect, Layer, Stream } from "effect" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { LayerNode } from "@opencode/util/effect/layer-node" import { Bus } from "@opencode/core/bus" @@ -13,6 +13,7 @@ import { ProjectTable } from "@opencode/core/project/sql" import { AbsolutePath } from "@opencode/core/schema" import { Hash } from "@opencode/util/hash" import { initGit } from "./fixture/git" +import { countingSpawner } from "./fixture/spawn" import { tmpdir } from "./fixture/tmpdir" import { testEffect } from "./lib/effect" @@ -747,4 +748,91 @@ describe("Project.resolve", () => { }) }), ) + + const counted = (fail?: (args: readonly string[]) => boolean) => { + const counter = countingSpawner(fail) + return { + spawns: counter.spawns, + layer: AppNodeBuilder.build(LayerNode.group([Project.node, Database.node, Bus.node]), [counter.node]), + } + } + + testEffect(Layer.empty).live("reuses git discovery until git metadata changes", () => + Effect.gen(function* () { + const tmp = yield* Effect.acquireRelease( + Effect.promise(() => tmpdir()), + (tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()), + ) + const nested = path.join(tmp.path, "packages", "app") + yield* Effect.promise(async () => { + await initRepo(tmp.path, { commit: true }) + await fs.mkdir(nested, { recursive: true }) + }) + const rootSha = Project.ID.make(yield* Effect.promise(() => rootCommit(tmp.path))) + const { layer, spawns } = counted() + yield* Effect.gen(function* () { + const project = yield* Project.Service + expect((yield* project.resolve(abs(nested))).id).toBe(rootSha) + expect(spawns()).toBeGreaterThan(0) + + const before = spawns() + expect((yield* project.resolve(abs(nested))).id).toBe(rootSha) + expect((yield* project.resolve(abs(tmp.path))).id).toBe(rootSha) + expect(spawns()).toBe(before) + + yield* Effect.promise(() => $`git remote add origin git@github.com:owner/repo.git`.cwd(tmp.path).quiet()) + expect((yield* project.resolve(abs(nested))).id).toBe(remoteID("github.com/owner/repo")) + expect(spawns()).toBeGreaterThan(before) + }).pipe(Effect.provide(layer)) + }), + ) + + testEffect(Layer.empty).live("does not cache a transient git failure", () => + Effect.gen(function* () { + const tmp = yield* Effect.acquireRelease( + Effect.promise(() => tmpdir()), + (tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()), + ) + yield* Effect.promise(() => initRepo(tmp.path, { commit: true, remote: "git@github.com:owner/repo.git" })) + let failRemote = true + const { layer } = counted((args) => { + if (!failRemote || !args.includes("remote")) return false + failRemote = false + return true + }) + yield* Effect.gen(function* () { + const project = yield* Project.Service + // Falls back like dev: the remote spawn failed, so identity comes from the root commit. + expect((yield* project.resolve(abs(tmp.path))).id).toBe( + Project.ID.make(yield* Effect.promise(() => rootCommit(tmp.path))), + ) + expect((yield* project.resolve(abs(tmp.path))).id).toBe(remoteID("github.com/owner/repo")) + }).pipe(Effect.provide(layer)) + }), + ) + + testEffect(Layer.empty).live("sees the first commit in a reftable repository", () => + Effect.gen(function* () { + const tmp = yield* Effect.acquireRelease( + Effect.promise(() => tmpdir()), + (tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()), + ) + const supported = yield* Effect.promise( + async () => (await $`git init -q --ref-format=reftable ${tmp.path}`.nothrow().quiet()).exitCode === 0, + ) + if (!supported) return + yield* Effect.gen(function* () { + const project = yield* Project.Service + expect((yield* project.resolve(abs(tmp.path))).id).toBe(Project.ID.global) + yield* Effect.promise(() => + $`git -c user.name=Test -c user.email=test@opencode.test -c commit.gpgsign=false commit --allow-empty -qm root` + .cwd(tmp.path) + .quiet(), + ) + expect((yield* project.resolve(abs(tmp.path))).id).toBe( + Project.ID.make(yield* Effect.promise(() => rootCommit(tmp.path))), + ) + }).pipe(Effect.provide(counted().layer)) + }), + ) }) diff --git a/packages/core/test/session-create.test.ts b/packages/core/test/session-create.test.ts index db72afb4e9b..8e440f761a5 100644 --- a/packages/core/test/session-create.test.ts +++ b/packages/core/test/session-create.test.ts @@ -37,6 +37,7 @@ import { Expected } from "./lib/session-message" import { testEffect } from "./lib/effect" import { LocationServiceMap } from "@opencode/core/location-service-map" import { initGit } from "./fixture/git" +import { countingSpawner } from "./fixture/spawn" import { offlineModels } from "./fixture/models" import { promptLocationNode } from "./fixture/prompt-location" import { globalProjectNode } from "./lib/project" @@ -131,6 +132,40 @@ describe("Session.create", () => { ), ) + testEffect(Layer.empty).live("repeat Session.create on the same location spawns no git", () => + withTmp((directory) => + Effect.gen(function* () { + yield* Effect.promise(() => initGit(directory)) + const counter = countingSpawner() + const layer = AppNodeBuilder.build( + LayerNode.group([ + Database.node, + Bus.node, + Project.node, + LocationServiceMap.node, + SessionProjector.node, + SessionStore.node, + Session.node, + ]), + [ + Bus.node.replace(Bus.configured({ persist: true })), + SessionExecution.node.replace(SessionExecution.noopLayer), + offlineModels, + counter.node, + ], + ) + yield* Effect.gen(function* () { + const sessions = yield* Session.Service + const ref = Location.Ref.make({ directory: AbsolutePath.make(directory) }) + yield* sessions.create({ location: ref }) + const before = counter.spawns() + yield* sessions.create({ location: ref }) + expect(counter.spawns()).toBe(before) + }).pipe(Effect.provide(layer)) + }), + ), + ) + projectIt.live("follows the directory's project identity established after creation", () => withTmp((directory) => Effect.gen(function* () {