mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 03:12:19 +03:00
fea3dd340f
Review (kvnloo): drop_dead_history only looked at facts with a `previous` list, so a tool only a deleted checkout pinned (no history) kept its entry forever. gc now judges every tool row: one with no owners (pre-history) or matching this checkout's own lock stays, otherwise a row lives while any owning checkout's install state dir exists. A tool left with no live row loses its fact. A checkout that finds an entry another checkout recorded now claims it, so that checkout's deletion cannot reclaim an entry still in use.
414 lines
19 KiB
Python
414 lines
19 KiB
Python
"""Native payload staging through PM's existing package authority."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import filecmp
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from contextlib import contextmanager
|
|
from dataclasses import asdict
|
|
from pathlib import Path
|
|
|
|
from pm.install import _lockfile
|
|
from pm.lock import Facts
|
|
from pm.registry import get_package, walk
|
|
from pm.store import current_target
|
|
from pm.testenv import DEVELOPER_TOOLS
|
|
|
|
# The browser tool installs it on first use; Squirrel.Mac before Electron 44.7 can't download a 1 GiB update ZIP.
|
|
_FIRST_USE_ON_MACOS = frozenset({"chromium"})
|
|
|
|
|
|
def _bundle_package_names(target: str | None = None) -> list[str]:
|
|
target = current_target() if target is None else target
|
|
left_out = _FIRST_USE_ON_MACOS if target.startswith("darwin-") else frozenset()
|
|
names = [
|
|
n
|
|
for n in _lockfile().names()
|
|
# Developer tools (just) are never part of a user install.
|
|
if (not get_package(n).internal or n == "uv") and n not in DEVELOPER_TOOLS and n not in left_out
|
|
]
|
|
if "python" not in names:
|
|
names.append("python")
|
|
return names
|
|
|
|
|
|
def _arch_guard(store_dir: Path) -> list[str]:
|
|
"""Every staged binary must be built for this machine's target — a
|
|
payload staged with a mismatched interpreter or PATH tool ships an
|
|
artifact that cannot run. Reads facts, probes each entry binary."""
|
|
from pm.package import machine_matches_binary
|
|
|
|
facts = Facts(store_dir / "facts.json")
|
|
problems = []
|
|
target = current_target()
|
|
for name in _lockfile().names():
|
|
package = get_package(name)
|
|
fact = facts.get(name)
|
|
if fact is None or "entry" not in fact:
|
|
continue
|
|
binary = package.binary(store_dir / fact["entry"], target)
|
|
if binary is None or not binary.is_file():
|
|
continue
|
|
verdict = machine_matches_binary(binary, target)
|
|
# A package that declares this target as emulated (x64 binary run
|
|
# under Windows ARM64 built-in emulation) is fine with the x64 PE.
|
|
if verdict is False and target not in package.emulated_arch_targets:
|
|
problems.append(f"{name}: {binary.name} is not a {target} binary")
|
|
return problems
|
|
|
|
|
|
|
|
from pm.uv_cache_prune import lock_package_names, prune_uv_cache_to_lock
|
|
|
|
__all__ = ["lock_package_names", "prune_uv_cache_to_lock", "stage_uv_cache"]
|
|
|
|
|
|
def stage_uv_cache(source: Path, destination: Path) -> None:
|
|
"""Ship everything a per-install venv rebuild resolves from offline.
|
|
|
|
sdist source trees (including Rust target/ outputs) are build-only bulk —
|
|
offline rebuilds install the built wheel entries, never recompile from
|
|
source — so they stay out. Built wheel ZIPs (including the copies uv keeps
|
|
beside sdist metadata.msgpack shards) ship: without them an offline
|
|
rebuild falls back to wanting the sdist download and fails closed.
|
|
|
|
On macOS and Linux, uv's wheel entries (``wheels-v*/pypi/<name>/<version>``,
|
|
a hash directory deeper for URL, git and named-index sources) are relative
|
|
symlinks into ``archive-v0/``, and they ship as links. Following them
|
|
stored every unpacked wheel a second time and gave the seeded machine
|
|
cache directories where uv renames its own links (#141891).
|
|
"""
|
|
def ignore(directory: str, names: list[str]) -> set[str]:
|
|
path = Path(directory)
|
|
parts = path.relative_to(source).parts
|
|
if not parts or not parts[0].startswith("sdists-v"):
|
|
return set()
|
|
# Source trees live under a revision selected by a sibling pointer.
|
|
if "src" in names and (path / "src").is_dir() and any(
|
|
(path.parent / pointer).is_file() for pointer in ("revision.http", "revision.rev")
|
|
):
|
|
return {"src"}
|
|
return set()
|
|
|
|
shutil.copytree(source, destination, ignore=ignore, symlinks=True)
|
|
|
|
|
|
def link_cache_to_venv(cache: Path, site_packages: Path) -> int:
|
|
"""Replace each unpacked-wheel file the venv holds byte for byte with a relative link to it.
|
|
|
|
The payload venv was installed from these same archives, so most of the cache is a second
|
|
copy. Squirrel.Mac before Electron 44.7 holds the whole update ZIP in one buffer and crashes
|
|
past 1 GiB, so the macOS ZIP ships each file once. The uv seed copies the bytes back out.
|
|
"""
|
|
archives = cache / "archive-v0"
|
|
linked = 0
|
|
for bucket in sorted(archives.iterdir()) if archives.is_dir() else ():
|
|
if bucket.is_symlink() or not bucket.is_dir():
|
|
continue
|
|
for directory, _dirs, names in os.walk(bucket):
|
|
for name in names:
|
|
path = Path(directory) / name
|
|
installed = site_packages / path.relative_to(bucket)
|
|
if (path.is_symlink() or installed.is_symlink() or not installed.is_file()
|
|
or not filecmp.cmp(path, installed, shallow=False)):
|
|
continue
|
|
staging = path.with_name(f".{name}.link")
|
|
os.symlink(os.path.relpath(installed, path.parent), staging)
|
|
os.replace(staging, path)
|
|
linked += 1
|
|
filecmp.clear_cache()
|
|
return linked
|
|
|
|
|
|
def stage_pm_runtime(root: Path, python: Path, repo: Path, *, offline: bool = False,
|
|
cache: Path | None = None) -> None:
|
|
"""Publish the same PM dependency graph as source installs, ready offline."""
|
|
from pm import stage_manager_runtime
|
|
from scripts.bundles.payload import seal_pm_runtime
|
|
|
|
destination = root / "pm-runtime"
|
|
if destination.exists():
|
|
shutil.rmtree(destination)
|
|
stage_manager_runtime(python=python, destination=destination, project=repo / "pm", offline=offline, cache=cache)
|
|
seal_pm_runtime(root, python)
|
|
|
|
|
|
def stage_native(args) -> int:
|
|
"""Isolate HOME and PM state, but retain the provider's reusable build cache."""
|
|
from pm.packages import uv_cache_dir
|
|
|
|
out = Path(args.out).resolve()
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
(out / "manifest.json").unlink(missing_ok=True)
|
|
root = Path(__file__).resolve().parents[2]
|
|
# Resolve before HOME isolation so the build warms the cache CI saves.
|
|
cache = Path(getattr(args, "cache", None) or os.environ.get("UV_CACHE_DIR") or uv_cache_dir()).resolve()
|
|
base_env = dict(os.environ)
|
|
if current_target() == "win32-arm64":
|
|
from pm.native_build import prepare_windows_environment
|
|
|
|
base_env = prepare_windows_environment(source=root, state=out.parent / ".build-deps", env=base_env)
|
|
# Rustup resolves its installed toolchain under HOME unless these are explicit.
|
|
# Preserve the build host's compiler and Cargo cache before isolating app state.
|
|
for key, directory in (("CARGO_HOME", ".cargo"), ("RUSTUP_HOME", ".rustup")):
|
|
base_env.setdefault(key, str(Path.home() / directory))
|
|
with tempfile.TemporaryDirectory(prefix=".build-", dir=out) as work:
|
|
env = {**base_env, "HOME": work, "USERPROFILE": work,
|
|
"HERMES_HOME": str(Path(work) / ".hermes"),
|
|
"HERMES_RUNTIME_DIR": str(out / "tools"),
|
|
"HERMES_PYTHON_SRC_ROOT": str(root),
|
|
"XDG_CACHE_HOME": str(Path(work) / "cache"),
|
|
"XDG_CONFIG_HOME": str(Path(work) / "config"),
|
|
"UV_CACHE_DIR": str(cache),
|
|
"PYTHONPATH": os.pathsep.join([str(root), *filter(None, sys.path)])}
|
|
command = [sys.executable, "-B", "-m", "scripts.bundles.native", "--out", str(out),
|
|
"--ref", args.ref or "HEAD", "--source", str(root)]
|
|
for name, product in getattr(args, "frontends", {}).items():
|
|
command += [f"--{name}", str(product)]
|
|
return subprocess.run(command, cwd=root, env=env, check=False).returncode
|
|
|
|
|
|
@contextmanager
|
|
def _phase(label: str):
|
|
"""Name each staging phase and its duration: several run minutes without output."""
|
|
print(f" {label}…", flush=True)
|
|
started = time.monotonic()
|
|
yield
|
|
print(f" ✓ {label} ({time.monotonic() - started:.1f}s)", flush=True)
|
|
|
|
|
|
def prune_staged_store(store_dir: Path, names: list[str]) -> None:
|
|
"""Prune this build's store, never the ambient user's tool store."""
|
|
facts = Facts(store_dir / "facts.json", strict=True)
|
|
facts.retain({package.name for package in walk(names)})
|
|
# No other checkout shares a build's store: only the selected pins are live here.
|
|
facts.drop_dead_pins(lambda name, row: row["entry"] == facts.get(name)["entry"])
|
|
keep = facts.entries_in_use()
|
|
for entry in store_dir.iterdir():
|
|
if entry.is_dir() and not entry.name.startswith(".") and entry.name not in keep:
|
|
shutil.rmtree(entry)
|
|
|
|
|
|
def prepare_native(*, out: Path, ref: str, source: Path, cache: Path,
|
|
tools: Path | None = None, env: dict | None = None) -> Path:
|
|
"""Prepare final payload dependencies inside the caller's isolated PM process.
|
|
|
|
The caller supplies its compiler environment; only the full standalone
|
|
wrapper provisions machine prerequisites and isolates HOME.
|
|
"""
|
|
from scripts.bundles.native_prepared import preparation_lock, prepared_path
|
|
|
|
out = Path(out).absolute()
|
|
if out != out.resolve():
|
|
raise ValueError("symlinked native output")
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
with preparation_lock(out):
|
|
prepared_path(out).unlink(missing_ok=True)
|
|
(out / "manifest.json").unlink(missing_ok=True)
|
|
return _prepare_native(out=out, ref=ref, source=Path(source).resolve(),
|
|
cache=Path(cache).absolute(), tools=tools, env=env)
|
|
|
|
|
|
def _prepare_native(*, out: Path, ref: str, source: Path, cache: Path,
|
|
tools: Path | None, env: dict | None) -> Path:
|
|
from pm import paths
|
|
from pm.package import InstallError
|
|
|
|
store_dir = out / "tools"
|
|
store_dir.mkdir(parents=True, exist_ok=True)
|
|
repo_dir = out / "hermes-agent"
|
|
from scripts.bundles.payload import INERT_SNAPSHOT_DIRS, snapshot
|
|
revision = subprocess.check_output(
|
|
["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=source,
|
|
text=True, encoding="utf-8").strip()
|
|
with _phase(f"staging repo snapshot ({ref})"):
|
|
snapshot(source, revision, repo_dir, exclude=INERT_SNAPSHOT_DIRS)
|
|
# PM's provider code reads its adjacent lock. Never combine that tool graph
|
|
# with a revision selecting different pins.
|
|
if (repo_dir / "pm/lock.json").read_bytes() != paths.lockfile_path().read_bytes():
|
|
raise ValueError("selected revision's PM lock differs from the builder; use a checkout at that revision")
|
|
build_env = os.environ if env is None else env
|
|
|
|
names = [
|
|
n for n in _bundle_package_names()
|
|
if get_package(n).missing_reason(current_target()) is None
|
|
]
|
|
from pm import prepare_tools, stage_tools
|
|
|
|
with _phase(f"preparing {len(names)} pinned tools"):
|
|
prepare_tools(names, out=Path(tools) if tools is not None else store_dir,
|
|
target=current_target(), cache=cache)
|
|
if tools is not None:
|
|
with _phase("copying verified tools into the payload store"):
|
|
stage_tools(names, source_store=Path(tools), out=store_dir, target=current_target())
|
|
|
|
# Prune the staged store BEFORE the venv sync and packaging: drop the
|
|
# fetch-<sha> download-cache archives (needed only at install time — dead
|
|
# weight in the shipped payload AND in the CI cache that restores this
|
|
# dir) and any orphaned package versions left over from an older lock
|
|
# the cache carried in. A lean staged store = a lean CI cache.
|
|
|
|
# Only this build's store is ours to prune; machine-wide partials are not.
|
|
# Cached facts may still name packages removed from the current selection.
|
|
# Retain the dependency closure before using facts as the deletion roots.
|
|
with _phase("pruning the staged tool store"):
|
|
prune_staged_store(store_dir, names)
|
|
|
|
|
|
facts = Facts(store_dir / "facts.json")
|
|
python_fact = facts.get("python")
|
|
if python_fact is None:
|
|
raise InstallError("venv", "no staged interpreter to build on")
|
|
python_bin = get_package("python").binary(
|
|
store_dir / python_fact["entry"], current_target()
|
|
)
|
|
|
|
if python_bin is None:
|
|
raise FileNotFoundError("staged Python executable is missing")
|
|
with _phase("pm-runtime (independent locked dependencies)"):
|
|
stage_pm_runtime(out, python_bin, repo_dir, cache=cache)
|
|
|
|
# Build + sync INSIDE the staged repo: the editable project install
|
|
# must point at the payload's own tree, not this checkout.
|
|
venv_dir = out / "venv"
|
|
if venv_dir.exists():
|
|
shutil.rmtree(venv_dir)
|
|
env = dict(build_env)
|
|
from pm import build_environment
|
|
|
|
# Cold native wheels need a larger budget than interactive installs.
|
|
build_environment(source=repo_dir, python=python_bin, out=venv_dir,
|
|
env=env, cache=cache, all_extras=True, sealed=True, explicit=True,
|
|
timeout=2 * 60 * 60)
|
|
print("✓ venv (all extras, on the staged interpreter)")
|
|
|
|
# Inventory the staged interpreter before publishing the bundle contract.
|
|
from pm.features import installed_extras, write_features
|
|
|
|
features = installed_extras(repo_dir, venv_dir, python_exe=python_bin)
|
|
write_features(features, out)
|
|
|
|
# The payload venv was built from this exact lock and feature set. Publish
|
|
# that state so the first runtime sync can reuse the shipped environment
|
|
# instead of rebuilding it before any plugin dependencies are requested.
|
|
from pm.packages import Venv
|
|
|
|
facts.record_state("venv", Venv(repo_dir).expected_stamp(features, plugin_dirs=[]), features)
|
|
print(f"✓ enabled-features.json ({len(features)} extras recorded)")
|
|
|
|
# Ship the full uv cache (build-only sdist sources are already excluded by
|
|
# stage_uv_cache). Any per-install venv rebuild —
|
|
# plugin extras, feature changes — resolves entirely from the shipped
|
|
# wheels with zero network access, on every profile of this install.
|
|
payload_cache = out / "uv-cache"
|
|
if payload_cache.exists():
|
|
shutil.rmtree(payload_cache, ignore_errors=True)
|
|
src_cache = cache
|
|
if src_cache.is_dir():
|
|
print(f" uv-cache: copying {src_cache} → payload...", flush=True)
|
|
stage_uv_cache(src_cache, payload_cache)
|
|
pruned = prune_uv_cache_to_lock(payload_cache, repo_dir)
|
|
print(f"✓ uv-cache (lock-scoped: pruned {pruned} stale entries; offline rebuilds resolve from shipped wheels)", flush=True)
|
|
else:
|
|
raise InstallError("uv-cache", "runtime dependency cache is missing")
|
|
if current_target().startswith("win32-"):
|
|
from scripts.bundles.msix_names import drop_msix_reserved_names
|
|
for tree in drop_msix_reserved_names(out):
|
|
print(f"✓ msix: dropped {tree.as_posix()} (holds a package-reserved name, never read at runtime)")
|
|
|
|
bad = _arch_guard(store_dir)
|
|
for line in bad:
|
|
print(f"✗ arch: {line}")
|
|
if bad:
|
|
raise InstallError("tools", "native architecture verification failed")
|
|
from scripts.bundles.payload import record_tools
|
|
recorded = {name: fact["entry"] for name in names if (fact := facts.get(name)) and "entry" in fact}
|
|
record_tools(out, paths.lockfile_path(), current_target(), recorded)
|
|
from scripts.build.inputs import AgentInputs, RESOURCE_ENV, dependency_site
|
|
from scripts.bundles.native_prepared import publish_prepared
|
|
|
|
site = dependency_site(venv_dir, python_fact["version"], current_target())
|
|
(site / "hermes-agent.pth").write_text(
|
|
Path(os.path.relpath(repo_dir, site)).as_posix() + "\n", encoding="utf-8")
|
|
# An MSIX holds no links, and uv's Windows cache uses pointer files instead.
|
|
if not current_target().startswith("win32-"):
|
|
linked = link_cache_to_venv(payload_cache, site)
|
|
print(f"✓ uv-cache: {linked} files the venv holds byte for byte ship once, as links into it")
|
|
from scripts.bundles.payload import relativize_links
|
|
relativize_links(out)
|
|
from scripts.bundles.bytecode import bake_bytecode
|
|
baked = bake_bytecode(out, python_bin)
|
|
print(f"✓ baked bytecode ({baked['modules']} modules, unchecked-hash, owner-writable caches)")
|
|
inputs = AgentInputs(
|
|
project=repo_dir / "pyproject.toml", code=repo_dir, repo="hermes-agent",
|
|
placement="contained", target=current_target(), python=python_bin,
|
|
site_packages=site, environment=venv_dir,
|
|
tools=store_dir, pm_runtime=out / "pm-runtime", ref=ref,
|
|
resources={name: repo_dir / name for name in RESOURCE_ENV},
|
|
features=out / "enabled-features.json",
|
|
)
|
|
return publish_prepared(out, source, revision, inputs)
|
|
|
|
|
|
def finish_native(prepared: Path, frontends: dict[str, Path]) -> int:
|
|
"""Consume verified job-local paths. No dependency acquisition or repair."""
|
|
from scripts.build.agent import assemble
|
|
from scripts.build.inputs import AgentInputs
|
|
from scripts.bundles.native_prepared import load_prepared, preparation_lock
|
|
|
|
prepared = Path(prepared).absolute()
|
|
if not prepared.name.endswith(".prepared.json") or not prepared.is_file():
|
|
raise ValueError("native preparation is missing or invalid; run preparation again")
|
|
out = prepared.with_name(prepared.name.removesuffix(".prepared.json"))
|
|
with preparation_lock(out):
|
|
(out / "manifest.json").unlink(missing_ok=True)
|
|
inputs = load_prepared(prepared)
|
|
values = asdict(inputs)
|
|
values["frontends"] = {name: Path(path).absolute() for name, path in frontends.items()}
|
|
print("assembling verified payload", flush=True)
|
|
assemble(AgentInputs.from_dict(values), out)
|
|
print(f"✓ manifest ({out / 'manifest.json'})")
|
|
return 0
|
|
|
|
|
|
def _stage_native(args) -> int:
|
|
from pm import paths
|
|
from pm.features import FeatureProbeError
|
|
from pm.package import InstallError
|
|
|
|
try:
|
|
prepared = prepare_native(
|
|
out=Path(args.out), ref=args.ref or "HEAD",
|
|
source=getattr(args, "source", None) or paths.repo_root(),
|
|
cache=Path(getattr(args, "cache", None) or os.environ["UV_CACHE_DIR"]),
|
|
tools=getattr(args, "tools", None),
|
|
)
|
|
return finish_native(prepared, getattr(args, "frontends", {}))
|
|
except (InstallError, FeatureProbeError) as exc:
|
|
print(f"✗ {exc}")
|
|
return 1
|
|
|
|
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--out", required=True)
|
|
parser.add_argument("--ref", default="HEAD")
|
|
parser.add_argument("--source", type=Path, required=True)
|
|
parser.add_argument("--tui", type=Path)
|
|
parser.add_argument("--web", type=Path)
|
|
parser.add_argument("--webapp", type=Path)
|
|
args = parser.parse_args()
|
|
args.frontends = {name: path for name in ("tui", "web", "webapp") if (path := getattr(args, name)) is not None}
|
|
return _stage_native(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|