"""Native payload staging through PM's existing package authority.""" from __future__ import annotations import argparse import filecmp import os import shutil import subprocess import sys import tempfile import time from contextlib import contextmanager from dataclasses import asdict from pathlib import Path from pm.install import _lockfile from pm.lock import Facts from pm.registry import get_package, walk from pm.store import current_target from pm.testenv import DEVELOPER_TOOLS # The browser tool installs it on first use; Squirrel.Mac before Electron 44.7 can't download a 1 GiB update ZIP. _FIRST_USE_ON_MACOS = frozenset({"chromium"}) def _bundle_package_names(target: str | None = None) -> list[str]: target = current_target() if target is None else target left_out = _FIRST_USE_ON_MACOS if target.startswith("darwin-") else frozenset() names = [ n for n in _lockfile().names() # Developer tools (just) are never part of a user install. if (not get_package(n).internal or n == "uv") and n not in DEVELOPER_TOOLS and n not in left_out ] if "python" not in names: names.append("python") return names def _tools_for_builder(names: list[str], store: Path, target: str) -> list[str]: """Admit entries whose glibc floor exceeds this builder after proving they came from the compatible userland.""" from pm.build_operations import verified_tools staged = [name for name in names if get_package(name).minimum_glibc.get(target) is not None and get_package(name).native_abi_reason(target)] if staged: # The compatible PM installed and probed these exact entries in this job's # payload store. Recheck lock, target, tree digests and links without executing # them on the old libc. Never substitute a missing entry or mint its facts. verified_tools(staged, source_store=store, target=target) return [name for name in names if name not in staged] def _arch_guard(store_dir: Path) -> list[str]: """Every staged binary must be built for this machine's target — a payload staged with a mismatched interpreter or PATH tool ships an artifact that cannot run. Reads facts, probes each entry binary.""" from pm.package import machine_matches_binary facts = Facts(store_dir / "facts.json") problems = [] target = current_target() for name in _lockfile().names(): package = get_package(name) fact = facts.get(name) if fact is None or "entry" not in fact: continue binary = package.binary(store_dir / fact["entry"], target) if binary is None or not binary.is_file(): continue verdict = machine_matches_binary(binary, target) # A package that declares this target as emulated (x64 binary run # under Windows ARM64 built-in emulation) is fine with the x64 PE. if verdict is False and target not in package.emulated_arch_targets: problems.append(f"{name}: {binary.name} is not a {target} binary") return problems from pm.uv_cache_prune import lock_package_names, prune_uv_cache_to_lock __all__ = ["lock_package_names", "prune_uv_cache_to_lock", "stage_uv_cache"] def stage_uv_cache(source: Path, destination: Path) -> None: """Ship everything a per-install venv rebuild resolves from offline. sdist source trees (including Rust target/ outputs) are build-only bulk — offline rebuilds install the built wheel entries, never recompile from source — so they stay out. Built wheel ZIPs (including the copies uv keeps beside sdist metadata.msgpack shards) ship: without them an offline rebuild falls back to wanting the sdist download and fails closed. On macOS and Linux, uv's wheel entries (``wheels-v*/pypi//``, a hash directory deeper for URL, git and named-index sources) are relative symlinks into ``archive-v0/``, and they ship as links. Following them stored every unpacked wheel a second time and gave the seeded machine cache directories where uv renames its own links (#141891). """ def ignore(directory: str, names: list[str]) -> set[str]: path = Path(directory) parts = path.relative_to(source).parts if not parts or not parts[0].startswith("sdists-v"): return set() # Source trees live under a revision selected by a sibling pointer. if "src" in names and (path / "src").is_dir() and any( (path.parent / pointer).is_file() for pointer in ("revision.http", "revision.rev") ): return {"src"} return set() shutil.copytree(source, destination, ignore=ignore, symlinks=True) def link_cache_to_venv(cache: Path, site_packages: Path) -> int: """Replace each unpacked-wheel file the venv holds byte for byte with a relative link to it. The payload venv was installed from these same archives, so most of the cache is a second copy. Squirrel.Mac before Electron 44.7 holds the whole update ZIP in one buffer and crashes past 1 GiB, so the macOS ZIP ships each file once. The uv seed copies the bytes back out. """ archives = cache / "archive-v0" linked = 0 for bucket in sorted(archives.iterdir()) if archives.is_dir() else (): if bucket.is_symlink() or not bucket.is_dir(): continue for directory, _dirs, names in os.walk(bucket): for name in names: path = Path(directory) / name installed = site_packages / path.relative_to(bucket) if (path.is_symlink() or installed.is_symlink() or not installed.is_file() or not filecmp.cmp(path, installed, shallow=False)): continue staging = path.with_name(f".{name}.link") os.symlink(os.path.relpath(installed, path.parent), staging) os.replace(staging, path) linked += 1 filecmp.clear_cache() return linked def stage_pm_runtime(root: Path, python: Path, repo: Path, *, offline: bool = False, cache: Path | None = None) -> None: """Publish the same PM dependency graph as source installs, ready offline.""" from pm import stage_manager_runtime from scripts.bundles.payload import seal_pm_runtime destination = root / "pm-runtime" if destination.exists(): shutil.rmtree(destination) stage_manager_runtime(python=python, destination=destination, project=repo / "pm", offline=offline, cache=cache) seal_pm_runtime(root, python) def stage_native(args) -> int: """Isolate HOME and PM state, but retain the provider's reusable build cache.""" from pm.packages import uv_cache_dir out = Path(args.out).resolve() out.mkdir(parents=True, exist_ok=True) (out / "manifest.json").unlink(missing_ok=True) root = Path(__file__).resolve().parents[2] # Resolve before HOME isolation so the build warms the cache CI saves. cache = Path(getattr(args, "cache", None) or os.environ.get("UV_CACHE_DIR") or uv_cache_dir()).resolve() base_env = dict(os.environ) if current_target() == "win32-arm64": from pm.native_build import prepare_windows_environment base_env = prepare_windows_environment(source=root, state=out.parent / ".build-deps", env=base_env) # Rustup resolves its installed toolchain under HOME unless these are explicit. # Preserve the build host's compiler and Cargo cache before isolating app state. for key, directory in (("CARGO_HOME", ".cargo"), ("RUSTUP_HOME", ".rustup")): base_env.setdefault(key, str(Path.home() / directory)) with tempfile.TemporaryDirectory(prefix=".build-", dir=out) as work: env = {**base_env, "HOME": work, "USERPROFILE": work, "HERMES_HOME": str(Path(work) / ".hermes"), "HERMES_RUNTIME_DIR": str(out / "tools"), "HERMES_PYTHON_SRC_ROOT": str(root), "XDG_CACHE_HOME": str(Path(work) / "cache"), "XDG_CONFIG_HOME": str(Path(work) / "config"), "UV_CACHE_DIR": str(cache), "PYTHONPATH": os.pathsep.join([str(root), *filter(None, sys.path)])} command = [sys.executable, "-B", "-m", "scripts.bundles.native", "--out", str(out), "--ref", args.ref or "HEAD", "--source", str(root)] for name, product in getattr(args, "frontends", {}).items(): command += [f"--{name}", str(product)] return subprocess.run(command, cwd=root, env=env, check=False).returncode @contextmanager def _phase(label: str): """Name each staging phase and its duration: several run minutes without output.""" print(f" {label}…", flush=True) started = time.monotonic() yield print(f" ✓ {label} ({time.monotonic() - started:.1f}s)", flush=True) def prune_staged_store(store_dir: Path, names: list[str]) -> None: """Prune this build's store, never the ambient user's tool store.""" facts = Facts(store_dir / "facts.json", strict=True) facts.retain({package.name for package in walk(names)}) # No other checkout shares a build's store: only the selected pins are live here. facts.drop_dead_pins(lambda name, row: row["entry"] == facts.get(name)["entry"]) keep = facts.entries_in_use() for entry in store_dir.iterdir(): if entry.is_dir() and not entry.name.startswith(".") and entry.name not in keep: shutil.rmtree(entry) def prepare_native(*, out: Path, ref: str, source: Path, cache: Path, tools: Path | None = None, env: dict | None = None) -> Path: """Prepare final payload dependencies inside the caller's isolated PM process. The caller supplies its compiler environment; only the full standalone wrapper provisions machine prerequisites and isolates HOME. """ from scripts.bundles.native_prepared import preparation_lock, prepared_path out = Path(out).absolute() if out != out.resolve(): raise ValueError("symlinked native output") out.mkdir(parents=True, exist_ok=True) with preparation_lock(out): prepared_path(out).unlink(missing_ok=True) (out / "manifest.json").unlink(missing_ok=True) return _prepare_native(out=out, ref=ref, source=Path(source).resolve(), cache=Path(cache).absolute(), tools=tools, env=env) def _prepare_native(*, out: Path, ref: str, source: Path, cache: Path, tools: Path | None, env: dict | None) -> Path: from pm import paths from pm.package import InstallError store_dir = out / "tools" store_dir.mkdir(parents=True, exist_ok=True) repo_dir = out / "hermes-agent" from scripts.bundles.payload import INERT_SNAPSHOT_DIRS, snapshot revision = subprocess.check_output( ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=source, text=True, encoding="utf-8").strip() with _phase(f"staging repo snapshot ({ref})"): snapshot(source, revision, repo_dir, exclude=INERT_SNAPSHOT_DIRS) # PM's provider code reads its adjacent lock. Never combine that tool graph # with a revision selecting different pins. if (repo_dir / "pm/lock.json").read_bytes() != paths.lockfile_path().read_bytes(): raise ValueError("selected revision's PM lock differs from the builder; use a checkout at that revision") build_env = os.environ if env is None else env names = [ n for n in _bundle_package_names() if get_package(n).missing_reason(current_target()) is None ] from pm import prepare_tools, stage_tools local = _tools_for_builder(names, store_dir, current_target()) with _phase(f"preparing {len(local)} pinned tools"): prepare_tools(local, out=Path(tools) if tools is not None else store_dir, target=current_target(), cache=cache) if tools is not None: with _phase("copying verified tools into the payload store"): stage_tools(local, source_store=Path(tools), out=store_dir, target=current_target()) # Prune the staged store BEFORE the venv sync and packaging: drop the # fetch- download-cache archives (needed only at install time — dead # weight in the shipped payload AND in the CI cache that restores this # dir) and any orphaned package versions left over from an older lock # the cache carried in. A lean staged store = a lean CI cache. # Only this build's store is ours to prune; machine-wide partials are not. # Cached facts may still name packages removed from the current selection. # Retain the dependency closure before using facts as the deletion roots. with _phase("pruning the staged tool store"): prune_staged_store(store_dir, names) facts = Facts(store_dir / "facts.json") python_fact = facts.get("python") if python_fact is None: raise InstallError("venv", "no staged interpreter to build on") python_bin = get_package("python").binary( store_dir / python_fact["entry"], current_target() ) if python_bin is None: raise FileNotFoundError("staged Python executable is missing") with _phase("pm-runtime (independent locked dependencies)"): stage_pm_runtime(out, python_bin, repo_dir, cache=cache) # Build + sync INSIDE the staged repo: the editable project install # must point at the payload's own tree, not this checkout. venv_dir = out / "venv" if venv_dir.exists(): shutil.rmtree(venv_dir) env = dict(build_env) from pm import build_environment # Cold native wheels need a larger budget than interactive installs. build_environment(source=repo_dir, python=python_bin, out=venv_dir, env=env, cache=cache, all_extras=True, sealed=True, explicit=True, timeout=2 * 60 * 60) print("✓ venv (all extras, on the staged interpreter)") # Inventory the staged interpreter before publishing the bundle contract. from pm.features import installed_extras, write_features features = installed_extras(repo_dir, venv_dir, python_exe=python_bin) write_features(features, out) # The payload venv was built from this exact lock and feature set. Publish # that state so the first runtime sync can reuse the shipped environment # instead of rebuilding it before any plugin dependencies are requested. from pm.packages import Venv facts.record_state("venv", Venv(repo_dir).expected_stamp(features, plugin_dirs=[]), features) print(f"✓ enabled-features.json ({len(features)} extras recorded)") # Ship the full uv cache (build-only sdist sources are already excluded by # stage_uv_cache). Any per-install venv rebuild — # plugin extras, feature changes — resolves entirely from the shipped # wheels with zero network access, on every profile of this install. payload_cache = out / "uv-cache" if payload_cache.exists(): shutil.rmtree(payload_cache, ignore_errors=True) src_cache = cache if src_cache.is_dir(): print(f" uv-cache: copying {src_cache} → payload...", flush=True) stage_uv_cache(src_cache, payload_cache) pruned = prune_uv_cache_to_lock(payload_cache, repo_dir) print(f"✓ uv-cache (lock-scoped: pruned {pruned} stale entries; offline rebuilds resolve from shipped wheels)", flush=True) else: raise InstallError("uv-cache", "runtime dependency cache is missing") if current_target().startswith("win32-"): from scripts.bundles.msix_names import drop_msix_reserved_names for tree in drop_msix_reserved_names(out): print(f"✓ msix: dropped {tree.as_posix()} (holds a package-reserved name, never read at runtime)") bad = _arch_guard(store_dir) for line in bad: print(f"✗ arch: {line}") if bad: raise InstallError("tools", "native architecture verification failed") from scripts.bundles.payload import record_tools recorded = {name: fact["entry"] for name in names if (fact := facts.get(name)) and "entry" in fact} record_tools(out, paths.lockfile_path(), current_target(), recorded) from scripts.build.inputs import AgentInputs, RESOURCE_ENV, dependency_site from scripts.bundles.native_prepared import publish_prepared site = dependency_site(venv_dir, python_fact["version"], current_target()) (site / "hermes-agent.pth").write_text( Path(os.path.relpath(repo_dir, site)).as_posix() + "\n", encoding="utf-8") # An MSIX holds no links, and uv's Windows cache uses pointer files instead. if not current_target().startswith("win32-"): linked = link_cache_to_venv(payload_cache, site) print(f"✓ uv-cache: {linked} files the venv holds byte for byte ship once, as links into it") from scripts.bundles.payload import relativize_links relativize_links(out) from scripts.bundles.bytecode import bake_bytecode baked = bake_bytecode(out, python_bin) print(f"✓ baked bytecode ({baked['modules']} modules, unchecked-hash, owner-writable caches)") inputs = AgentInputs( project=repo_dir / "pyproject.toml", code=repo_dir, repo="hermes-agent", placement="contained", target=current_target(), python=python_bin, site_packages=site, environment=venv_dir, tools=store_dir, pm_runtime=out / "pm-runtime", ref=ref, resources={name: repo_dir / name for name in RESOURCE_ENV}, features=out / "enabled-features.json", ) return publish_prepared(out, source, revision, inputs) def finish_native(prepared: Path, frontends: dict[str, Path]) -> int: """Consume verified job-local paths. No dependency acquisition or repair.""" from scripts.build.agent import assemble from scripts.build.inputs import AgentInputs from scripts.bundles.native_prepared import load_prepared, preparation_lock prepared = Path(prepared).absolute() if not prepared.name.endswith(".prepared.json") or not prepared.is_file(): raise ValueError("native preparation is missing or invalid; run preparation again") out = prepared.with_name(prepared.name.removesuffix(".prepared.json")) with preparation_lock(out): (out / "manifest.json").unlink(missing_ok=True) inputs = load_prepared(prepared) values = asdict(inputs) values["frontends"] = {name: Path(path).absolute() for name, path in frontends.items()} print("assembling verified payload", flush=True) assemble(AgentInputs.from_dict(values), out) print(f"✓ manifest ({out / 'manifest.json'})") return 0 def _stage_native(args) -> int: from pm import paths from pm.features import FeatureProbeError from pm.package import InstallError try: prepared = prepare_native( out=Path(args.out), ref=args.ref or "HEAD", source=getattr(args, "source", None) or paths.repo_root(), cache=Path(getattr(args, "cache", None) or os.environ["UV_CACHE_DIR"]), tools=getattr(args, "tools", None), ) return finish_native(prepared, getattr(args, "frontends", {})) except (InstallError, FeatureProbeError) as exc: print(f"✗ {exc}") return 1 def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", required=True) parser.add_argument("--ref", default="HEAD") parser.add_argument("--source", type=Path, required=True) parser.add_argument("--tui", type=Path) parser.add_argument("--web", type=Path) parser.add_argument("--webapp", type=Path) args = parser.parse_args() args.frontends = {name: path for name in ("tui", "web", "webapp") if (path := getattr(args, name)) is not None} return _stage_native(args) if __name__ == "__main__": raise SystemExit(main())