"""Bake the payload's bytecode cache at staging time. A sealed payload ships no __pycache__: the runtime redirects bytecode writes to a user-level cache (signature-breaking on macOS, read-only mount on AppImage/MSIX), so every fresh install pays a cold-compile stall on first launch. Baking turns that into a warm read: * compileall runs with the payload's OWN staged interpreter (a pyc minted by any other interpreter is silently ignored — the magic tag is part of the cache filename); * the roots are the agent tree, both site-packages and the staged interpreter's own library: a payload Python spawned without a cache prefix (Desktop's ticket bridge; ``-I`` drops an inherited one) imports the stdlib too; * ``--invalidation-mode unchecked-hash``: the pyc is trusted without source validation and never rewritten, so staging-repack mtimes cannot invalidate it and a stale source cannot trigger a rewrite into the sealed tree (on macOS that rewrite breaks the app's code signature, and Gatekeeper then holds the next update); * the baked pycs are chmodded owner-writable (0o644) BEFORE packaging: Squirrel.Mac's ShipIt clears the quarantine attribute off every file in the .app and aborts the whole update with EACCES on any read-only one (#138659), and on Windows a read-only pyc stopped the next prepare's rmtree (#137717). A read-only bit never blocked a rewrite anyway: CPython replaces a pyc through its writable __pycache__ dir. What keeps default imports (all the launcher does) off these pycs is unchecked-hash plus the asserted coverage below; * the runtime's sys.pycache_prefix redirect is dropped when the baked marker is present (the prefix relocates READS too — see scripts/build/launcher_wrapper.py), so imports read the source- adjacent baked pycs, which is Python's default multi-root lookup: payload modules read theirs; plugin/user modules keep caching beside their own sources under HERMES_HOME. Coverage is the performance contract: every parseable module under the baked roots must have a pyc. Unparseable fixtures (deliberately invalid test data) are counted and skipped, not failed. """ import subprocess from pathlib import Path MARKER = ".hermes-baked-pycache" def _interpreter_stdlib(root: Path, python: Path) -> Path: """The staged interpreter's own library (``Lib`` beside ``python.exe``, ``lib/python3.N`` above ``bin/``), found by CPython's landmark ``os.py``.""" home = python.parent found = [lib for lib in [home / "Lib", *sorted((home.parent / "lib").glob("python3.*"))] if (lib / "os.py").is_file()] if len(found) != 1: raise ValueError(f"expected exactly one standard library beside {python}, found {len(found)}") if not found[0].resolve().is_relative_to(root): raise ValueError(f"the staged interpreter's standard library is outside the payload: {found[0]}") return found[0] def _import_roots(root: Path, python: Path) -> list[Path]: """The payload's import roots, in launcher order (repo, venv site, PM), then the stdlib.""" roots = [root / "hermes-agent"] for source_root in roots + [root / "venv", root / "pm-runtime"]: if not source_root.is_dir(): raise FileNotFoundError(f"missing import root: {source_root}") sites = sorted((root / "venv").glob("lib/python*/site-packages")) + \ sorted((root / "venv").glob("Lib/site-packages")) if len(sites) != 1: raise ValueError(f"expected exactly one staged venv site-packages, found {len(sites)}") roots.append(sites[0]) pm_sites = sorted((root / "pm-runtime").glob("lib/python*/site-packages")) + \ sorted((root / "pm-runtime").glob("Lib/site-packages")) if len(pm_sites) != 1: raise ValueError(f"expected exactly one PM runtime site-packages, found {len(pm_sites)}") roots.append(pm_sites[0]) roots.append(_interpreter_stdlib(root, python)) return roots def _uncovered(root: Path) -> tuple[list[Path], int]: """Parseable .py files without a pyc, and unparseable fixtures skipped.""" from importlib.util import cache_from_source missing: list[Path] = [] unparseable = 0 for path in root.rglob("*.py"): if "__pycache__" in path.parts: continue if Path(cache_from_source(path)).exists(): continue try: compile(path.read_bytes(), str(path), "exec") except SyntaxError: # Deliberately invalid test fixtures ship in the snapshot; they # can never be imported, so no pyc is expected. unparseable += 1 else: missing.append(path) return missing, unparseable def bake_bytecode(root: Path, python: Path) -> dict: """Compile every payload module with the staged interpreter and seal the caches owner-writable. ``root`` is the payload root; ``python`` MUST be the payload's own staged interpreter binary.""" root = Path(root).resolve() staged = Path(python).absolute() python = staged.resolve() if not python.is_file(): raise FileNotFoundError(f"staged interpreter is missing: {python}") roots = _import_roots(root, staged) total = 0 for source_root in roots: # -f: compileall keeps a timestamp pyc whose mtime still matches (uv's, the build's # own imports'), and those ship stale once packaging moves the sources' mtimes. subprocess.run( [str(python), "-I", "-m", "compileall", "-q", "-f", "--invalidation-mode", "unchecked-hash", str(source_root)], check=True, timeout=30 * 60, stdin=subprocess.DEVNULL) missing, _unparseable = _uncovered(source_root) if missing: sample = ", ".join(str(m.relative_to(root)) for m in missing[:5]) raise ValueError(f"{len(missing)} payload modules have no bytecode pyc " f"({sample}…): a fresh install would cold-compile them " "every launch") total += sum(1 for _ in source_root.rglob("__pycache__/*.pyc")) # Pycs get owner write BEFORE packaging: Squirrel.Mac's ShipIt clears # the quarantine attribute from every file in the .app and aborts the # whole update with EACCES on any file lacking it (#138659) — a # read-only pyc wedges every macOS update at the swap step. Default # imports still leave these pycs alone: unchecked-hash never # revalidates against the source and asserted coverage leaves no cache # miss to write. `--check-hash-based-pycs always` or `-O` can still # write here, as they could when the pycs were read-only. for source_root in roots: for cache in source_root.rglob("__pycache__"): for pyc in cache.iterdir(): if pyc.is_file(): pyc.chmod(0o644) (root / MARKER).write_text("unchecked-hash\n", encoding="utf-8") return {"modules": total, "marker": str(root / MARKER)}