mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 03:12:19 +03:00
58e0b85e91
A sealed macOS bundle is user-writable, so any payload Python that runs without
a cache prefix rewrites stale bytecode inside the signed app. codesign then
reports "file modified", spctl rejects the app, and the next Squirrel.Mac
update waits on a Gatekeeper prompt. The shipped Nov 10 canary (da06acacc9),
inventoried on a real Mac, had:
- stdlib: 1,061 modules, 140 timestamp pycs (all stale: they record the
2024-01-01 source mtime, the .py files carry the build time) and 921 with
no pyc at all. bake_bytecode never compiled the interpreter's own library.
- venv site-packages: all 25,439 pycs timestamp-based and stale; pm-runtime:
all 61. uv's --compile-bytecode writes timestamp pycs, and compileall
without -f keeps a timestamp pyc whose mtime still matches, even in
unchecked-hash mode, so the bake left them as they were. Packaging then
moved the sources' mtimes.
- hermes-agent: unchecked-hash, as intended.
On that build, Desktop's ticket helper rewrote 94 stdlib pycs inside the app,
and a `python -I` import of the gateway (which ignores PYTHONPYCACHEPREFIX)
wrote 248 (103 stdlib, 145 venv).
The bake now also compiles the staged interpreter's library, found by
CPython's own landmark (os.py) beside the binary and refused when it lies
outside the payload, and forces every root (-f) so existing timestamp pycs
become unchecked-hash too. The same payload baked with this change on the Mac:
every module in all four roots unchecked-hash, nothing stale or missing, and
neither the ticket helper nor the `python -I` import wrote a file. The bake
took 25 s there; the stdlib grows by about 16 MiB. The prepared-payload
digests skip __pycache__, so they are unchanged.
Windows and Linux install trees are read-only, so their payloads never hit the
seal break. The Windows launcher reads these source-adjacent pycs (it sets no
cache prefix for a baked payload), so there the change only removes
recompiles.
140 lines
6.8 KiB
Python
140 lines
6.8 KiB
Python
"""Bake the payload's bytecode cache at staging time.
|
|
|
|
A sealed payload ships no __pycache__: the runtime redirects bytecode
|
|
writes to a user-level cache (signature-breaking on macOS, read-only
|
|
mount on AppImage/MSIX), so every fresh install pays a cold-compile
|
|
stall on first launch. Baking turns that into a warm read:
|
|
|
|
* compileall runs with the payload's OWN staged interpreter (a pyc
|
|
minted by any other interpreter is silently ignored — the magic tag
|
|
is part of the cache filename);
|
|
* the roots are the agent tree, both site-packages and the staged
|
|
interpreter's own library: a payload Python spawned without a cache
|
|
prefix (Desktop's ticket bridge; ``-I`` drops an inherited one)
|
|
imports the stdlib too;
|
|
* ``--invalidation-mode unchecked-hash``: the pyc is trusted without
|
|
source validation and never rewritten, so staging-repack mtimes
|
|
cannot invalidate it and a stale source cannot trigger a rewrite
|
|
into the sealed tree (on macOS that rewrite breaks the app's code
|
|
signature, and Gatekeeper then holds the next update);
|
|
* the baked pycs are chmodded owner-writable (0o644) BEFORE packaging:
|
|
Squirrel.Mac's ShipIt clears the quarantine attribute off every file
|
|
in the .app and aborts the whole update with EACCES on any read-only
|
|
one (#138659), and on Windows a read-only pyc stopped the next
|
|
prepare's rmtree (#137717). A read-only bit never blocked a rewrite
|
|
anyway: CPython replaces a pyc through its writable __pycache__ dir.
|
|
What keeps default imports (all the launcher does) off these pycs is
|
|
unchecked-hash plus the asserted coverage below;
|
|
* the runtime's sys.pycache_prefix redirect is dropped when the baked
|
|
marker is present (the prefix relocates READS too — see
|
|
scripts/build/launcher_wrapper.py), so imports read the source-
|
|
adjacent baked pycs, which is Python's default multi-root lookup:
|
|
payload modules read theirs; plugin/user modules keep caching beside
|
|
their own sources under HERMES_HOME.
|
|
|
|
Coverage is the performance contract: every parseable module under
|
|
the baked roots must have a pyc. Unparseable fixtures (deliberately
|
|
invalid test data) are counted and skipped, not failed.
|
|
"""
|
|
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
MARKER = ".hermes-baked-pycache"
|
|
|
|
|
|
def _interpreter_stdlib(root: Path, python: Path) -> Path:
|
|
"""The staged interpreter's own library (``Lib`` beside ``python.exe``,
|
|
``lib/python3.N`` above ``bin/``), found by CPython's landmark ``os.py``."""
|
|
home = python.parent
|
|
found = [lib for lib in [home / "Lib", *sorted((home.parent / "lib").glob("python3.*"))]
|
|
if (lib / "os.py").is_file()]
|
|
if len(found) != 1:
|
|
raise ValueError(f"expected exactly one standard library beside {python}, found {len(found)}")
|
|
if not found[0].resolve().is_relative_to(root):
|
|
raise ValueError(f"the staged interpreter's standard library is outside the payload: {found[0]}")
|
|
return found[0]
|
|
|
|
|
|
def _import_roots(root: Path, python: Path) -> list[Path]:
|
|
"""The payload's import roots, in launcher order (repo, venv site, PM), then the stdlib."""
|
|
roots = [root / "hermes-agent"]
|
|
for source_root in roots + [root / "venv", root / "pm-runtime"]:
|
|
if not source_root.is_dir():
|
|
raise FileNotFoundError(f"missing import root: {source_root}")
|
|
sites = sorted((root / "venv").glob("lib/python*/site-packages")) + \
|
|
sorted((root / "venv").glob("Lib/site-packages"))
|
|
if len(sites) != 1:
|
|
raise ValueError(f"expected exactly one staged venv site-packages, found {len(sites)}")
|
|
roots.append(sites[0])
|
|
pm_sites = sorted((root / "pm-runtime").glob("lib/python*/site-packages")) + \
|
|
sorted((root / "pm-runtime").glob("Lib/site-packages"))
|
|
if len(pm_sites) != 1:
|
|
raise ValueError(f"expected exactly one PM runtime site-packages, found {len(pm_sites)}")
|
|
roots.append(pm_sites[0])
|
|
roots.append(_interpreter_stdlib(root, python))
|
|
return roots
|
|
|
|
|
|
def _uncovered(root: Path) -> tuple[list[Path], int]:
|
|
"""Parseable .py files without a pyc, and unparseable fixtures skipped."""
|
|
from importlib.util import cache_from_source
|
|
missing: list[Path] = []
|
|
unparseable = 0
|
|
for path in root.rglob("*.py"):
|
|
if "__pycache__" in path.parts:
|
|
continue
|
|
if Path(cache_from_source(path)).exists():
|
|
continue
|
|
try:
|
|
compile(path.read_bytes(), str(path), "exec")
|
|
except SyntaxError:
|
|
# Deliberately invalid test fixtures ship in the snapshot; they
|
|
# can never be imported, so no pyc is expected.
|
|
unparseable += 1
|
|
else:
|
|
missing.append(path)
|
|
return missing, unparseable
|
|
|
|
|
|
def bake_bytecode(root: Path, python: Path) -> dict:
|
|
"""Compile every payload module with the staged interpreter and seal the
|
|
caches owner-writable. ``root`` is the payload root; ``python`` MUST be the
|
|
payload's own staged interpreter binary."""
|
|
root = Path(root).resolve()
|
|
staged = Path(python).absolute()
|
|
python = staged.resolve()
|
|
if not python.is_file():
|
|
raise FileNotFoundError(f"staged interpreter is missing: {python}")
|
|
roots = _import_roots(root, staged)
|
|
total = 0
|
|
for source_root in roots:
|
|
# -f: compileall keeps a timestamp pyc whose mtime still matches (uv's, the build's
|
|
# own imports'), and those ship stale once packaging moves the sources' mtimes.
|
|
subprocess.run(
|
|
[str(python), "-I", "-m", "compileall", "-q", "-f",
|
|
"--invalidation-mode", "unchecked-hash", str(source_root)],
|
|
check=True, timeout=30 * 60, stdin=subprocess.DEVNULL)
|
|
missing, _unparseable = _uncovered(source_root)
|
|
if missing:
|
|
sample = ", ".join(str(m.relative_to(root)) for m in missing[:5])
|
|
raise ValueError(f"{len(missing)} payload modules have no bytecode pyc "
|
|
f"({sample}…): a fresh install would cold-compile them "
|
|
"every launch")
|
|
total += sum(1 for _ in source_root.rglob("__pycache__/*.pyc"))
|
|
# Pycs get owner write BEFORE packaging: Squirrel.Mac's ShipIt clears
|
|
# the quarantine attribute from every file in the .app and aborts the
|
|
# whole update with EACCES on any file lacking it (#138659) — a
|
|
# read-only pyc wedges every macOS update at the swap step. Default
|
|
# imports still leave these pycs alone: unchecked-hash never
|
|
# revalidates against the source and asserted coverage leaves no cache
|
|
# miss to write. `--check-hash-based-pycs always` or `-O` can still
|
|
# write here, as they could when the pycs were read-only.
|
|
for source_root in roots:
|
|
for cache in source_root.rglob("__pycache__"):
|
|
for pyc in cache.iterdir():
|
|
if pyc.is_file():
|
|
pyc.chmod(0o644)
|
|
(root / MARKER).write_text("unchecked-hash\n", encoding="utf-8")
|
|
return {"modules": total, "marker": str(root / MARKER)}
|