"""Container-boot reconciliation of per-profile gateway s6 services. Wired into the image as /etc/cont-init.d/02-reconcile-profiles. Runs as root after 01-hermes-setup (the stage2 hook) has chowned the volume and seeded $HERMES_HOME, but before s6-rc starts user services. """ from __future__ import annotations import json import logging import os import re from dataclasses import dataclass from pathlib import Path from typing import Literal, Sequence from hermes_cli.gateway_multiplex_s6 import AUTOSTART_STATES as _AUTOSTART_STATES, fold_named_slot_intent log = logging.getLogger(__name__) # Older installs only have gateway_state; newer lifecycle commands persist desired_state separately # so a transient runtime state can't erase operator intent. # Transient sub-states of a RUNNING gateway (not an operator stop, not a failed boot). A gateway # hard-killed in one of them with no `desired_state` would otherwise stay DOWN on every later boot # (observed: staging stranded at `draining`); map them to `running`, mirroring gateway/run.py. # `starting` / `startup_failed` are excluded: auto-restarting a mid-boot death is the crash-loop. # A gateway only ever reaches these while it is up and serving, so they are NOT an operator stop and NOT a # failed boot: - `draining` — written by the drain watcher / scale-to-zero go-dormant path when an # in-flight quiesce begins (gateway/run.py). - `degraded` — written when the gateway comes up with some # platforms queued for retry, then "falls through to the normal running state" (gateway/run.py #5196): the # process is up, serving cron + whatever platforms connected, and the reconnect watcher takes the rest from # there. When a gateway is hard-killed *while in one of these states* (a container/VM recreate SIGTERMs it # before `_stop_impl` reaches its terminal-state persist), the last value left in gateway_state.json is the # transient sub-state. With no explicit `desired_state` to fall back to, treating that literal value as the # autostart intent would leave the gateway DOWN on every subsequent boot — the gateway never comes back, the # dashboard is up but messaging stays dark (observed on a relay-opted-in staging instance stranded at # `draining`, 2026-06; `degraded` is the same wedge class). Map these transient sub-states to `running` so a # stranded marker reads as the run-intent it actually represents. This mirrors gateway/run.py's #42675 # handling, which persists `running` (not the mid-shutdown `draining`) when an unexpected signal tears the # gateway down — extended here to the case where the gateway died before it could persist anything at all. _TRANSIENT_RUNNING_STATES = frozenset({"draining", "degraded"}) # Container-namespaced state is garbage post-restart (an equal PID is a different process). _STALE_RUNTIME_FILES = ("gateway.pid", "processes.json") ReconcileActionLabel = Literal["started", "registered", "skipped"] @dataclass(frozen=True) class ReconcileAction: """One profile's outcome from a single reconciliation pass.""" profile: str prior_state: str | None action: ReconcileActionLabel # "clean" / "unclean" (sentinel still says running — SIGKILL/OOM/VM death) / "unknown". Boot # is the one place that can stamp a violent death into the volume-persisted log. prior_exit: str = "unknown" #: A named slot whose autostart intent was inherited by the root slot (multiplex-only), or — #: on the root slot — True when it is starting because of one. folded_into_root: bool = False def _slot_action(profile: str, profile_dir: Path, prior_state: str | None, start: bool, *, folded_into_root: bool = False) -> ReconcileAction: return ReconcileAction(profile=profile, prior_state=prior_state, action="started" if start else "registered", prior_exit=_read_prior_exit_label(profile_dir), folded_into_root=folded_into_root) def _named_profile_dirs(hermes_home: Path) -> list[tuple[str, Path]]: """Every real named profile under ``$HERMES_HOME/profiles`` (``SOUL.md`` is the marker).""" profiles_root = hermes_home / "profiles" if not profiles_root.is_dir(): return [] found: list[tuple[str, Path]] = [] for entry in sorted(profiles_root.iterdir()): if not entry.is_dir() or not (entry / "SOUL.md").exists(): continue # "default" is reserved for the root profile slot. if entry.name == "default": log.warning("profiles/default/ exists — skipping to avoid colliding with the " "reserved root-profile s6 slot") continue found.append((entry.name, entry)) return found def reconcile_profile_gateways( *, hermes_home: Path, scandir: Path, dry_run: bool = False, container_argv: Sequence[str] | None = None) -> list[ReconcileAction]: """Recreate s6 service registrations for every persistent profile. Always registers a ``gateway-default`` slot for the root profile (the implicit profile at the top of ``$HERMES_HOME``): ``hermes_cli.gateway`` maps an empty profile suffix to it, so it is what ``hermes gateway start`` (no ``-p``) targets. Without it, bare ``hermes gateway start`` inside the container would land on ``s6-svc -u /run/hermes-service/gateway-default`` → uncaught ``CalledProcessError`` → traceback to the user (PR #30136 review). """ actions: list[ReconcileAction] = [] # ONE multiplexing gateway per container: named slots are registered (so `hermes -p X gateway # start` has a target and `s6-svstat` can report them) but are never booted from their persisted # run intent, except a `gateway.standalone` profile's (below). # This is the s6 leg of the multiplex-only convergence: an image upgraded from a release that # booted N per-profile slots comes back up with one multiplexing root gateway and no manual # step, instead of N processes fighting over the same profiles. # # It used to be gated on `gateway.multiplex_profiles`, which made the UNSET default (on) boot # the slots anyway — the container shipped the opt-out topology by accident. The key is retired # as a topology switch (hermes_cli/gateway_multiplex_mode.py), so there is nothing to read. named = [(name, entry, _read_desired_state(entry)) for name, entry in _named_profile_dirs(hermes_home)] # The exception is a `gateway.standalone` profile: the root multiplexer never serves it # (profiles_to_serve excludes it), so it boots its own slot from its own intent, beside the root. from hermes_cli.profiles import profile_is_standalone standalone = {name for name, entry, _prior in named if profile_is_standalone(entry)} # A legacy `gateway run` container with no state yet seeds `running` (pre-s6 behavior). legacy_default_state = _maybe_migrate_legacy_gateway_run_state( hermes_home, container_argv=container_argv, dry_run=dry_run) default_prior_state = legacy_default_state or _read_desired_state(hermes_home) # The root slot INHERITS every named slot's autostart intent, because it is the one process # that serves them. Without this an image only ever driven as `hermes -p coder gateway start` # booted with ZERO gateways: it has no root state (or "stopped"), every named slot is now # registered down unconditionally, and every action reported "registered" — a container that # looks healthy while nothing is listening. fold = fold_named_slot_intent( default_prior_state, ((name, prior) for name, _dir, prior in named if name not in standalone)) folded, default_should_start = list(fold.folded), fold.root_should_start # A foreground `gateway run --no-supervise` CMD IS the container's one gateway; booting the # s6 root slot from persisted intent as well puts two pollers on one bot token (#152152). # A standalone profile's foreground gateway serves only that profile, so the root slot stays. foreground = _foreground_gateway_profile( tuple(container_argv) if container_argv is not None else _read_container_argv(), hermes_home) if foreground is not None and foreground not in standalone: default_should_start = False if folded and default_prior_state not in _AUTOSTART_STATES: log.warning("%s", boot_notice(folded, foreground=not default_should_start)) if not dry_run: _cleanup_stale_runtime_files(hermes_home) _register_service(scandir, "default", start=default_should_start) actions.append(_slot_action("default", hermes_home, default_prior_state, default_should_start, folded_into_root=bool(folded))) for name, entry, prior_state in named: # Registered down unless standalone: any other started named slot IS a second gateway on this host. # ...and a standalone profile the CMD already serves in the foreground stays down too. should_start = name in standalone and prior_state in _AUTOSTART_STATES and name != foreground if not dry_run: _cleanup_stale_runtime_files(entry) _register_service(scandir, name, start=should_start) actions.append(_slot_action(name, entry, prior_state, should_start, folded_into_root=name in folded)) if not dry_run: _write_reconcile_log(hermes_home, actions) return actions def boot_notice(folded: Sequence[str], *, foreground: bool = False) -> str: """One line naming the profiles whose autostart intent the root slot took over, or, when the container CMD is itself the foreground gateway, that it serves them and the root slot stays down.""" served_by = ("the foreground gateway (the container CMD) serves them, so the ROOT slot (gateway-default) " "stays down" if foreground else "the ROOT slot (gateway-default) was started instead and multiplexes them") return ("reconcile: profile gateway(s) " + ", ".join(sorted(folded)) + " asked to autostart; one gateway per container serves every non-standalone profile, so " + served_by + ".") def _maybe_migrate_legacy_gateway_run_state( hermes_home: Path, *, container_argv: Sequence[str] | None, dry_run: bool) -> str | None: """Seed root gateway_state for pre-s6 `gateway run` containers (the tini image let users run the gateway as the container command; post-s6 it would register down and never start).""" state_file = hermes_home / "gateway_state.json" if state_file.exists(): return None if os.environ.get("HERMES_GATEWAY_NO_SUPERVISE", "").lower() in ("1", "true", "yes"): return None argv = tuple(container_argv) if container_argv is not None else _read_container_argv() if not _is_legacy_gateway_run_request(argv): return None if not dry_run: import time state_file.write_text(json.dumps({ "gateway_state": "running", "desired_state": "running", "timestamp": int(time.time()), "migrated_from": "legacy-container-cmd", }) + "\n", encoding="utf-8") return "running" def _cmdline_argv(cmdline: Path) -> tuple[str, ...]: raw = cmdline.read_bytes() return tuple(part.decode("utf-8", "replace") for part in raw.split(b"\0") if part) def _read_container_argv() -> tuple[str, ...]: """Best-effort argv of the container's main program (the one holding ``main-wrapper.sh``): PID 1 first (s6 v2 ``/init``), then every ``/proc/*/cmdline`` (s6 v3 ``s6-svscan``).""" def _cmdlines(): yield Path("/proc/1/cmdline") try: for entry in Path("/proc").iterdir(): if entry.name.isdigit(): yield entry / "cmdline" except OSError: return for cmdline in _cmdlines(): try: argv = _cmdline_argv(cmdline) except OSError: continue if any("main-wrapper.sh" in part for part in argv): return argv return () def _strip_container_argv_prefix(argv: Sequence[str]) -> list[str]: """Strip the s6/wrapper prefix off the container argv, leaving the hermes args. Drops everything through the ``main-wrapper.sh`` token — the stable boundary the image owns — rather than peeling tokens positionally (which broke on the s6 v2→v3 bump). """ args = list(argv) wrapper_idx = next((i for i, a in enumerate(args) if a.endswith("main-wrapper.sh")), None) if wrapper_idx is not None: args = args[wrapper_idx + 1 :] elif args and Path(args[0]).name == "init": # defensive: `init` with no wrapper token args = args[1:] if args and args[0].endswith("entrypoint-dispatch.sh"): # non-PID-1 dispatch shim args = args[1:] if args and Path(args[0]).name == "hermes": # the wrapper re-execs `hermes ` args = args[1:] return _strip_python_launcher(args) _PYTHON_RE = re.compile(r"python(\d+(\.\d+)*)?(\.exe)?") _PYTHON_OPERAND_FLAGS = frozenset({"-X", "-W"}) def _strip_python_launcher(args: list[str]) -> list[str]: """Drop a ``python [opts] -m hermes_cli.main`` / ``python [opts] .../hermes_cli/main.py`` launcher: main-wrapper.sh execs an executable CMD as-is, and that process is the same CLI.""" if not args or not _PYTHON_RE.fullmatch(Path(args[0]).name): return args i = 1 while i < len(args) and args[i].startswith("-") and args[i] != "-m": i += 2 if args[i] in _PYTHON_OPERAND_FLAGS else 1 if args[i:i + 2] == ["-m", "hermes_cli.main"]: return args[i + 2:] if i < len(args) and args[i].replace("\\", "/").endswith("hermes_cli/main.py"): return args[i + 1:] return args def _is_legacy_gateway_run_request(argv: Sequence[str]) -> bool: """True for Docker commands equivalent to `gateway run`.""" from hermes_cli._parser import gateway_run_no_supervise args = _strip_container_argv_prefix(argv) if gateway_run_no_supervise(args): return False return len(args) >= 2 and args[0] == "gateway" and args[1] == "run" def _foreground_gateway_profile(argv: Sequence[str], hermes_home: Path) -> str | None: """The profile a container CMD's own foreground gateway serves, or None when the CMD runs no gateway outside s6. Foreground means ``gateway run`` with ``--no-supervise`` or HERMES_GATEWAY_NO_SUPERVISE, parsed as the CLI does: the profile is ``-p``/``--profile`` wherever it sits, else the sticky ``active_profile`` the unsupervised run follows (#22502).""" import shlex from gateway.status import looks_like_gateway_command_line from hermes_cli._parser import gateway_run_no_supervise, scan_profile_flag_quietly args = _strip_container_argv_prefix(argv) if not looks_like_gateway_command_line(shlex.join(["hermes", *args])): return None scanned = scan_profile_flag_quietly(args) if scanned is None: # an invalid -p: the CMD exits 2 itself and runs no gateway return None profile, consumed, index = scanned if index is not None: # stripped before argparse, as main._apply_profile_override does args = args[:index] + args[index + consumed:] # Bare `gateway` also runs, but only `gateway run` takes `--no-supervise`. if not gateway_run_no_supervise(args) and os.environ.get("HERMES_GATEWAY_NO_SUPERVISE", "").lower() not in ("1", "true", "yes"): return None if profile is not None: return profile from hermes_cli.profiles import get_active_profile return get_active_profile(hermes_home) def _is_dashboard_container(argv: Sequence[str]) -> bool: """True when the container's command is the dashboard (which never supervises gateways).""" args = _strip_container_argv_prefix(argv) return bool(args) and args[0] == "dashboard" def _read_desired_state(profile_dir: Path) -> str | None: """Persisted ``desired_state`` (operator intent), else legacy ``gateway_state``; missing or unparseable files count as "no state" so a corrupt file can't bork boot.""" state_file = profile_dir / "gateway_state.json" if not state_file.exists(): return None try: data = json.loads(state_file.read_text(encoding="utf-8-sig")) except (OSError, json.JSONDecodeError): log.warning("could not read %s; treating as no prior state", state_file) return None desired_state = data.get("desired_state") if desired_state is not None: return desired_state gateway_state = data.get("gateway_state") return "running" if gateway_state in _TRANSIENT_RUNNING_STATES else gateway_state def _cleanup_stale_runtime_files(profile_dir: Path) -> None: """Remove PID-namespace-bound runtime files that would confuse the new gateway's checks.""" for name in _STALE_RUNTIME_FILES: (profile_dir / name).unlink(missing_ok=True) def _read_prior_exit_label(profile_dir: Path) -> str: """Exception-free ``lifecycle_ledger.read_prior_exit_label`` — forensics never block boot.""" try: from gateway.lifecycle_ledger import read_prior_exit_label return read_prior_exit_label(profile_dir) except Exception: return "unknown" def _write_exec(path: Path, content: str) -> None: path.write_text(content, encoding="utf-8") path.chmod(0o755) def _register_service(scandir: Path, profile: str, *, start: bool) -> None: """Recreate the s6 service slot for one profile. Mirrors ``S6ServiceManager.register_profile_gateway`` but sets start state via the ``down`` marker (cont-init.d runs before s6-svscan has a control socket). Built in a sibling temp dir and ``Path.replace``d into place so an interrupted write never leaves a half-built dir. This matches :meth:`S6ServiceManager.register_profile_gateway` (PR #30136 review item O4) — even though cont-init.d runs before s6-svscan starts scanning, an atomic publication keeps the contract uniform between the two registration paths and protects against a half-populated dir if the script is interrupted mid-write. """ import shutil from hermes_cli.service_manager import ( S6ServiceManager, _seed_supervise_skeleton, validate_profile_name) validate_profile_name(profile) service_dir = scandir / f"gateway-{profile}" # Dot-prefixed so s6-svscan skips the staging dir: a non-dotted name gets supervised AS ROOT # by a concurrent rescan, creating a root-owned ``supervise/`` → EACCES in the seed below. tmp_dir = service_dir.with_name("." + service_dir.name + ".tmp") if tmp_dir.exists(): shutil.rmtree(tmp_dir, ignore_errors=True) tmp_dir.mkdir(parents=True) try: (tmp_dir / "type").write_text("longrun\n", encoding="utf-8") # Manager's own rendering keeps both registration paths consistent; per-profile env # comes from the profile's config.yaml, so extra_env is empty. _write_exec(tmp_dir / "run", S6ServiceManager._render_run_script(profile, extra_env={})) _write_exec(tmp_dir / "finish", S6ServiceManager._render_finish_script()) (tmp_dir / "log").mkdir() _write_exec(tmp_dir / "log" / "run", S6ServiceManager._render_log_run(profile)) if not start: # `hermes -p gateway start` brings it up later (s6-svc -u) (tmp_dir / "down").touch() # Pre-create supervise/ with hermes ownership BEFORE publishing so s6-supervise inherits # it and runtime s6-svc calls as the hermes user won't EACCES. _seed_supervise_skeleton(tmp_dir) if service_dir.exists(): shutil.rmtree(service_dir) tmp_dir.replace(service_dir) # atomic publish except Exception: shutil.rmtree(tmp_dir, ignore_errors=True) raise # ~3000 lines ≈ a year of daily reboots on a 5-profile container; rotated to .1 when crossed. _LOG_ROTATE_BYTES = 256 * 1024 def _write_reconcile_log(hermes_home: Path, actions: list[ReconcileAction]) -> None: """Append one line per profile to $HERMES_HOME/logs/container-boot.log (rotated to ``.1``) — a separate greppable file for "why didn't my profile come back up". Size-bounded: when the file exceeds ``_LOG_ROTATE_BYTES`` (defaults to 256 KiB ≈ 3000 reconcile lines), the current file is renamed to ``container-boot.log.1`` (replacing any previous rotation) before the new entries are appended. This gives long- lived containers a soft cap of ~512 KiB across the two files without pulling in logrotate or s6-log machinery just for this one append-only file (PR #30136 review item O3). """ import time log_dir = hermes_home / "logs" log_dir.mkdir(parents=True, exist_ok=True) log_path = log_dir / "container-boot.log" try: if log_path.exists() and log_path.stat().st_size >= _LOG_ROTATE_BYTES: log_path.replace(log_dir / "container-boot.log.1") except OSError as exc: # non-fatal — keep appending rather than lose the entry log.warning("could not rotate %s: %s", log_path, exc) ts = time.strftime("%Y-%m-%dT%H:%M:%S%z") with log_path.open("a", encoding="utf-8") as f: for a in actions: f.write(f"{ts} profile={a.profile} prior_state={a.prior_state} " f"action={a.action} prior_exit={a.prior_exit} " f"folded_into_root={a.folded_into_root}\n") def main() -> int: """Entry point invoked from /etc/cont-init.d/02-reconcile-profiles.""" # A dashboard-only container must not reconcile: with a shared bind-mounted HERMES_HOME both # containers race to flock() the same s6-log files → "Resource busy" restart storm. Detected # from PID 1 argv, not an operator flag (a flag can be forgotten in a hand-written manifest). if _is_dashboard_container(_read_container_argv()): print("reconcile: skipping (dashboard container — does not need per-profile gateways)") return 0 from hermes_cli.service_manager import S6_DYNAMIC_SCANDIR hermes_home = Path(os.environ.get("HERMES_HOME", "/opt/data")) scandir = Path(os.environ.get("S6_PROFILE_GATEWAY_SCANDIR", str(S6_DYNAMIC_SCANDIR))) actions = reconcile_profile_gateways(hermes_home=hermes_home, scandir=scandir) folded = [a.profile for a in actions if a.profile != "default" and a.folded_into_root] if folded: print(boot_notice(folded, foreground=actions[0].action != "started")) for a in actions: print(f"reconcile: profile={a.profile} prior_state={a.prior_state} action={a.action}") return 0 if __name__ == "__main__": raise SystemExit(main())