mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 02:46:38 +03:00
29267cf49c
A bare shutil.which outside hermes_platform fails tests/test_managed_runtime_resolution.py; locate_command is the reviewed resolver.
250 lines
14 KiB
Python
250 lines
14 KiB
Python
"""``hermes update --force``: stop every gateway this update owns at once, without a drain.
|
||
|
||
The default pre-swap pause (``update_cmd_posix_pause._stop_gateways``) asks each gateway to finish
|
||
its in-flight work first and waits up to the restart budget (30 min by default). ``--force`` is the
|
||
explicit "stop everything now" the user asked for: it names what each gateway was running, stops it
|
||
immediately, kills anything still alive after a short bound, then lets the update and the normal
|
||
restart run.
|
||
|
||
A supervised gateway (systemd unit, launchd job) is stopped through its supervisor from the start:
|
||
that stop is what keeps ``Restart=``/``KeepAlive`` from bringing it back on the old tree, and it is
|
||
the path that carries the privilege (``sudo -n``) a system unit needs. A bare gateway takes the
|
||
force contract (``pause-for-update {"force": true}``) when it answers it, else the planned-stop
|
||
marker + SIGTERM. Either way the gateway enters ``stop()`` at once: running turns are interrupted,
|
||
tool and background processes killed (``process_registry.kill_all``) and background delegations
|
||
interrupted as resumable (``async_delegation.interrupt_all``). Gateways are stopped concurrently,
|
||
each against its own deadline measured from its own stop request.
|
||
|
||
A gateway that had to be killed never ran that ``kill_all``: its background processes live in
|
||
their own systemd scopes or sessions, so they outlive it. :func:`_sweep_left_behind` stops what its
|
||
process checkpoints still name as its own (``owner_pid``). A stop that does not settle (the supervisor's stop job still
|
||
running, the gateway or one of its scopes still alive after the kill) raises ``StopUnsettled``, which aborts
|
||
the update before the checkout moves: a rollback start would queue behind that stop job.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import signal
|
||
import threading
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from contextlib import suppress
|
||
from pathlib import Path
|
||
|
||
#: How long a gateway gets to run its own immediate stop before it is killed.
|
||
FORCE_KILL_AFTER_S = 12.0
|
||
#: After the kill: how long to wait for the process (and its supervisor's stop job) to settle.
|
||
_SETTLE_S = 5.0
|
||
#: After the kill: how long the supervisor's stop job gets to settle once its unit is empty.
|
||
_STOP_JOB_SETTLE_S = 30.0
|
||
|
||
|
||
def _checkpoint_homes(entry: dict) -> list[Path]:
|
||
"""Homes whose ``processes.json`` this gateway's processes are recorded in: its own and, for the
|
||
host multiplexer, every profile it serves (a served profile's turn checkpoints in that home).
|
||
Read while the gateway is alive: the host record names only a live owner."""
|
||
homes = [Path(entry["home"])] if entry.get("home") else []
|
||
with suppress(Exception):
|
||
from gateway.host_topology import host_gateway_topology
|
||
from hermes_cli.profiles import get_profile_dir
|
||
topology = host_gateway_topology()
|
||
if topology is not None and topology.pid == entry["pid"]:
|
||
homes += [get_profile_dir(name) for name in topology.profiles]
|
||
unique: dict[str, Path] = {}
|
||
for home in homes:
|
||
unique.setdefault(os.path.realpath(home), home)
|
||
return list(unique.values())
|
||
|
||
|
||
def _kill_scope(unit: str) -> None:
|
||
"""SIGKILL every process in a worker scope: what a timed-out ``systemctl stop`` left (a detached
|
||
child ignoring SIGTERM). Same user bus and client as ``process_registry._stop_systemd_unit``."""
|
||
import subprocess
|
||
from hermes_platform.resolver import locate_command
|
||
from tools.process_registry import systemd_user_bus_env
|
||
if not (found := locate_command("systemctl").command):
|
||
return
|
||
with suppress(OSError, subprocess.TimeoutExpired):
|
||
subprocess.run([*found, "--user", "kill", "--signal=SIGKILL", unit], capture_output=True, timeout=15,
|
||
stdin=subprocess.DEVNULL, env=systemd_user_bus_env(), check=False)
|
||
|
||
|
||
def _sweep_one(record: dict) -> tuple[bool, str | None]:
|
||
"""Stop one checkpointed process (its tree, then its scope): ``(stopped, leftover)``. *stopped*:
|
||
it was ours and is gone; *leftover*: its scope that is still not stopped after SIGKILL.
|
||
A PID whose start time does not match the recorded one was recycled and is never signalled."""
|
||
from tools.process_registry import ProcessRegistry, _stop_systemd_unit
|
||
pid, born = record.get("pid"), record.get("host_start_time")
|
||
ours = isinstance(pid, int) and born is not None and ProcessRegistry._host_pid_is_ours(pid, born)
|
||
if ours:
|
||
ProcessRegistry._terminate_host_pid(pid, born)
|
||
leftover = None
|
||
if (unit := record.get("systemd_unit")) and not _stop_systemd_unit(unit): # reaps double-forked descendants
|
||
_kill_scope(unit)
|
||
leftover = None if _stop_systemd_unit(unit) else unit
|
||
return ours and not ProcessRegistry._host_pid_is_ours(pid, born), leftover
|
||
|
||
|
||
def _sweep_left_behind(homes: list[Path], owner_pid: int) -> tuple[int, int, list[str]]:
|
||
"""What the killed gateway's ``stop()`` would have killed (``kill_all(source="gateway_shutdown")``:
|
||
persisted jobs included): the host processes its checkpoints name as written by *owner_pid*.
|
||
``processes.json`` is shared by every interpreter on the home (an ACP editor's background
|
||
commands too), so an entry another process wrote, or one with no ``owner_pid`` (written before
|
||
it was recorded), is not this gateway's and is left alone.
|
||
``(stopped, skipped, leftover scopes)``."""
|
||
records, skipped = [], 0
|
||
for home in homes:
|
||
with suppress(OSError, ValueError):
|
||
listed = json.loads((home / "processes.json").read_text(encoding="utf-8-sig"))
|
||
host = [r for r in listed if isinstance(r, dict) and (r.get("pid_scope") or "host") == "host"]
|
||
owned = [r for r in host if r.get("owner_pid") == owner_pid]
|
||
records += owned
|
||
skipped += len(host) - len(owned)
|
||
if not records:
|
||
return 0, skipped, []
|
||
with ThreadPoolExecutor(max_workers=min(8, len(records)), thread_name_prefix="update-force-sweep") as pool:
|
||
results = list(pool.map(_sweep_one, records))
|
||
return sum(stopped for stopped, _ in results), skipped, [unit for _, unit in results if unit]
|
||
|
||
|
||
def _live_work(entry: dict) -> list | None:
|
||
"""What the gateway is running right now: the live ``active-work`` verb, else its state file."""
|
||
from hermes_cli.update_cmd_drain_report import read_active_work
|
||
if not entry.get("home"):
|
||
return None
|
||
home = Path(entry["home"])
|
||
with suppress(Exception):
|
||
from gateway.control_socket import query_gateway_control
|
||
answer = query_gateway_control(home, "active-work", timeout=3.0)
|
||
if isinstance(answer, dict) and isinstance(answer.get("active_work"), list):
|
||
return list(answer["active_work"])
|
||
return read_active_work(home)
|
||
|
||
|
||
def _ask_force_stop(entry: dict) -> bool:
|
||
"""True when the gateway took a force stop over its control socket (``ack["force"] is True``).
|
||
An older gateway either rejects the parameter or answers with a drain ACK without the key;
|
||
neither counts, so the signal path stops it."""
|
||
if not entry.get("home"):
|
||
return False
|
||
from gateway.control_socket import query_gateway_control
|
||
with suppress(Exception):
|
||
ack = query_gateway_control(Path(entry["home"]), "pause-for-update", params={"force": True})
|
||
return bool(isinstance(ack, dict) and ack.get("force") is True
|
||
and (ack.get("pausing") or ack.get("already_stopping")))
|
||
return False
|
||
|
||
|
||
def _signal_stop(entry: dict) -> None:
|
||
"""Planned-stop marker + SIGTERM: the gateway's immediate ``stop()``, read as a planned stop."""
|
||
from hermes_cli.update_cmd_windows import _write_update_planned_stop_marker
|
||
if entry.get("home"):
|
||
_write_update_planned_stop_marker(Path(entry["home"]), entry["pid"])
|
||
with suppress(ProcessLookupError, PermissionError):
|
||
os.kill(entry["pid"], signal.SIGTERM)
|
||
|
||
|
||
def _kill(entry: dict) -> None:
|
||
"""SIGKILL the gateway: through systemd for a unit (its privileged path reaches a system
|
||
unit another account runs), else directly."""
|
||
from hermes_cli.update_cmd_posix_pause import _manage_cmd
|
||
if entry.get("kind") == "systemd" and (cmd := _manage_cmd(entry["scope"])) is not None:
|
||
from hermes_cli.update_cmd_fleet import _systemctl
|
||
_systemctl(cmd + ["kill", "--signal=SIGKILL", entry["unit"]], timeout=10)
|
||
return
|
||
with suppress(ProcessLookupError, PermissionError):
|
||
os.kill(entry["pid"], getattr(signal, "SIGKILL", signal.SIGTERM))
|
||
|
||
|
||
def _describe(entry: dict, work: list | None) -> str:
|
||
from hermes_cli.update_cmd_drain_report import describe_active_work_unit
|
||
from hermes_cli.update_cmd_posix_pause import _names
|
||
name = _names([entry])
|
||
if not work:
|
||
return f" ⏹ {name}: stopped by --force (no active work reported)"
|
||
home = Path(entry["home"]) if entry.get("home") else None
|
||
units = "; ".join(describe_active_work_unit(u, home) for u in work if isinstance(u, dict))
|
||
return f" ⏹ {name}: stopped by --force, interrupting {len(work)} unit(s): {units}"
|
||
|
||
|
||
def _stop_one(token: dict, entry: dict, lock: threading.Lock) -> tuple[bool, str] | None:
|
||
"""Stop *entry* within its own deadline. ``None`` when it is gone (and, if supervised, its
|
||
supervisor is stopped), else ``(unsettled, why)``: *unsettled* when its stop is still running."""
|
||
from hermes_cli import update_pause_record as pause_record
|
||
from hermes_cli.update_cmd_posix_pause import _names, _stop_at_once, _wait_gone
|
||
work = _live_work(entry)
|
||
homes = _checkpoint_homes(entry)
|
||
with lock: # the pause record is one file shared by every entry
|
||
pause_record.mark_stop_sent(token, entry["pid"]) # before the request: a crash keeps it owed
|
||
supervisor, failed = None, []
|
||
if entry.get("kind"):
|
||
def _supervisor_stop() -> None:
|
||
try:
|
||
_stop_at_once(entry)
|
||
except Exception as exc: # health: allow BLE001 -- reported as this entry's refusal below
|
||
failed.append(str(exc))
|
||
supervisor = threading.Thread(target=_supervisor_stop, name=f"force-stop-{entry['pid']}", daemon=True)
|
||
supervisor.start()
|
||
elif not _ask_force_stop(entry):
|
||
_signal_stop(entry)
|
||
print(_describe(entry, work), flush=True)
|
||
# One deadline for the whole service: the gateway exiting is not enough while its supervisor's
|
||
# stop job still waits on other processes in the unit (a tool child ignoring SIGTERM).
|
||
deadline = time.monotonic() + FORCE_KILL_AFTER_S
|
||
born = {entry["pid"]: entry.get("ct")}
|
||
alive = bool(_wait_gone([entry["pid"]], FORCE_KILL_AFTER_S, born))
|
||
if supervisor is not None:
|
||
supervisor.join(max(0.0, deadline - time.monotonic()))
|
||
if alive or (supervisor is not None and supervisor.is_alive()):
|
||
print(f" ⚠ gateway PID {entry['pid']} still stopping {FORCE_KILL_AFTER_S:.0f}s after the stop — killing it",
|
||
flush=True)
|
||
_kill(entry) # a unit: SIGKILL to every process in its cgroup, so the stop job can settle
|
||
swept, skipped, leftover = _sweep_left_behind(homes, entry["pid"]) # its scopes/sessions: outside that cgroup
|
||
if swept:
|
||
print(f" ⏹ {_names([entry])}: also stopped {swept} background process(es) it left behind", flush=True)
|
||
if skipped:
|
||
print(f" ℹ {_names([entry])}: left {skipped} recorded background process(es) running that another "
|
||
"process (or an older Hermes) recorded", flush=True)
|
||
if leftover:
|
||
return True, (f"background scope(s) {', '.join(leftover)} of gateway PID {entry['pid']} still running "
|
||
f"after SIGKILL (check `systemctl --user status {leftover[0]}`)")
|
||
if _wait_gone([entry["pid"]], _SETTLE_S, born):
|
||
return True, (f"gateway PID {entry['pid']} is still running after SIGKILL "
|
||
f"(check `ps -o pid,stat,cmd -p {entry['pid']}`; a D-state process waits on I/O)")
|
||
if supervisor is not None:
|
||
# Keep ownership of the stop job until it settles: a rollback start would queue behind it.
|
||
supervisor.join(_STOP_JOB_SETTLE_S)
|
||
if supervisor.is_alive():
|
||
return True, f"{_supervisor_name(entry)} is still stopping gateway PID {entry['pid']} ({_check_hint(entry)})"
|
||
return (False, failed[0]) if failed else None
|
||
|
||
|
||
def _supervisor_name(entry: dict) -> str:
|
||
return f"systemd's stop job for {entry['unit']}" if entry.get("kind") == "systemd" \
|
||
else f"launchd's bootout of {entry.get('label')}"
|
||
|
||
|
||
def _check_hint(entry: dict) -> str:
|
||
if entry.get("kind") == "systemd":
|
||
user = " --user" if entry.get("scope") == "user" else ""
|
||
return f"check `systemctl{user} status {entry['unit']}` and `systemctl{user} list-jobs`"
|
||
return f"check `launchctl print {entry.get('domain')}/{entry.get('label')}`"
|
||
|
||
|
||
def force_stop_gateways(token: dict, entries: list[dict]) -> None:
|
||
"""Stop every entry now, concurrently; kill each survivor ``FORCE_KILL_AFTER_S`` after its own
|
||
stop request. Raises ``StopUnsettled`` when any stop is still running (the update must not
|
||
restart or swap past it), else ``PauseRefused`` naming every entry that is not stopped."""
|
||
from hermes_cli.update_cmd_posix_pause import PauseRefused, StopUnsettled
|
||
if not entries:
|
||
return
|
||
lock = threading.Lock()
|
||
with ThreadPoolExecutor(max_workers=len(entries), thread_name_prefix="update-force") as pool:
|
||
refusals = [r for r in pool.map(lambda e: _stop_one(token, e, lock), entries) if r]
|
||
if unsettled := [why for still_running, why in refusals if still_running]:
|
||
raise StopUnsettled("; ".join(unsettled))
|
||
if refusals:
|
||
raise PauseRefused("; ".join(why for _, why in refusals))
|