mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 02:46:38 +03:00
80350460d5
On a CPU-capped host (a 10% cgroup quota, a loaded runner) a cold `gateway run`
needs a few seconds of CPU that take minutes of wall time. The fixed 60 s
deadline of ensure_gateway_runtime (9c7dbc6e6f) then turns every cold
`hermes -z` into `Gateway starting: deadline`: main failed 3 of 3 runs at a
10% quota, each at about 61 s.
- A default ensure (no explicit timeout) extends its deadline while the
pending start's owner (the home's verified running gateway, else the
spawned child) keeps accruing CPU, at least 0.1 s per 15 s window, up to a
300 s cap. A control probe that times out on such an owner is retried
instead of ending the wait. Only a start this call is waiting on (one it
spawned, joined or saw pending, a multiplexer root included) is sampled: a
wedged READY gateway that spins still fails at 60 s, as before. An owner that makes no progress (stopped,
blocked on a lock) still fails at the base 60 s, and an explicit --timeout
stays a hard deadline. On macOS the one-off launchd job gets the moved
deadline too, or its poll() would stop reporting an exit past the original.
- A client's own idle-exit gateway imports the /api/ws handler on a worker
thread before READY, waiting at most 30 s for it, and an attach fetches the
handler on a worker thread too. Left to the first upgrade, that ~0.7 s CPU
import ran on the event loop inside the client's 10 s open timeout and
failed the attach right after READY on a starved host.
At a 10% quota a cold `hermes -z` now attaches and answers (3 of 3); with no
quota the attach time is unchanged (3.90 s vs main 3.92 s), and a SIGSTOPped
gateway still fails with `starting: deadline`.
94 lines
4.8 KiB
Python
94 lines
4.8 KiB
Python
"""Adaptive startup deadline for ``ensure_gateway_runtime``: a cold start that keeps burning CPU is
|
|
slow, not hung.
|
|
|
|
On a CPU-starved host (a cgroup quota, a loaded CI runner) a cold boot's few seconds of CPU can take
|
|
minutes of wall time, and a fixed deadline turns every such start into ``starting: deadline``. A
|
|
default ensure therefore waits past its base deadline while the start's owner process measurably
|
|
progresses, up to :data:`MAX_ADAPTIVE_TIMEOUT`. An owner that makes no CPU progress (blocked on a
|
|
lock, deadlocked, stopped) still fails at the base deadline, so a hung start is reported as promptly
|
|
as before. Progress is read on the home's verified running gateway, else the process this call
|
|
spawned (not its descendants: on Windows a uv venv launcher waits on its child, so a start there
|
|
keeps the base deadline until the gateway publishes its lock).
|
|
|
|
An explicit timeout is hard. That covers ``hermes gateway ensure/attach --timeout`` (Desktop runs it
|
|
under its own outer timer) and the Ink TUI bootstrap; the adaptive wait serves the in-process clients
|
|
(``hermes -z``, ``chat -q``, cron, kanban workers, ACP).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
import time
|
|
from pathlib import Path
|
|
|
|
# Hard ceiling on a default ensure's wait while its owner keeps progressing (a cold boot measured
|
|
# ~160 s of wall time at a 5% CPU quota).
|
|
MAX_ADAPTIVE_TIMEOUT = 300.0
|
|
# CPU the owner must accrue per window to count as progressing. A parked or stopped process accrues
|
|
# ~0; a boot even at a 5% quota accrues several times this.
|
|
PROGRESS_WINDOW_S = 15.0
|
|
_MIN_CPU_PER_WINDOW_S = 0.1
|
|
# A progressing start keeps the deadline one window (plus slack) ahead, so it survives until the next
|
|
# verdict; an owner that stalls after progressing fails within about one window. Sampled at most this
|
|
# often (a few lock-record reads and one psutil call).
|
|
_LEAD_S = PROGRESS_WINDOW_S + 5.0
|
|
_SAMPLE_EVERY_S = 1.0
|
|
|
|
|
|
def _starting_owner_pid(homes, child) -> int | None:
|
|
"""The pending start's process: a verified running gateway of one of ``homes`` (the profile's
|
|
own, or the multiplexer root this call started: runtime lock held, PID record matching a live
|
|
gateway of that home), else the child this call spawned. Never a bare lock record, which a
|
|
backup or restore holding the home also writes."""
|
|
from gateway.status import get_running_pid
|
|
for home in dict.fromkeys(homes):
|
|
if (pid := get_running_pid(Path(home) / "gateway.pid", cleanup_stale=False)) is not None:
|
|
return pid
|
|
return getattr(child, "pid", None)
|
|
|
|
|
|
def _cpu_seconds(pid: int) -> float | None:
|
|
import psutil
|
|
|
|
try:
|
|
times = psutil.Process(pid).cpu_times()
|
|
except (psutil.Error, OSError): # gone, or another user's (AccessDenied): no progress evidence
|
|
return None
|
|
return times.user + times.system
|
|
|
|
|
|
class StartRunway:
|
|
"""The ensure loop's deadline: ``timeout`` (hard), or the default base deadline that
|
|
:meth:`extend` pushes forward while the start's owner keeps progressing."""
|
|
|
|
def __init__(self, timeout: float | None, default: float):
|
|
self._adaptive = timeout is None
|
|
timeout = default if timeout is None else timeout
|
|
if not math.isfinite(timeout) or timeout <= 0:
|
|
raise ValueError("timeout must be finite and positive")
|
|
now = time.monotonic()
|
|
self.deadline = now + timeout
|
|
self._cap = now + max(timeout, MAX_ADAPTIVE_TIMEOUT)
|
|
self._window: tuple[int, float, float] | None = None # (owner pid, when, cpu) at window start
|
|
self._sampled = 0.0
|
|
|
|
def extend(self, homes, child) -> float:
|
|
"""The deadline after one pending iteration; never earlier than before, never past the cap.
|
|
Nothing to wait on yet (no ``homes``, no ``child``) is not a sample: it must not spend the
|
|
throttle that the start's first window needs. A due deadline is never throttled: it gets the
|
|
verdict sample whatever the last one's age."""
|
|
now = time.monotonic()
|
|
throttled = now - self._sampled < _SAMPLE_EVERY_S and now < self.deadline
|
|
if not self._adaptive or (not homes and child is None) or throttled:
|
|
return self.deadline
|
|
self._sampled = now
|
|
pid = _starting_owner_pid(homes, child)
|
|
cpu = _cpu_seconds(pid) if pid is not None else None
|
|
if pid is None or cpu is None or self._window is None or self._window[0] != pid:
|
|
# No reading, or a new owner: open a fresh window and claim nothing yet.
|
|
self._window = (pid, now, cpu) if pid is not None and cpu is not None else None
|
|
elif now - self._window[1] >= PROGRESS_WINDOW_S:
|
|
if cpu - self._window[2] >= _MIN_CPU_PER_WINDOW_S:
|
|
self.deadline = min(self._cap, max(self.deadline, now + _LEAD_S))
|
|
self._window = (pid, now, cpu)
|
|
return self.deadline
|