Files
hermes-agent/tests/hermes_cli/test_gateway_runtime_progress.py
kshitijk4poor 80350460d5 fix(gateway): a default ensure waits out a CPU-starved cold start
On a CPU-capped host (a 10% cgroup quota, a loaded runner) a cold `gateway run`
needs a few seconds of CPU that take minutes of wall time. The fixed 60 s
deadline of ensure_gateway_runtime (9c7dbc6e6f) then turns every cold
`hermes -z` into `Gateway starting: deadline`: main failed 3 of 3 runs at a
10% quota, each at about 61 s.

- A default ensure (no explicit timeout) extends its deadline while the
  pending start's owner (the home's verified running gateway, else the
  spawned child) keeps accruing CPU, at least 0.1 s per 15 s window, up to a
  300 s cap. A control probe that times out on such an owner is retried
  instead of ending the wait. Only a start this call is waiting on (one it
  spawned, joined or saw pending, a multiplexer root included) is sampled: a
  wedged READY gateway that spins still fails at 60 s, as before. An owner that makes no progress (stopped,
  blocked on a lock) still fails at the base 60 s, and an explicit --timeout
  stays a hard deadline. On macOS the one-off launchd job gets the moved
  deadline too, or its poll() would stop reporting an exit past the original.
- A client's own idle-exit gateway imports the /api/ws handler on a worker
  thread before READY, waiting at most 30 s for it, and an attach fetches the
  handler on a worker thread too. Left to the first upgrade, that ~0.7 s CPU
  import ran on the event loop inside the client's 10 s open timeout and
  failed the attach right after READY on a starved host.

At a 10% quota a cold `hermes -z` now attaches and answers (3 of 3); with no
quota the attach time is unchanged (3.90 s vs main 3.92 s), and a SIGSTOPped
gateway still fails with `starting: deadline`.
2026-11-09 00:56:23 +05:30

91 lines
3.5 KiB
Python

"""A default ensure waits out a slow cold start whose owner keeps burning CPU, never a stalled one."""
import pytest
from hermes_cli import gateway_runtime_progress as progress
class _Clock:
def __init__(self):
self.now = 1000.0
def __call__(self):
return self.now
@pytest.fixture
def clock(monkeypatch):
clock = _Clock()
monkeypatch.setattr(progress.time, "monotonic", clock)
monkeypatch.setattr(progress, "_starting_owner_pid", lambda homes, child: 4242)
return clock
def _run(clock, monkeypatch, runway, cpu, *, until=1000, step=1.0):
"""Drive ``extend`` once per ``step`` until the clock reaches ``until`` or the deadline passes.
``cpu(t)`` is the owner's CPU seconds at elapsed ``t``. Returns the elapsed time it gave up at."""
start = clock.now
monkeypatch.setattr(progress, "_cpu_seconds", lambda pid: cpu(clock.now - start))
while clock.now - start < until:
deadline = runway.extend(("home",), None)
if clock.now >= deadline:
return clock.now - start
clock.now += step
return None
def test_progressing_owner_outlives_the_base_deadline_until_the_cap(clock, monkeypatch):
runway = progress.StartRunway(None, 60.0)
# A boot at a 5% quota: 0.05 s of CPU per wall second, still not READY.
gave_up = _run(clock, monkeypatch, runway, lambda t: 0.05 * t)
assert gave_up == pytest.approx(progress.MAX_ADAPTIVE_TIMEOUT, abs=1.0)
def test_owner_with_no_cpu_progress_fails_at_the_base_deadline(clock, monkeypatch):
runway = progress.StartRunway(None, 60.0)
gave_up = _run(clock, monkeypatch, runway, lambda t: 2.0) # stopped / blocked on a lock
assert gave_up == pytest.approx(60.0, abs=1.0)
def test_owner_that_stalls_after_progressing_fails_within_two_windows(clock, monkeypatch):
runway = progress.StartRunway(None, 60.0)
gave_up = _run(clock, monkeypatch, runway, lambda t: 0.05 * min(t, 90.0))
assert 90.0 < gave_up <= 90.0 + 2 * progress.PROGRESS_WINDOW_S + progress._LEAD_S + 1
def test_explicit_timeout_is_a_hard_deadline(clock, monkeypatch):
runway = progress.StartRunway(10.0, 60.0)
gave_up = _run(clock, monkeypatch, runway, lambda t: 0.05 * t)
assert gave_up == pytest.approx(10.0, abs=1.0)
def test_unreadable_owner_cpu_never_extends(clock, monkeypatch):
runway = progress.StartRunway(None, 60.0)
gave_up = _run(clock, monkeypatch, runway, lambda t: None) # AccessDenied / gone
assert gave_up == pytest.approx(60.0, abs=1.0)
def test_owner_cpu_readable_only_after_the_first_sample_still_extends(clock, monkeypatch):
runway = progress.StartRunway(None, 60.0)
gave_up = _run(clock, monkeypatch, runway, lambda t: None if t < 3 else 0.05 * t)
assert gave_up == pytest.approx(progress.MAX_ADAPTIVE_TIMEOUT, abs=1.0)
@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan")])
def test_invalid_timeout_is_refused(timeout):
with pytest.raises(ValueError):
progress.StartRunway(timeout, 60.0)
def test_cpu_seconds_reads_this_process():
import os
cpu = progress._cpu_seconds(os.getpid())
assert cpu is not None and cpu > 0
def test_a_lock_record_without_a_running_gateway_is_never_the_owner(tmp_path, monkeypatch):
"""A backup/restore holding the home writes a lock record too; its CPU is not the start's."""
from types import SimpleNamespace
from gateway import status
monkeypatch.setattr(status, "get_running_pid", lambda pid_path, cleanup_stale: None)
assert progress._starting_owner_pid((tmp_path,), SimpleNamespace(pid=77)) == 77