mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 11:22:02 +03:00
80350460d5
On a CPU-capped host (a 10% cgroup quota, a loaded runner) a cold `gateway run`
needs a few seconds of CPU that take minutes of wall time. The fixed 60 s
deadline of ensure_gateway_runtime (9c7dbc6e6f) then turns every cold
`hermes -z` into `Gateway starting: deadline`: main failed 3 of 3 runs at a
10% quota, each at about 61 s.
- A default ensure (no explicit timeout) extends its deadline while the
pending start's owner (the home's verified running gateway, else the
spawned child) keeps accruing CPU, at least 0.1 s per 15 s window, up to a
300 s cap. A control probe that times out on such an owner is retried
instead of ending the wait. Only a start this call is waiting on (one it
spawned, joined or saw pending, a multiplexer root included) is sampled: a
wedged READY gateway that spins still fails at 60 s, as before. An owner that makes no progress (stopped,
blocked on a lock) still fails at the base 60 s, and an explicit --timeout
stays a hard deadline. On macOS the one-off launchd job gets the moved
deadline too, or its poll() would stop reporting an exit past the original.
- A client's own idle-exit gateway imports the /api/ws handler on a worker
thread before READY, waiting at most 30 s for it, and an attach fetches the
handler on a worker thread too. Left to the first upgrade, that ~0.7 s CPU
import ran on the event loop inside the client's 10 s open timeout and
failed the attach right after READY on a starved host.
At a 10% quota a cold `hermes -z` now attaches and answers (3 of 3); with no
quota the attach time is unchanged (3.90 s vs main 3.92 s), and a SIGSTOPped
gateway still fails with `starting: deadline`.
91 lines
3.5 KiB
Python
91 lines
3.5 KiB
Python
"""A default ensure waits out a slow cold start whose owner keeps burning CPU, never a stalled one."""
|
|
import pytest
|
|
|
|
from hermes_cli import gateway_runtime_progress as progress
|
|
|
|
|
|
class _Clock:
|
|
def __init__(self):
|
|
self.now = 1000.0
|
|
|
|
def __call__(self):
|
|
return self.now
|
|
|
|
|
|
@pytest.fixture
|
|
def clock(monkeypatch):
|
|
clock = _Clock()
|
|
monkeypatch.setattr(progress.time, "monotonic", clock)
|
|
monkeypatch.setattr(progress, "_starting_owner_pid", lambda homes, child: 4242)
|
|
return clock
|
|
|
|
|
|
def _run(clock, monkeypatch, runway, cpu, *, until=1000, step=1.0):
|
|
"""Drive ``extend`` once per ``step`` until the clock reaches ``until`` or the deadline passes.
|
|
``cpu(t)`` is the owner's CPU seconds at elapsed ``t``. Returns the elapsed time it gave up at."""
|
|
start = clock.now
|
|
monkeypatch.setattr(progress, "_cpu_seconds", lambda pid: cpu(clock.now - start))
|
|
while clock.now - start < until:
|
|
deadline = runway.extend(("home",), None)
|
|
if clock.now >= deadline:
|
|
return clock.now - start
|
|
clock.now += step
|
|
return None
|
|
|
|
|
|
def test_progressing_owner_outlives_the_base_deadline_until_the_cap(clock, monkeypatch):
|
|
runway = progress.StartRunway(None, 60.0)
|
|
# A boot at a 5% quota: 0.05 s of CPU per wall second, still not READY.
|
|
gave_up = _run(clock, monkeypatch, runway, lambda t: 0.05 * t)
|
|
assert gave_up == pytest.approx(progress.MAX_ADAPTIVE_TIMEOUT, abs=1.0)
|
|
|
|
|
|
def test_owner_with_no_cpu_progress_fails_at_the_base_deadline(clock, monkeypatch):
|
|
runway = progress.StartRunway(None, 60.0)
|
|
gave_up = _run(clock, monkeypatch, runway, lambda t: 2.0) # stopped / blocked on a lock
|
|
assert gave_up == pytest.approx(60.0, abs=1.0)
|
|
|
|
|
|
def test_owner_that_stalls_after_progressing_fails_within_two_windows(clock, monkeypatch):
|
|
runway = progress.StartRunway(None, 60.0)
|
|
gave_up = _run(clock, monkeypatch, runway, lambda t: 0.05 * min(t, 90.0))
|
|
assert 90.0 < gave_up <= 90.0 + 2 * progress.PROGRESS_WINDOW_S + progress._LEAD_S + 1
|
|
|
|
|
|
def test_explicit_timeout_is_a_hard_deadline(clock, monkeypatch):
|
|
runway = progress.StartRunway(10.0, 60.0)
|
|
gave_up = _run(clock, monkeypatch, runway, lambda t: 0.05 * t)
|
|
assert gave_up == pytest.approx(10.0, abs=1.0)
|
|
|
|
|
|
def test_unreadable_owner_cpu_never_extends(clock, monkeypatch):
|
|
runway = progress.StartRunway(None, 60.0)
|
|
gave_up = _run(clock, monkeypatch, runway, lambda t: None) # AccessDenied / gone
|
|
assert gave_up == pytest.approx(60.0, abs=1.0)
|
|
|
|
|
|
def test_owner_cpu_readable_only_after_the_first_sample_still_extends(clock, monkeypatch):
|
|
runway = progress.StartRunway(None, 60.0)
|
|
gave_up = _run(clock, monkeypatch, runway, lambda t: None if t < 3 else 0.05 * t)
|
|
assert gave_up == pytest.approx(progress.MAX_ADAPTIVE_TIMEOUT, abs=1.0)
|
|
|
|
|
|
@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan")])
|
|
def test_invalid_timeout_is_refused(timeout):
|
|
with pytest.raises(ValueError):
|
|
progress.StartRunway(timeout, 60.0)
|
|
|
|
|
|
def test_cpu_seconds_reads_this_process():
|
|
import os
|
|
cpu = progress._cpu_seconds(os.getpid())
|
|
assert cpu is not None and cpu > 0
|
|
|
|
|
|
def test_a_lock_record_without_a_running_gateway_is_never_the_owner(tmp_path, monkeypatch):
|
|
"""A backup/restore holding the home writes a lock record too; its CPU is not the start's."""
|
|
from types import SimpleNamespace
|
|
from gateway import status
|
|
monkeypatch.setattr(status, "get_running_pid", lambda pid_path, cleanup_stale: None)
|
|
assert progress._starting_owner_pid((tmp_path,), SimpleNamespace(pid=77)) == 77
|