Files
hermes-agent/tests/hermes_cli/test_plugin_idle_tasks.py
Victor Kyriazakos a7ac9bba45 fix(plugins): a slow plugin idle task no longer holds up the gateway's housekeeping
Plugin idle tasks ran synchronously on the thread that polled them. On the
gateway that is the housekeeping thread: it also stamps the runtime
heartbeat and runs the cron delivery drain, the misfire catch-up, the plugin
update check and state.db maintenance. An idle task that made model calls for
minutes stalled all of them, and after two minutes gateway status reported
the live gateway as wedged.

Each due task now starts on its own thread, carrying the poller's profile
home and secret scope, and the poll returns at once. The run claim still
keeps one run at a time across pollers and processes, and a run of this
process is never doubled even when it outlasts the claim's stale age. The
worker records the run and releases the claim when the task returns or
raises. The curator's own pass is unchanged.
2026-11-13 15:30:30 -08:00

170 lines
6.7 KiB
Python

"""ctx.register_idle_task: the curator's idle scheduler, for plugins."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
import pytest
from hermes_cli.plugins import PluginContext, PluginManager, PluginManifest
from hermes_cli.plugins_idle_tasks import cmd_idle_tasks, load_state, registered_idle_tasks, run_plugin_idle_tasks
T0 = datetime(2026, 10, 1, 12, 0, tzinfo=UTC)
IDLE = 3 * 3600.0
@pytest.fixture
def home(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
return tmp_path
def _ctx(name="wisdom"):
manager = PluginManager()
return manager, PluginContext(PluginManifest(name=name, source="user"), manager)
def _poll(manager, now, idle=IDLE):
"""One poll, waited out: ``{"<plugin>:<task>": last_status}`` for the runs it started."""
started = run_plugin_idle_tasks(idle_for_seconds=idle, manager=manager, now=now)
for worker in started.values():
worker.join(timeout=10)
assert not worker.is_alive()
tasks = registered_idle_tasks(manager)
return {key: load_state(tasks[key].state_path())["last_status"] for key in started}
def _recorder(calls, result="checked 3 skills"):
def fn(run):
calls.append(run)
return result
return fn
def test_first_poll_seeds_and_defers_one_interval_then_runs_on_the_interval(home):
manager, ctx = _ctx()
calls = []
ctx.register_idle_task("suggest", _recorder(calls), interval_hours=24)
assert _poll(manager, T0) == {}
state = load_state(ctx.state.data_dir / "idle-tasks" / "suggest.json")
assert state["last_run_at"] == T0.isoformat() and state["run_count"] == 0 and not calls
assert _poll(manager, T0 + timedelta(hours=23)) == {}
assert _poll(manager, T0 + timedelta(hours=24)) == {"wisdom:suggest": "ok"}
assert len(calls) == 1
run = calls[0]
assert (run.name, run.idle_for_seconds, run.last_run_at, run.run_count) == ("suggest", IDLE, T0.isoformat(), 0)
state = load_state(ctx.state.data_dir / "idle-tasks" / "suggest.json")
assert (state["run_count"], state["last_status"], state["last_summary"]) == (1, "ok", "checked 3 skills")
assert _poll(manager, T0 + timedelta(hours=25)) == {}
def test_immediate_first_run_and_min_idle_gate(home):
manager, ctx = _ctx()
calls = []
ctx.register_idle_task("suggest", _recorder(calls), interval_hours=24, min_idle_hours=2, first_run="immediate")
assert _poll(manager, T0, idle=3600.0) == {} # host active one hour ago: not idle long enough
assert _poll(manager, T0) == {"wisdom:suggest": "ok"}
assert calls[0].last_run_at is None
def test_registration_validates_and_floors_the_interval(home):
manager, ctx = _ctx()
for bad in ({"name": "Bad Name"}, {"first_run": "now"}, {"fn": "not callable"}, {"interval_hours": "weekly"},
{"interval_hours": float("inf")}, {"interval_hours": float("nan")},
{"min_idle_hours": float("inf")}, {"min_idle_hours": float("nan")}):
kwargs = {"name": "ok", "fn": _recorder([]), "interval_hours": 24} | bad
with pytest.raises(ValueError):
ctx.register_idle_task(kwargs.pop("name"), kwargs.pop("fn"), **kwargs)
calls = []
ctx.register_idle_task("tight", _recorder(calls), interval_hours=0.001, first_run="immediate")
assert _poll(manager, T0) == {"wisdom:tight": "ok"}
assert _poll(manager, T0 + timedelta(minutes=59)) == {} # floored to one hour
assert _poll(manager, T0 + timedelta(hours=1)) == {"wisdom:tight": "ok"}
def test_pause_and_resume_from_the_cli(home, capsys):
manager, ctx = _ctx()
calls = []
ctx.register_idle_task("suggest", _recorder(calls), interval_hours=1, first_run="immediate")
_poll(manager, T0)
cmd_idle_tasks("wisdom", pause="suggest")
assert _poll(manager, T0 + timedelta(hours=5)) == {}
cmd_idle_tasks()
assert "wisdom:suggest paused runs=1" in capsys.readouterr().out
cmd_idle_tasks("wisdom", resume="suggest")
assert _poll(manager, T0 + timedelta(hours=5)) == {"wisdom:suggest": "ok"}
with pytest.raises(SystemExit):
cmd_idle_tasks("wisdom", pause="missing")
def test_a_run_never_overlaps_itself(home):
manager, ctx = _ctx()
nested = []
def fn(run):
nested.append(_poll(manager, T0 + timedelta(hours=5))) # a second poller while this run holds the claim
ctx.register_idle_task("suggest", fn, interval_hours=1, first_run="immediate")
assert _poll(manager, T0) == {"wisdom:suggest": "ok"}
assert nested == [{}]
lock = ctx.state.data_dir / "idle-tasks" / ".suggest.lock"
lock.write_text("4242") # another process holds it
assert _poll(manager, T0 + timedelta(hours=5)) == {}
lock.unlink()
assert _poll(manager, T0 + timedelta(hours=5)) == {"wisdom:suggest": "ok"}
def test_a_failing_task_is_contained_and_recorded(home):
manager, ctx = _ctx()
other = []
def boom(run):
raise RuntimeError("model unreachable")
ctx.register_idle_task("boom", boom, interval_hours=1, first_run="immediate")
ctx.register_idle_task("fine", _recorder(other), interval_hours=1, first_run="immediate")
assert _poll(manager, T0) == {"wisdom:boom": "error", "wisdom:fine": "ok"}
state = load_state(ctx.state.data_dir / "idle-tasks" / "boom.json")
assert state["last_status"] == "error" and "model unreachable" in state["last_error"]
assert state["run_count"] == 1 and len(other) == 1
def test_profile_scoping_and_disposal(home, tmp_path):
manager, ctx = _ctx()
calls = []
registration = ctx.register_idle_task("suggest", _recorder(calls), interval_hours=1, first_run="immediate")
elsewhere = PluginManager(scope_key=str(tmp_path / "profiles" / "coder"))
PluginContext(PluginManifest(name="wisdom", source="user"), elsewhere).register_idle_task(
"suggest", _recorder(calls), interval_hours=1, first_run="immediate")
assert _poll(elsewhere, T0) == {} # another profile's manager runs nothing under this home
assert not calls
assert _poll(manager, T0) == {"wisdom:suggest": "ok"}
assert (home / "plugin-data").is_dir() and ctx.state.data_dir.is_relative_to(home)
registration.dispose()
assert _poll(manager, T0 + timedelta(hours=5)) == {}
def test_the_gateway_idle_poll_runs_the_curator_and_the_plugin_tasks(home, monkeypatch):
import agent.curator
from gateway import run as gateway_run
from hermes_cli import plugins_idle_tasks
seen = []
monkeypatch.setattr(agent.curator, "maybe_run_curator", lambda **kw: seen.append(("curator", kw["idle_for_seconds"])))
monkeypatch.setattr(plugins_idle_tasks, "run_plugin_idle_tasks", lambda **kw: seen.append(("plugins", kw["idle_for_seconds"])))
gateway_run._housekeeping_curator()
assert seen == [("curator", float("inf")), ("plugins", float("inf"))]