mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:04:08 +03:00
1cec6709ae
Idle-task workers were detached threads that always wrote the task's state when the plugin's function returned, so a run still going when the gateway stopped (or the multiplexer restarted in the same process) wrote profile state after housekeeping had stopped. Gateway housekeeping now polls idle tasks under a stop fence tied to its stop event. Once housekeeping stops, no run starts under it; when the loop exits it waits a bounded time (inside the housekeeping drain budget) for its runs and returns. A run that ends after the stop records nothing: the task's state stays as it was before the run, so the task runs again on a later poll, and the run still releases its claim. Closing the fence waits for a record in progress, so none lands after the stop. Plugin code cannot be stopped; the idle-task contract now says so and states the guarantee. Hosts without a stop (CLI/TUI startup, the serve maintenance tick) are unchanged.
395 lines
18 KiB
Python
395 lines
18 KiB
Python
"""Plugin idle tasks (``ctx.register_idle_task``): the skill curator's scheduler, for plugins.
|
|
|
|
A plugin registers a session-less job over its own state. Every host that ticks the curator also
|
|
polls the registered idle tasks of the active profile's plugin manager (:func:`run_idle_jobs`):
|
|
gateway housekeeping (once per served profile), CLI/TUI startup maintenance and ``hermes serve``'s
|
|
maintenance ticker. A task runs when the host has been idle ``min_idle_hours``, the task is not
|
|
paused, ``interval_hours`` have passed since its last run (the first observation seeds
|
|
``last_run_at`` and defers one interval, like the curator, unless ``first_run="immediate"``) and no
|
|
other run of it holds the claim. Each due task runs on its own thread, started with the poller's
|
|
contextvars (profile home and secret scope), and the poll returns at once: a slow task never holds
|
|
up the host's housekeeping. The claim keeps one run at a time across pollers and processes; the
|
|
worker records the run and releases the claim when ``fn`` returns or raises. An exception is logged
|
|
and recorded, never raised. A host that stops (gateway housekeeping) polls under an
|
|
:class:`IdleTaskFence`: once it stops, no run starts and a run still going records nothing.
|
|
|
|
State is per (profile, plugin, task): ``<plugin data dir>/idle-tasks/<name>.json`` with
|
|
``last_run_at``, ``run_count``, ``paused``, ``last_summary``, ``last_status``, ``last_error``.
|
|
``hermes plugins idle-tasks`` lists it and pauses/resumes a task.
|
|
|
|
No capability gates this surface: it calls nothing privileged itself; model calls go through
|
|
``ctx.llm`` and its existing gates.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import logging
|
|
import math
|
|
import re
|
|
import threading
|
|
import time
|
|
import weakref
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Iterator, Optional
|
|
|
|
logger = logging.getLogger("hermes_cli.plugins")
|
|
|
|
MIN_INTERVAL_HOURS = 1.0
|
|
_FIRST_RUN = ("defer", "immediate")
|
|
_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
|
|
_STATE_DIR = "idle-tasks"
|
|
_SUMMARY_MAX_CHARS = 2000
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class IdleTaskRun:
|
|
"""What the task function receives: plain data, so it crosses the isolated plugin host too (the
|
|
plugin reaches its own ``ctx`` through the closure that registered the task)."""
|
|
|
|
name: str
|
|
idle_for_seconds: float
|
|
last_run_at: Optional[str] # ISO-8601, None on an immediate first run
|
|
run_count: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class IdleTaskSpec:
|
|
ctx: Any # PluginContext
|
|
name: str
|
|
fn: Callable[[IdleTaskRun], Optional[str]]
|
|
interval_hours: float
|
|
min_idle_hours: float
|
|
first_run: str
|
|
|
|
@property
|
|
def plugin_id(self) -> str:
|
|
return self.ctx.plugin_id
|
|
|
|
def state_path(self) -> Path:
|
|
"""Resolved per call: the active profile's plugin data dir."""
|
|
return self.ctx.state.data_dir / _STATE_DIR / f"{self.name}.json"
|
|
|
|
|
|
# Registered tasks per plugin manager (one manager per profile home), keyed "<plugin id>:<name>".
|
|
_registries: weakref.WeakKeyDictionary[Any, dict[str, IdleTaskSpec]] = weakref.WeakKeyDictionary()
|
|
|
|
# This process's running workers, keyed by their claim's lock path. A run may outlast the claim's
|
|
# stale age; the claim alone would then let the next poll start a second run beside it.
|
|
_in_flight: dict[str, threading.Thread] = {}
|
|
_in_flight_lock = threading.Lock()
|
|
|
|
|
|
# How long a stopping host (gateway housekeeping) waits for its idle-task runs. Inside the
|
|
# gateway's housekeeping drain budget (``_HOUSEKEEPING_SHUTDOWN_DRAIN_TIMEOUT``), so stop never hangs.
|
|
SHUTDOWN_JOIN_SECONDS = 4.0
|
|
|
|
|
|
class IdleTaskFence:
|
|
"""A stopping host's hold on the idle-task runs it starts (gateway housekeeping). Once the fence
|
|
is closed (:meth:`close`, or its ``stop`` event set) no run starts under it, and a run still
|
|
going when it ends records nothing: the task's state stays as it was before the run, so the
|
|
task runs again on a later poll, and its claim is released. Plugin code cannot be stopped, so
|
|
:meth:`close` waits a bounded time for the runs, then returns."""
|
|
|
|
def __init__(self, stop: Any = None):
|
|
self._stop, self._closed = stop, False
|
|
self._lock = threading.Lock() # held by a start and by a record, so close() waits for either
|
|
self._workers: list[threading.Thread] = []
|
|
|
|
@property
|
|
def closed(self) -> bool:
|
|
return self._closed or (self._stop is not None and self._stop.is_set())
|
|
|
|
def start(self, worker: threading.Thread) -> bool:
|
|
"""Start ``worker`` under the fence; False, and not started, once the fence is closed."""
|
|
with self._lock:
|
|
if self.closed:
|
|
return False
|
|
worker.start()
|
|
self._workers = [w for w in self._workers if w.is_alive()] + [worker]
|
|
return True
|
|
|
|
@contextlib.contextmanager
|
|
def recording(self) -> Iterator[bool]:
|
|
"""Held while a run records its state; yields False once the fence is closed (record nothing)."""
|
|
with self._lock:
|
|
yield not self.closed
|
|
|
|
def close(self, timeout: Optional[float] = None) -> bool:
|
|
"""Close the fence and wait up to ``timeout`` seconds (default :data:`SHUTDOWN_JOIN_SECONDS`)
|
|
for its runs; True when none still runs. No run starts or records under it once this returns."""
|
|
with self._lock:
|
|
self._closed, workers = True, list(self._workers)
|
|
deadline = time.monotonic() + max(0.0, SHUTDOWN_JOIN_SECONDS if timeout is None else timeout)
|
|
for worker in workers:
|
|
worker.join(max(0.0, deadline - time.monotonic()))
|
|
running = [w.name for w in workers if w.is_alive()]
|
|
if running:
|
|
logger.warning("Idle task run(s) still going at shutdown, left unrecorded: %s", ", ".join(running))
|
|
return not running
|
|
|
|
|
|
def registered_idle_tasks(manager: Any) -> dict[str, IdleTaskSpec]:
|
|
return _registries.setdefault(manager, {})
|
|
|
|
|
|
def build_idle_task_spec(
|
|
ctx: Any, name: Any, fn: Any, *, interval_hours: Any, min_idle_hours: Any, first_run: Any,
|
|
) -> IdleTaskSpec:
|
|
"""The registration, or ``ValueError`` for a bad name, callable, interval or ``first_run``."""
|
|
me = ctx.manifest.name
|
|
if not isinstance(name, str) or not _NAME_RE.match(name):
|
|
raise ValueError(f"Plugin '{me}' idle task name {name!r} must match {_NAME_RE.pattern}")
|
|
if not callable(fn):
|
|
raise ValueError(f"Plugin '{me}' idle task {name!r}: fn must be callable")
|
|
if first_run not in _FIRST_RUN:
|
|
raise ValueError(f"Plugin '{me}' idle task {name!r}: first_run must be one of {_FIRST_RUN}")
|
|
try:
|
|
interval, idle = float(interval_hours), float(min_idle_hours)
|
|
except (TypeError, ValueError):
|
|
raise ValueError(f"Plugin '{me}' idle task {name!r}: interval_hours and min_idle_hours must be numbers") from None
|
|
if not (math.isfinite(interval) and math.isfinite(idle)): # inf would overflow timedelta; NaN never compares
|
|
raise ValueError(f"Plugin '{me}' idle task {name!r}: interval_hours and min_idle_hours must be finite")
|
|
if interval < MIN_INTERVAL_HOURS:
|
|
logger.warning("Plugin '%s' idle task %r: interval_hours=%r floored to %s", me, name, interval_hours,
|
|
MIN_INTERVAL_HOURS)
|
|
interval = MIN_INTERVAL_HOURS
|
|
return IdleTaskSpec(ctx, name, fn, interval, idle if idle >= 0 else 0.0, first_run)
|
|
|
|
|
|
# --- state ------------------------------------------------------------------------------------
|
|
|
|
|
|
def _base_state(plugin_id: str, name: str) -> dict[str, Any]:
|
|
return {"plugin": plugin_id, "task": name, "last_run_at": None, "run_count": 0, "paused": False,
|
|
"last_summary": None, "last_status": None, "last_error": None, "last_run_duration_seconds": None,
|
|
"interval_hours": None}
|
|
|
|
|
|
def load_state(path: Path, plugin_id: str = "", name: str = "") -> dict[str, Any]:
|
|
import json
|
|
|
|
state = _base_state(plugin_id, name or path.stem)
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8-sig")) if path.exists() else {}
|
|
except (OSError, ValueError) as e:
|
|
logger.debug("Unreadable idle task state %s: %s", path, e)
|
|
return state
|
|
if isinstance(data, dict):
|
|
state.update({k: v for k, v in data.items() if k in state})
|
|
return state
|
|
|
|
|
|
def save_state(path: Path, state: dict[str, Any]) -> None:
|
|
from utils import atomic_json_write
|
|
|
|
try:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
atomic_json_write(path, state, indent=2, sort_keys=True)
|
|
except Exception as e:
|
|
logger.debug("Failed to save idle task state %s: %s", path, e, exc_info=True)
|
|
|
|
|
|
def set_paused(path: Path, paused: bool) -> dict[str, Any]:
|
|
state = load_state(path)
|
|
state["paused"] = bool(paused)
|
|
save_state(path, state)
|
|
return state
|
|
|
|
|
|
def iter_state_files(plugin_id: Optional[str] = None) -> Iterator[tuple[Path, dict[str, Any]]]:
|
|
"""Every idle task state of the active profile (optionally one plugin's), sorted."""
|
|
from hermes_constants import get_hermes_home
|
|
|
|
for path in sorted((get_hermes_home() / "plugin-data").glob(f"*/{_STATE_DIR}/*.json")):
|
|
state = load_state(path)
|
|
if plugin_id is None or state.get("plugin") == plugin_id:
|
|
yield path, state
|
|
|
|
|
|
def next_due(state: dict[str, Any]) -> Optional[str]:
|
|
from agent.idle_schedule import parse_iso
|
|
|
|
last, interval = parse_iso(state.get("last_run_at")), state.get("interval_hours")
|
|
if last is None or not isinstance(interval, (int, float)) or not math.isfinite(interval):
|
|
return None
|
|
return (last + timedelta(hours=interval)).isoformat(timespec="seconds")
|
|
|
|
|
|
# --- scheduler --------------------------------------------------------------------------------
|
|
|
|
|
|
def _due(spec: IdleTaskSpec, state: dict[str, Any], path: Path, now: datetime) -> bool:
|
|
from agent.idle_schedule import interval_due
|
|
|
|
if state["paused"]:
|
|
return False
|
|
if state["last_run_at"] is None and spec.first_run == "immediate":
|
|
return True
|
|
state["interval_hours"] = spec.interval_hours
|
|
return interval_due(state, spec.interval_hours, now=now, save=lambda s: save_state(path, s),
|
|
seed_summary="deferred first run: runs after one interval")
|
|
|
|
|
|
def _execute(spec: IdleTaskSpec, run: IdleTaskRun, path: Path, lock: Path, now: datetime,
|
|
fence: Optional[IdleTaskFence] = None) -> None:
|
|
"""The worker body: run ``spec.fn``, then record the run (unless ``fence`` closed meanwhile)
|
|
and release the claim, whatever happened."""
|
|
from agent.idle_schedule import release_run
|
|
|
|
started, status, summary, error = time.monotonic(), "ok", None, None
|
|
try:
|
|
result = spec.fn(run)
|
|
if isinstance(result, str) and result.strip():
|
|
summary = result.strip()[:_SUMMARY_MAX_CHARS]
|
|
except Exception as exc:
|
|
# Boundary: a plugin's job never breaks the host.
|
|
status, error = "error", f"{type(exc).__name__}: {exc}"[:500]
|
|
logger.warning("Plugin %s idle task %s failed", spec.plugin_id, spec.name, exc_info=True)
|
|
finally:
|
|
try:
|
|
with contextlib.nullcontext(True) if fence is None else fence.recording() as open_:
|
|
if open_:
|
|
fresh = load_state(path, spec.plugin_id, spec.name) # keep a pause set while it ran
|
|
fresh.update(last_run_at=now.isoformat(), run_count=int(fresh["run_count"] or 0) + 1,
|
|
last_status=status, last_summary=summary, last_error=error,
|
|
interval_hours=spec.interval_hours,
|
|
last_run_duration_seconds=round(time.monotonic() - started, 3))
|
|
save_state(path, fresh)
|
|
else:
|
|
status = "abandoned at shutdown, not recorded"
|
|
finally:
|
|
release_run(lock)
|
|
with _in_flight_lock:
|
|
_in_flight.pop(str(lock), None)
|
|
logger.info("plugin idle task plugin=%s task=%s status=%s", spec.plugin_id, spec.name, status)
|
|
|
|
|
|
def _start(worker: threading.Thread, fence: Optional[IdleTaskFence]) -> bool:
|
|
if fence is None:
|
|
worker.start()
|
|
return True
|
|
return fence.start(worker)
|
|
|
|
|
|
def _run_one(spec: IdleTaskSpec, idle_for_seconds: float, now: datetime,
|
|
fence: Optional[IdleTaskFence] = None) -> Optional[threading.Thread]:
|
|
"""Start ``spec`` on its own thread if every gate passes; the started worker, or None when not run.
|
|
|
|
The worker carries the poller's contextvars (profile home and secret scope), so a host polling
|
|
under a profile's scope runs that profile's task in it. Under a closed ``fence`` nothing starts."""
|
|
from agent.idle_schedule import claim_run, release_run
|
|
from agent.memory_provider import spawn_context_thread
|
|
|
|
if idle_for_seconds < spec.min_idle_hours * 3600.0 or (fence is not None and fence.closed):
|
|
return None
|
|
path = spec.state_path()
|
|
state = load_state(path, spec.plugin_id, spec.name)
|
|
if not _due(spec, state, path, now):
|
|
return None
|
|
lock = path.with_name(f".{spec.name}.lock")
|
|
run = IdleTaskRun(spec.name, idle_for_seconds, state["last_run_at"], int(state["run_count"] or 0))
|
|
with _in_flight_lock:
|
|
# A run of this process still holds the claim: its age says nothing about a crash.
|
|
if str(lock) in _in_flight or not claim_run(lock): # or another process is running it
|
|
return None
|
|
worker = spawn_context_thread(_execute, args=(spec, run, path, lock, now, fence),
|
|
name=f"idle-task-{spec.plugin_id}-{spec.name}")
|
|
_in_flight[str(lock)] = worker
|
|
try:
|
|
started = _start(worker, fence)
|
|
except RuntimeError: # no thread to be had
|
|
started = False
|
|
logger.warning("Plugin %s idle task %s could not start", spec.plugin_id, spec.name, exc_info=True)
|
|
if not started: # the claim must not outlive the attempt
|
|
_in_flight.pop(str(lock), None)
|
|
release_run(lock)
|
|
return None
|
|
return worker
|
|
|
|
|
|
def run_plugin_idle_tasks(
|
|
*, idle_for_seconds: float, manager: Any = None, now: Optional[datetime] = None,
|
|
fence: Optional[IdleTaskFence] = None,
|
|
) -> dict[str, threading.Thread]:
|
|
"""Poll the active profile's registered idle tasks and return at once; ``{"<plugin>:<task>":
|
|
worker}`` for the runs it started, each on its own thread. A manager polled outside its own
|
|
profile runs nothing; a host that stops passes its ``fence``. Never raises."""
|
|
from hermes_constants import hermes_home_key
|
|
from hermes_cli.plugins import get_plugin_manager
|
|
|
|
started: dict[str, threading.Thread] = {}
|
|
try:
|
|
manager = manager if manager is not None else get_plugin_manager()
|
|
if manager.scope_key != hermes_home_key():
|
|
return started
|
|
now = now or datetime.now(UTC)
|
|
for key, spec in list(registered_idle_tasks(manager).items()):
|
|
if (worker := _run_one(spec, idle_for_seconds, now, fence)) is not None:
|
|
started[key] = worker
|
|
except Exception:
|
|
logger.debug("plugin idle task poll failed", exc_info=True)
|
|
return started
|
|
|
|
|
|
def run_idle_jobs(
|
|
*, idle_for_seconds: float, on_summary: Optional[Callable[[str], None]] = None,
|
|
fence: Optional[IdleTaskFence] = None,
|
|
) -> None:
|
|
"""The idle poll every curator host runs: the curator, then the plugins' idle tasks."""
|
|
from agent.curator import maybe_run_curator
|
|
|
|
maybe_run_curator(idle_for_seconds=idle_for_seconds, on_summary=on_summary)
|
|
run_plugin_idle_tasks(idle_for_seconds=idle_for_seconds, fence=fence)
|
|
|
|
|
|
def cmd_idle_tasks(plugin: Optional[str] = None, *, pause: Optional[str] = None, resume: Optional[str] = None) -> None:
|
|
"""``hermes plugins idle-tasks [plugin] [--pause TASK | --resume TASK]``: list the active
|
|
profile's idle task states, or pause/resume one task (the plugin name is required then)."""
|
|
target = pause or resume
|
|
if target is not None:
|
|
if not plugin:
|
|
raise SystemExit("hermes plugins idle-tasks: name the plugin to pause or resume a task")
|
|
hit = next((path for path, state in iter_state_files(plugin) if state["task"] == target), None)
|
|
if hit is None:
|
|
raise SystemExit(f"No idle task {target!r} of plugin {plugin!r} has run or been seeded in this profile")
|
|
set_paused(hit, pause is not None)
|
|
print(f"{plugin}:{target} {'paused' if pause else 'resumed'}")
|
|
return
|
|
rows = list(iter_state_files(plugin))
|
|
if not rows:
|
|
print("No plugin idle tasks have been seeded in this profile.")
|
|
return
|
|
for _path, s in rows:
|
|
status = "paused" if s["paused"] else (s["last_status"] or "seeded")
|
|
print(f"{s['plugin']}:{s['task']} {status} runs={s['run_count']} last={s['last_run_at'] or '-'} "
|
|
f"next={next_due(s) or '-'}")
|
|
if s["last_summary"] or s["last_error"]:
|
|
print(f" {s['last_error'] or s['last_summary']}")
|
|
|
|
|
|
class PluginIdleTaskCtxMixin:
|
|
"""``PluginContext.register_idle_task``."""
|
|
|
|
def register_idle_task(
|
|
self, name: str, fn: Callable[[IdleTaskRun], Optional[str]], *, interval_hours: float,
|
|
min_idle_hours: float = 2.0, first_run: str = "defer",
|
|
):
|
|
"""Run ``fn`` in the background when Hermes is idle, at most once per ``interval_hours``
|
|
(floored to 1) and only after ``min_idle_hours`` without activity. ``fn`` runs on its own
|
|
thread and the host does not wait for it, so keep a run short and safe to repeat; it
|
|
receives an :class:`IdleTaskRun` (``name``, ``idle_for_seconds``, ``last_run_at``,
|
|
``run_count``) and may return a short summary string. The first run waits one interval
|
|
unless ``first_run="immediate"``. Returns the :class:`PluginRegistration` (dispose to stop
|
|
polling). Raises ``ValueError`` for a bad name (``[a-z0-9][a-z0-9_-]*``), a non-callable
|
|
``fn``, a non-numeric interval, or an unknown ``first_run``."""
|
|
spec = build_idle_task_spec(self, name, fn, interval_hours=interval_hours,
|
|
min_idle_hours=min_idle_hours, first_run=first_run)
|
|
tasks = registered_idle_tasks(self._manager)
|
|
key = f"{self.plugin_id}:{name}"
|
|
return self._register_entry("idle_task", key, tasks, spec, "Plugin %s registered idle task: %s", name,
|
|
previous=tasks.get(key))
|