Files
hermes-agent/tests/tools/test_delegate_resumable.py
yoniebans d7ba925486 test: the newer tests seed and read through the storage interface instead of the legacy session store
Twenty-three of main's tests built, patched or read the legacy hermes_state.SessionDB (or hermes_state_registry) while the code they test reads through the storage interface. Their setup now opens the store with open_persistence(profile_scope(home=...)) or storage_handle, seeds with seed_session and seed_messages, reads with session_row, transcript_rows and conversation, wraps the backend holder (holder_type) where a test slows storage, and uses raw_holder for the raw resumable_subagents rows. Every assertion keeps its subject and expected value.

test_state_db_opens_tracked takes the backup query from hermes_cli.backup_integrity, drives the approvals scan through scan_approval_history (its reader needs the messages columns the fixture now declares), and points the journal-mode switch at the fixture store through HERMES_HOME.
2026-11-17 17:51:23 +01:00

237 lines
11 KiB
Python

"""Durable resumable subagents: a stopped child leaves a state.db record that ``delegate_task(action='continue')``
rebuilds it from after the process is gone, the record says how it may resume, and its worktree is kept from the
startup sweep for the record's life.
Only the model is scripted; the child build, registry, store, async dispatch, boot recovery and the worktree
sweep are the real ones, against a temp HERMES_HOME.
"""
import json
import os
import subprocess
import time
from pathlib import Path
from unittest.mock import patch
import pytest
from tests.fixtures.storage_holder import raw_holder
from tests.tools.test_delegate_continue import _completion, _parent, _ScriptedModel, _spawn
def _db_rows(parent):
with raw_holder(parent.persistence)._read_ctx() as conn:
return {sid: (state, pinned, {**json.loads(raw), "stop_cause": cause}) for sid, state, pinned, raw, cause in
conn.execute("SELECT subagent_id, state, pinned, record_json, stop_cause FROM resumable_subagents")}
def _forget_process_state():
"""What a restart loses: the in-process registry of finished children."""
from tools.delegate_tool_registry import _active_subagents_lock, _recent_subagents
with _active_subagents_lock:
_recent_subagents.clear()
class _FailingModel(_ScriptedModel):
"""First turn ends on a usage limit with a reset time, as the conversation loop reports it."""
def __init__(self, reason="billing", resets_at=None):
super().__init__()
self.reason, self.resets_at = reason, resets_at
def __call__(self, **kw):
child = super().__call__(**kw)
ok = child.run_conversation.side_effect
def run_conversation(user_message, conversation_history=None, **rest):
result = ok(user_message, conversation_history=conversation_history, **rest)
if len(self.turns) == 1:
result = {**result, "completed": False, "failed": True, "error": "limit", "failure_reason": self.reason,
**({"failure_resets_at": self.resets_at, "usage_limit": {"window": "5h"}}
if self.resets_at else {})}
return result
child.run_conversation.side_effect = run_conversation
return child
def test_a_limit_stopped_child_is_continued_after_a_restart_on_its_own_transcript():
from tools.delegate_tool import delegate_task
resets = time.time() + 3600
model = _FailingModel(resets_at=resets)
parent = _parent()
with patch("run_agent.AIAgent", side_effect=model):
out = json.loads(delegate_task(tasks=[{"goal": "write the parser"}], background=True, parent_agent=parent))
done = _completion(out["delegation_id"])
sid = out["subagent_ids"][0]
assert '"failure_resets_at"' in json.dumps(done), done # the parent sees when the limit resets
state, _pinned, record = _db_rows(parent)[sid]
assert (state, record["resume_mode"], record["failure_resets_at"]) == ("resumable", "AUTO_AT", resets)
assert record["usage_limit"] == {"window": "5h"}
_forget_process_state()
cont = json.loads(delegate_task(action="continue", subagent_id=sid, message="the limit reset, go on",
background=True, parent_agent=parent))
assert cont.get("status") == "dispatched", cont
_completion(cont["delegation_id"])
first, again = model.builds
assert again["session_id"] == model.turns[0]["session_id"] and again["model"] == first["model"]
assert [m["content"] for m in model.turns[1]["history"]][:1] == ["write the parser"]
assert _db_rows(parent)[sid][0] == "completed"
@pytest.mark.parametrize("cause", ["gateway", "user"])
def test_a_gateway_shutdown_leaves_a_resumable_record_and_a_user_stop_does_not(cause):
import threading
from tools.async_delegation import interrupt_all
from tools.delegate_tool import delegate_task
from tools.delegate_tool_registry import _active_subagents, _active_subagents_lock, interrupt_subagent
started, release = threading.Event(), threading.Event()
model = _ScriptedModel()
def build(**kw):
child = model(**kw)
ok = child.run_conversation.side_effect
def run_conversation(user_message, **rest):
started.set()
release.wait(10)
ok(user_message, **rest)
return {"final_response": "Operation interrupted.", "interrupted": True, "completed": False}
child.run_conversation.side_effect = run_conversation
return child
parent = _parent()
with patch("run_agent.AIAgent", side_effect=build):
out = json.loads(delegate_task(tasks=[{"goal": "long job"}], background=True, parent_agent=parent))
sid = out["subagent_ids"][0]
assert started.wait(10)
if cause == "gateway":
interrupt_all(reason="gateway shutdown (test)", resumable=True) # gateway/run_shutdown.py's call
else:
with _active_subagents_lock:
assert sid in _active_subagents
interrupt_subagent(sid)
release.set()
_completion(out["delegation_id"])
state, pinned, record = _db_rows(parent)[sid]
if cause == "gateway":
assert (state, record["resume_mode"], record["stop_cause"]) == ("resumable", "AUTO_ON_BOOT", "gateway_shutdown")
else:
assert (state, pinned) == ("stopped", 0)
def _git(*args, cwd):
subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True,
env={**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t",
"GIT_COMMITTER_EMAIL": "t@t"})
def test_a_resumable_childs_isolation_worktree_survives_its_stop_and_continue_runs_there(tmp_path, monkeypatch):
"""Found live: the stop path archived and removed the worktree of a child that was about to be continued."""
import threading
from tools.async_delegation import interrupt_all
from tools.delegate_tool import delegate_task
repo = tmp_path / "repo"
repo.mkdir()
_git("init", "-q", "-b", "main", cwd=repo)
(repo / "a.txt").write_text("a")
_git("add", "a.txt", cwd=repo)
_git("commit", "-qm", "init", cwd=repo)
monkeypatch.setattr("tools.delegate_tool_child_run._get_worktree_isolation", lambda: True)
monkeypatch.setattr("tools.delegate_tool._resolve_workspace_hint", lambda _p: str(repo))
monkeypatch.setattr("tools.delegate_tool_child_run._resolve_workspace_hint", lambda _p: str(repo))
started, release = threading.Event(), threading.Event()
model = _ScriptedModel()
def build(**kw):
child = model(**kw)
ok = child.run_conversation.side_effect
def run_conversation(user_message, **rest):
from tools.terminal_tool import get_session_cwd
model.turns.append({"cwd": get_session_cwd(rest.get("task_id"))})
if len(model.builds) == 1:
(Path(get_session_cwd(rest.get("task_id"))) / "a.txt").write_text("child edit")
started.set()
release.wait(10)
ok(user_message, **rest)
return {"final_response": "Operation interrupted.", "interrupted": True, "completed": False}
return ok(user_message, **rest)
child.run_conversation.side_effect = run_conversation
return child
parent = _parent()
with patch("run_agent.AIAgent", side_effect=build):
out = json.loads(delegate_task(tasks=[{"goal": "edit"}], background=True, parent_agent=parent))
sid = out["subagent_ids"][0]
assert started.wait(10)
interrupt_all(reason="gateway shutdown (test)", resumable=True)
release.set()
_completion(out["delegation_id"])
tree = Path(_db_rows(parent)[sid][2]["worktree"]["path"])
assert (tree / "a.txt").read_text() == "child edit" # kept, edit and all
_forget_process_state()
cont = json.loads(delegate_task(action="continue", subagent_id=sid, message="go on", background=True,
parent_agent=parent))
_completion(cont["delegation_id"])
assert os.path.realpath(model.turns[-2]["cwd"]) == os.path.realpath(tree)
assert not tree.exists() # the continue completed: its finalize archives and removes the tree as before
def test_crashed_childs_worktree_survives_the_sweep_until_its_record_expires(tmp_path):
"""The owner died mid-run (no stop record): boot recovery pins the tree the child last worked in; the real
sweep keeps it past 24 h; once the record expires the same sweep archives and removes it."""
import sqlite3
from hermes_cli.worktree_sweep import plan_sweep, run_sweep, sweep_settings
from persistence_assembly import open_persistence, profile_scope
from tools.async_delegation import recover_abandoned_delegations
from tools.delegate_tool_resumable import _upsert, _write
from hermes_constants import get_hermes_home
repo = tmp_path / "repo"
repo.mkdir()
_git("init", "-q", "-b", "main", cwd=repo)
(repo / "a.txt").write_text("a")
_git("add", "a.txt", cwd=repo)
_git("commit", "-qm", "init", cwd=repo)
tree = repo / ".worktrees" / "subagent-sa-crash"
_git("worktree", "add", "-q", str(tree), "-b", "subagent-sa-crash", cwd=repo)
(tree / "work.txt").write_text("unsaved child work")
open_persistence(profile_scope(home=get_hermes_home()), eager=True).close()
db = get_hermes_home() / "state.db"
assert _write(db, lambda ledger: _upsert(ledger, "sa-crash", {
"session_id": "s", "cwd": str(tree), "owner_pid": 2 ** 22 + 7, "owners": ["parent-sess"],
"worktrees_at_start": [str(repo)]}, state="running", pinned=True, expires_at=time.time() + 3600,
expected=None))
recover_abandoned_delegations()
old = time.time() - 30 * 3600
git_dir = repo / ".git" / "worktrees" / "subagent-sa-crash"
for path in (tree, tree / "work.txt", git_dir / "HEAD", git_dir / "logs" / "HEAD"):
os.utime(path, (old, old))
plan = {d.name: d.action for d in plan_sweep(str(repo), 24)}
assert plan["subagent-sa-crash"] == "skip-resumable", plan
run_sweep(str(repo), 24, sweep_settings())
assert tree.is_dir()
conn = sqlite3.connect(db) # health: allow HX015 -- test ages the record
with conn:
conn.execute("UPDATE resumable_subagents SET expires_at=?", (time.time() - 1,)) # its 48 h are over
conn.close()
assert {d.name: d.action for d in plan_sweep(str(repo), 24)}["subagent-sa-crash"] == "archive-remove"
run_sweep(str(repo), 24, sweep_settings())
assert not tree.exists()
refs = subprocess.run(["git", "for-each-ref", "refs/hermes/archive"], cwd=repo, capture_output=True, text=True,
check=True)
assert "subagent-sa-crash" in refs.stdout