diff --git a/agent/kanban_turn_recovery.py b/agent/kanban_turn_recovery.py new file mode 100644 index 0000000000..e54df56446 --- /dev/null +++ b/agent/kanban_turn_recovery.py @@ -0,0 +1,380 @@ +"""In-place recovery for kanban worker turns that die on a retryable failure. + +A worker spawned by the kanban dispatcher used to end its run silently when a +provider call failed: the one-shot CLI returned, the process exited ``rc=0`` +with no terminal ``kanban_complete`` / ``kanban_block`` call, the dispatcher +booked a *protocol violation* and respawned the task from scratch — discarding +the whole session context the worker had accumulated. Under a provider stream +storm that turns one slow morning into several cold restarts per card. + +This module retries such a turn IN PLACE: same session, same conversation +history, bounded budget, explicit backoff, and a continuation nudge that tells +the model to finish the task with a real terminal call. When the budget is +exhausted (or the retry is not authorised) the caller exits NON-ZERO so the run +is booked honestly instead of masquerading as a clean exit. + +Retry authority is POSITIVE and TYPED — never inferred from an outcome shape: + +* only a ``failed`` turn whose classifier marked the failure ``retryable`` is + eligible, and quota/billing walls are left to the dispatcher's + cooldown/breaker accounting (``KANBAN_RATE_LIMIT_EXIT_CODE``, see + ``hermes_cli/cli_single_query.py::_single_query_exit_code``); +* ``interrupted=True`` is never retried (the interrupt was persisted and + cleared by ``agent/turn_recovery.py::abort_turn_on_interrupt`` — re-entering + the model would resurrect explicitly cancelled work); +* a turn that ended through the bounded terminal settlement is never retried + (``agent/turn_finalizer.py::_record_kanban_budget_exhausted`` already recorded + ``outcome="timed_out"`` and released the claim — #87096); +* before EVERY attempt the worker proves it still owns the exact live run and + unexpired claim lease (``worker_claim_is_live``). The proof is the carrier the + dispatcher pinned at spawn — ``HERMES_KANBAN_DB`` + ``HERMES_KANBAN_RUN_ID`` + + ``HERMES_KANBAN_CLAIM_LOCK`` are REQUIRED and compared exactly, and the + task/run ``claim_expires`` lease must be unexpired; any missing coordinate or + expired lease means no proof, no retry (fail closed). + +Incomplete-but-not-failed results (``partial`` / ``completed=False``) are +deliberately NOT retry authority: truncation and compression repair are owned +inside the conversation loop (#89289) and by the dispatcher. They still fail the +process exit code (``hermes_cli/cli_single_query.py::_single_query_exit_code``) so such a run is booked +honestly instead of ending as a silent ``rc=0``. +""" + +from __future__ import annotations + +import logging +import os +import sqlite3 +import time +from pathlib import Path +from typing import Any, Callable, Optional + +from agent.error_classifier import FailoverReason + +logger = logging.getLogger(__name__) + +#: Recovery attempts when ``HERMES_KANBAN_TURN_RECOVERY`` is unset. +DEFAULT_MAX_RECOVERY_ATTEMPTS = 3 + +#: Backoff before recovery attempt N (1-based). The last entry repeats. +RECOVERY_DELAYS_SECONDS: tuple[float, ...] = (15.0, 45.0, 90.0) + +#: Failure reasons that must NOT be retried in place: quota walls and billing +#: blocks need the dispatcher's cooldown/breaker accounting, not another call. +#: Aligned with the exit side of main's ``cli._TRANSIENT_PROVIDER_REASONS`` +#: (which also releases ``timeout`` / ``overloaded`` / ``server_error`` runs as +#: neutral 75s AFTER in-place recovery has been exhausted). The two sets answer +#: different questions: this one says who may re-enter the model here; main's +#: says how the process exit is booked. +_NON_RECOVERABLE_REASONS = frozenset({ + FailoverReason.rate_limit.value, + FailoverReason.billing.value, + FailoverReason.upstream_rate_limit.value, +}) + +#: ``turn_exit_reason`` prefixes that mean the turn already went through a +#: durable terminal settlement (the run may be closed and its claim released): +#: ``agent/turn_finalizer.py`` writes ``max_iterations_reached(N/M)`` right after +#: recording ``outcome="timed_out"`` for an out-of-budget kanban worker (#87096). +_TERMINAL_TURN_EXIT_PREFIXES = ("max_iterations_reached",) + +_OFF_VALUES = frozenset({"0", "false", "no", "off"}) + + +def kanban_task_id() -> Optional[str]: + """The dispatcher-set kanban task id, or ``None`` when this is not a worker run. + + Single source of truth for every kanban-worker predicate. The env value is + stripped exactly once, here, so a whitespace-only value can never read as + "worker" to one caller and "not a worker" to another (the exit-code guard and + the recovery gate must agree). + """ + task_id = (os.environ.get("HERMES_KANBAN_TASK") or "").strip() + return task_id or None + + +def kanban_turn_recovery_enabled() -> bool: + """On when ``HERMES_KANBAN_TASK`` is set and the attempt budget is non-zero.""" + if kanban_task_id() is None: + return False + return max_recovery_attempts() > 0 + + +def max_recovery_attempts() -> int: + """``HERMES_KANBAN_TURN_RECOVERY`` parsed as an attempt count (0 disables). + + Unset/blank -> :data:`DEFAULT_MAX_RECOVERY_ATTEMPTS`; explicit 0/false/no/off + -> 0; anything unparseable -> the default. Clamped to [0, 10] so a bad value + can never create an unbounded loop. + """ + raw = (os.environ.get("HERMES_KANBAN_TURN_RECOVERY") or "").strip() + if not raw: + return DEFAULT_MAX_RECOVERY_ATTEMPTS + if raw.lower() in _OFF_VALUES: + return 0 + try: + value = int(raw) + except (TypeError, ValueError): + return DEFAULT_MAX_RECOVERY_ATTEMPTS + return max(0, min(value, 10)) + + +def recovery_delay_seconds(attempt: int) -> float: + """Backoff before recovery attempt ``attempt`` (1-based); last entry repeats.""" + if attempt < 1: + attempt = 1 + index = min(attempt - 1, len(RECOVERY_DELAYS_SECONDS) - 1) + return RECOVERY_DELAYS_SECONDS[index] + + +def _turn_exit_is_terminal(result: dict) -> bool: + reason = str(result.get("turn_exit_reason") or "") + return reason.startswith(_TERMINAL_TURN_EXIT_PREFIXES) + + +def should_recover_turn(result: Any, *, attempt: int) -> bool: + """True when a settled worker turn grants in-place retry authority. + + ``attempt`` is the number of recovery attempts ALREADY made. Authority is + positive and typed: an enabled recovery within budget, a FAILED turn whose + classifier marked the failure ``retryable``, no quota/billing wall, no + explicit cancellation, and no durable terminal settlement. Everything else — + including ``partial`` / ``completed=False`` — is not authority to re-enter + the model. See the module docstring for the full rule set. + """ + if not kanban_turn_recovery_enabled(): + return False + if attempt >= max_recovery_attempts(): + return False + if not isinstance(result, dict): + return False + if result.get("interrupted") is True: + return False + if _turn_exit_is_terminal(result): + return False + if result.get("failed") is not True: + return False + if result.get("failure_retryable") is not True: + return False + if str(result.get("failure_reason") or "") in _NON_RECOVERABLE_REASONS: + return False + return True + + +def _kanban_db_path() -> Optional[str]: + """The dispatcher-pinned board DB — no ambient resolution, ever. + + The dispatcher pins ``HERMES_KANBAN_DB`` (plus run id + claim lock) at spawn + (``hermes_cli/kanban_db_dispatch.py``); that pin IS the exact authority + carrier the retry proof must re-check. If the pin is absent there is no + exact carrier to re-prove, so this returns ``None`` and the caller fails + closed: resolving whatever board is ambient *now* would silently move the + proof onto a different board — precisely the weaker proof this fail-closed + contract exists to prevent. + """ + pinned = (os.environ.get("HERMES_KANBAN_DB") or "").strip() + return pinned or None + + +def _now() -> int: + """Wall-clock seconds — one seam so tests can advance the clock across a backoff.""" + return int(time.time()) + + +def worker_claim_is_live() -> bool: + """True when THIS process still owns a live, unexpired run/claim lease. + + Verified read-only against the dispatcher-pinned board before EVERY retry. + All three pinned coordinates are REQUIRED — ``HERMES_KANBAN_DB``, + ``HERMES_KANBAN_RUN_ID``, ``HERMES_KANBAN_CLAIM_LOCK`` — and compared with + exact equality; a missing pin means there is no exact authority carrier to + re-prove, never a fallback to ambient board state. + + The lease itself is part of the liveness contract: the canonical stale-claim + selector (``hermes_cli/kanban_db.py::release_stale_claims``) treats + ``status='running' AND claim_expires < now`` as stale and reclaims the task, + and the worker-liveness path treats an expired claim as non-live. So both + the task row AND the open run row must carry an UNEXPIRED ``claim_expires`` + (``heartbeat_claim`` extends the task expiry and mirrors it onto the active + run row). An expired lease means mutation authority has lapsed even before + the reconciler processes the row; a NULL expiry is not a provable live + lease either and fails closed. + + FAIL-CLOSED: any missing coordinate, missing/unreadable DB, mismatched + run/lock/pid, closed run, or expired lease returns False and the caller + does not re-enter the model — an honest non-zero exit is always available, + whereas mutating state under a released or expired claim is not recoverable. + """ + task_id = kanban_task_id() + if task_id is None: + return False + db_path = _kanban_db_path() + if not db_path or not Path(db_path).exists(): + logger.warning("kanban claim check: pinned board db %r not found — not retrying in place", db_path) + return False + run_id_env = (os.environ.get("HERMES_KANBAN_RUN_ID") or "").strip() + lock_env = (os.environ.get("HERMES_KANBAN_CLAIM_LOCK") or "").strip() + if not run_id_env or not lock_env: + logger.warning( + "kanban claim check: missing dispatcher-pinned run-id/claim-lock carrier " + "— no exact authority to re-prove, not retrying in place" + ) + return False + now = _now() + try: + conn = sqlite3.connect(f"{Path(db_path).absolute().as_uri()}?mode=ro", uri=True) + try: + conn.row_factory = sqlite3.Row + row = conn.execute( + "SELECT status, worker_pid, current_run_id, claim_lock, claim_expires " + "FROM tasks WHERE id = ?", + (task_id,), + ).fetchone() + if row is None: + return False + if (row["status"] or "") != "running": + return False + worker_pid = row["worker_pid"] + if worker_pid is not None and int(worker_pid) != os.getpid(): + return False + # A running task with no run pointer is not proof of a live run: the + # dispatcher writes it on claim and clears it on release/end. + current_run_id = row["current_run_id"] + if current_run_id is None: + return False + if str(current_run_id) != run_id_env: + return False + if (row["claim_lock"] or "") != lock_env: + return False + if row["claim_expires"] is None or int(row["claim_expires"]) < now: + return False + run = conn.execute( + "SELECT ended_at, worker_pid, claim_expires FROM task_runs WHERE id = ?", + (int(current_run_id),), + ).fetchone() + if run is None: + return False + if run["ended_at"] is not None: + return False + run_pid = run["worker_pid"] + if run_pid is not None and int(run_pid) != os.getpid(): + return False + if run["claim_expires"] is None or int(run["claim_expires"]) < now: + return False + return True + finally: + conn.close() + except Exception: + logger.warning("kanban claim check failed — not retrying in place", exc_info=True) + return False + + +def _truncate(text: str, limit: int = 300) -> str: + text = (text or "").strip().replace("\n", " ") + return text[: limit - 1] + "…" if len(text) > limit else text + + +def build_recovery_nudge(result: Any, *, attempt: int, max_attempts: int) -> str: + """The synthetic user turn sent after a failed worker turn. + + Mirrors the stop-nudge contract (agent/kanban_stop.py): plain text is not a + terminal state; the worker must finish with ``kanban_complete`` / + ``kanban_block``. Emphasises "same session, do not start over" so the model + reuses everything it already read/wrote instead of re-doing the work. + """ + error = "" + if isinstance(result, dict): + error = _truncate(str(result.get("error") or result.get("final_response") or "")) + task_id = kanban_task_id() or "this task" + return ( + "[System: the previous turn ended UNFINISHED — the API call failed after " + f"all retries and the turn was interrupted (recovery attempt {attempt}/{max_attempts}). " + f"Error: {error or 'provider stream failure'}.\n\n" + f"Task `{task_id}` is still `running`. This is the SAME session, with your full " + "context — nothing you already read, wrote, or computed is lost. Do NOT start over.\n\n" + "Do this immediately:\n" + "1. If the interrupted turn was mid tool-call, re-check the on-disk state of " + "whatever it was writing before continuing (the action may not have executed).\n" + "2. Continue the task from where it stopped and finish any remaining deliverable.\n" + "3. End with a terminal `kanban_complete(summary=..., artifacts=[...])` if the work " + "is done, or `kanban_block(reason=...)` if you are blocked.]" + ) + + +def _emit_recovery_skipped(emit: Optional[Callable[[str], None]]) -> None: + """The one "no proof, no retry" line — fail-closed is worth saying out loud.""" + message = ( + f"[kanban] in-place recovery skipped for {kanban_task_id() or 'task'}: this worker no " + "longer holds a live run/claim (already settled, reclaimed, or unverifiable)" + ) + logger.warning("%s", message) + if emit is not None: + try: + emit(message) + except Exception: + logger.debug("kanban turn-recovery emit failed", exc_info=True) + + +def recover_failed_kanban_turns( + turn_fn: Callable[[str], Any], + get_result: Callable[[], Any], + *, + sleep_fn: Callable[[float], None] = time.sleep, + emit: Optional[Callable[[str], None]] = None, + claim_check: Optional[Callable[[], bool]] = None, +) -> int: + """Retry an authorised kanban worker turn in place; returns attempts made. + + ``turn_fn(nudge)`` runs one more turn with the given synthetic user message. + ``get_result()`` returns the LATEST settled turn result (``None`` stops the + loop). Before EVERY attempt the worker must still own its live run/claim + (``claim_check``, default :func:`worker_claim_is_live`); the loop is bounded + by the attempt budget even when ``get_result`` never changes. The loop + deliberately does NOT extend its own claim lease during the backoff — + self-holding a claim is claim-lifetime policy (#95318), not this module's; + if the lease expires while sleeping, the re-proof fails and the attempt is + not made. + """ + attempts = 0 + check = claim_check or worker_claim_is_live + while True: + result = get_result() + if not should_recover_turn(result, attempt=attempts): + return attempts + if not check(): + _emit_recovery_skipped(emit) + return attempts + attempts += 1 + delay = recovery_delay_seconds(attempts) + message = ( + f"[kanban] provider failure on {kanban_task_id() or 'task'} — retrying the turn in place " + f"(attempt {attempts}/{max_recovery_attempts()}) after {int(delay)}s; session context preserved" + ) + logger.warning("%s", message) + if emit is not None: + try: + emit(message) + except Exception: + logger.debug("kanban turn-recovery emit failed", exc_info=True) + sleep_fn(delay) + # Re-prove ownership AFTER the backoff and immediately before model re-entry: the + # dispatcher can reclaim or settle this run while we sleep, and the pre-sleep proof + # alone would let that one attempt run without live ownership (round-5 finding F1 — + # TOCTOU window). No proof, no retry; the caller's honest exit still applies. + if not check(): + _emit_recovery_skipped(emit) + return attempts + turn_fn(build_recovery_nudge(result, attempt=attempts, max_attempts=max_recovery_attempts())) + + +__all__ = [ + "DEFAULT_MAX_RECOVERY_ATTEMPTS", + "RECOVERY_DELAYS_SECONDS", + "build_recovery_nudge", + "kanban_task_id", + "kanban_turn_recovery_enabled", + "max_recovery_attempts", + "recover_failed_kanban_turns", + "recovery_delay_seconds", + "should_recover_turn", + "worker_claim_is_live", +] diff --git a/hermes_cli/cli_single_query.py b/hermes_cli/cli_single_query.py index 3736eb4e07..bb0caa5b4b 100644 --- a/hermes_cli/cli_single_query.py +++ b/hermes_cli/cli_single_query.py @@ -159,13 +159,19 @@ def _single_query_exit_code(result, *, credentials_rate_limited: bool = False, (EX_CONFIG): the dispatcher blocks the card at once. The same code applies before a turn when credential resolution explicitly requires re-authentication; unknown startup failures retain exit 1. + + The worker predicate is the STRIPPED ``kanban_task_id()`` (from + ``agent.kanban_turn_recovery``) so a whitespace-only value is not a worker here + either — the exit mapping and the recovery gate must agree on what a worker is. """ from cli import _TERMINAL_PROVIDER_REASONS, _TRANSIENT_PROVIDER_REASONS + from agent.kanban_turn_recovery import kanban_task_id + if not isinstance(result, dict): - if credentials_rate_limited and os.environ.get("HERMES_KANBAN_TASK"): + if credentials_rate_limited and kanban_task_id(): from hermes_cli.kanban_db import KANBAN_RATE_LIMIT_EXIT_CODE return KANBAN_RATE_LIMIT_EXIT_CODE - if credentials_terminal and os.environ.get("HERMES_KANBAN_TASK"): + if credentials_terminal and kanban_task_id(): from hermes_cli.kanban_db import KANBAN_TERMINAL_PROVIDER_EXIT_CODE return KANBAN_TERMINAL_PROVIDER_EXIT_CODE return 1 @@ -173,7 +179,7 @@ def _single_query_exit_code(result, *, credentials_rate_limited: bool = False, return 130 if not (result.get("failed") or result.get("partial") or result.get("completed") is False): return 0 - if os.environ.get("HERMES_KANBAN_TASK"): + if kanban_task_id(): reason = result.get("failure_reason") if reason in _TRANSIENT_PROVIDER_REASONS: from hermes_cli.kanban_db import KANBAN_RATE_LIMIT_EXIT_CODE @@ -220,6 +226,29 @@ def _run_quiet_single_query(cli, effective_query, emitter=None): # The exit line below reports session_id to stderr for automation wrappers; # without this sync it would point at the ended parent after compression. _sync_cli_session_id_from_agent(cli) + # Kanban worker: a turn that died on a retryable provider failure is retried + # IN PLACE (same session, context preserved) instead of ending the run silently. + # Authority (typed retryable failure, no interrupt / terminal settlement) and the + # live run/claim proof live in agent/kanban_turn_recovery.py. Runs BEFORE the turn + # report so everything downstream — report, follow-ups, response, goal gate, exit — + # sees the post-recovery disposition. + from agent.kanban_turn_recovery import recover_failed_kanban_turns as _recover_turns + + def _quiet_recover_turn(nudge): + nonlocal result + _history = result.get("messages") if isinstance(result, dict) else None + result = cli.agent.run_conversation( + user_message=nudge, + conversation_history=_history or cli.conversation_history, + **author_kwargs, + ) + _sync_cli_session_id_from_agent(cli) + + _recover_turns( + _quiet_recover_turn, + lambda: result, + emit=lambda msg: print(msg, file=sys.stderr, flush=True), + ) # The turn is over and persisted: the one-shot exit linger that follows protects nested # notify_on_complete replies and is NOT part of the spawner's delivery (#113608). The # report carries what this run will print, so a spawner booking a child still lingering @@ -279,9 +308,24 @@ def _run_quiet_single_query(cli, effective_query, emitter=None): elif response: print(response) + # ONE exit-code decision point for this driver, shared with the non-quiet one-shot + # path via _single_query_exit_code: a quota/billing wall keeps the EX_TEMPFAIL + # sentinel so the dispatcher releases the task without counting a failure, and any + # other unfinished worker turn exits non-zero instead of ending as a silent rc=0 + # that reads as a protocol violation. Computed BEFORE goal continuation — the + # post-recovery disposition must be authoritative for the rest of the driver. + _exit_code = _single_query_exit_code(result) + # Kanban goal_mode: keep working in THIS session until a judge agrees the card is # done, the worker terminates it, or the turn budget runs out (sticky block). - if os.environ.get("HERMES_KANBAN_GOAL_MODE") == "1": + # ONLY a settled, authorized turn may continue: a result that remains + # failed/unfinished — recovery exhausted or DENIED (lease expired, claim lost, + # quota wall, interrupt, terminal settlement) — must reach the shared exit path + # below without goal continuation. goal_run_status() checks only run identity, + # not the claim lock or either expiry, so an unreaped expired-lease row still + # reports "running" and a continue verdict would re-enter the model under an + # authority this process can no longer prove (round-3 finding). + if os.environ.get("HERMES_KANBAN_GOAL_MODE") == "1" and _exit_code == 0: try: _run_kanban_goal_loop_q(cli, response) except Exception as _goal_exc: @@ -290,7 +334,6 @@ def _run_quiet_single_query(cli, effective_query, emitter=None): if emitter is None: print(f"\nsession_id: {cli.session_id}", file=sys.stderr) - _exit_code = _single_query_exit_code(result) if emitter is not None: _exit_code = emitter.emit_result(result, session_id=cli.session_id or "", exit_code=_exit_code) exit_single_query(_exit_code) @@ -516,10 +559,33 @@ def _run_single_query_mode(cli, query, image, quiet, oneshot, stream_json: bool cli.console.print(f"[bold blue]{t('cli.single_query.query_label')}[/] {_query_label}") cli._show_security_advisories() response = cli.chat(query, images=single_query_images or None) + # Kanban worker: a failed-silently turn used to end the run as rc=0 with no + # terminal kanban call, so the dispatcher booked a protocol violation and + # cold-restarted the task from scratch. Retry the authorised turn IN PLACE + # (see agent/kanban_turn_recovery.py) and CARRY THE RECOVERED RESPONSE: the + # goal judge below must evaluate the deliverable of the latest settled turn, + # never the stale provider error of the turn recovery replaced (re-review P2). + from agent.kanban_turn_recovery import recover_failed_kanban_turns + + def _nonquiet_recover_turn(nudge): + nonlocal response + response = cli.chat(nudge) + return response + + recover_failed_kanban_turns( + _nonquiet_recover_turn, + lambda: getattr(cli, "_last_turn_result", None), + emit=lambda msg: print(msg, file=sys.stderr, flush=True), + ) # Kanban goal_mode on the `-q` path: same judge loop as `-Q`, but each follow-up turn # runs through cli.chat so the worker log keeps its live tool feed (the dispatcher # used to force -Q here, which left goal_mode cards with a blank Worker log). - if os.environ.get("HERMES_KANBAN_GOAL_MODE") == "1": + # Gated on the post-recovery disposition exactly like the `-Q` block: a result that + # remains failed/unfinished — recovery exhausted or DENIED (claim lost, quota wall, + # interrupt, terminal settlement) — must reach the shared exit path below with zero + # further model entry, because goal_run_status() checks run identity only and would + # otherwise continue under an authority this process can no longer prove. + if os.environ.get("HERMES_KANBAN_GOAL_MODE") == "1" and _single_query_exit_code(cli._last_turn_result) == 0: try: _run_kanban_goal_loop_chat(cli, response or "") except Exception as _goal_exc: diff --git a/tests/agent/test_kanban_turn_recovery.py b/tests/agent/test_kanban_turn_recovery.py new file mode 100644 index 0000000000..e46763e369 --- /dev/null +++ b/tests/agent/test_kanban_turn_recovery.py @@ -0,0 +1,1136 @@ +"""Tests for kanban worker in-place turn recovery and the shared exit policy. + +Covers the retry-authority rules (positive and typed: retryable failed turns +only; interrupts and durable terminal settlements are never retried), the live +run/claim proof taken before every attempt, the backoff schedule, the +continuation nudge contract, the single exit-code decision point shared by the +``chat -q`` and quiet ``-Q`` one-shot paths, and the call-site wiring in +``cli.py`` driven through ``cli.main`` (the settled-outcome hook the ``-q`` +route reads, ``cli_chat_turn_mixin.py::_last_turn_result``, is main's own). +""" + +from __future__ import annotations + +import os +import sqlite3 +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from agent.kanban_turn_recovery import ( + DEFAULT_MAX_RECOVERY_ATTEMPTS, + RECOVERY_DELAYS_SECONDS, + build_recovery_nudge, + kanban_task_id, + kanban_turn_recovery_enabled, + max_recovery_attempts, + recover_failed_kanban_turns, + recovery_delay_seconds, + should_recover_turn, + worker_claim_is_live, +) +from hermes_cli.kanban_db import ( + KANBAN_RATE_LIMIT_EXIT_CODE, + KANBAN_TERMINAL_PROVIDER_EXIT_CODE, +) + +KANBAN_ENV = ( + "HERMES_KANBAN_TASK", + "HERMES_KANBAN_TURN_RECOVERY", + "HERMES_KANBAN_RUN_ID", + "HERMES_KANBAN_CLAIM_LOCK", + "HERMES_KANBAN_DB", + "HERMES_KANBAN_BOARD", +) + + +@pytest.fixture +def clear_kanban_env(monkeypatch): + for var in KANBAN_ENV: + monkeypatch.delenv(var, raising=False) + return monkeypatch + + +def _failed(*, retryable: bool = True, reason: str = "timeout", + error: str = "peer closed connection") -> dict: + return { + "failed": True, + "failure_retryable": retryable, + "failure_reason": reason, + "error": error, + "final_response": f"API call failed after 3 retries: {error}", + "completed": False, + "messages": [], + } + + +def _success() -> dict: + return {"failed": False, "completed": True, "final_response": "done", "messages": []} + + +# ── enablement / budget ────────────────────────────────────────────── + + +def test_disabled_without_kanban_task(clear_kanban_env): + assert kanban_task_id() is None + assert kanban_turn_recovery_enabled() is False + calls: list[str] = [] + attempts = recover_failed_kanban_turns( + lambda nudge: calls.append(nudge), lambda: _failed(), sleep_fn=lambda s: None + ) + assert attempts == 0 + assert calls == [] + + +def test_env_zero_disables(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + clear_kanban_env.setenv("HERMES_KANBAN_TURN_RECOVERY", "0") + assert kanban_turn_recovery_enabled() is False + assert should_recover_turn(_failed(), attempt=0) is False + + +def test_max_attempts_parsing_and_clamp(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + assert max_recovery_attempts() == DEFAULT_MAX_RECOVERY_ATTEMPTS # unset + clear_kanban_env.setenv("HERMES_KANBAN_TURN_RECOVERY", "2") + assert max_recovery_attempts() == 2 + clear_kanban_env.setenv("HERMES_KANBAN_TURN_RECOVERY", "99") + assert max_recovery_attempts() == 10 # clamped + clear_kanban_env.setenv("HERMES_KANBAN_TURN_RECOVERY", "abc") + assert max_recovery_attempts() == DEFAULT_MAX_RECOVERY_ATTEMPTS + clear_kanban_env.setenv("HERMES_KANBAN_TURN_RECOVERY", "false") + assert max_recovery_attempts() == 0 + + +def test_delay_schedule_repeats_last_entry(): + assert RECOVERY_DELAYS_SECONDS == (15.0, 45.0, 90.0) + assert recovery_delay_seconds(1) == 15.0 + assert recovery_delay_seconds(3) == 90.0 + assert recovery_delay_seconds(9) == 90.0 + assert recovery_delay_seconds(0) == 15.0 + + +def test_kanban_task_id_strips_and_rejects_blank(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", " ") + assert kanban_task_id() is None # whitespace-only is NOT a worker, anywhere + assert kanban_turn_recovery_enabled() is False + clear_kanban_env.setenv("HERMES_KANBAN_TASK", " t_probe ") + assert kanban_task_id() == "t_probe" + assert kanban_turn_recovery_enabled() is True + + +# ── retry authority: positive and typed ────────────────────────────── + + +def test_retryable_failed_turn_is_eligible(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + assert should_recover_turn(_failed(reason="timeout"), attempt=0) is True + assert should_recover_turn(_failed(reason="timeout"), attempt=2) is True # budget 3 + assert should_recover_turn(_failed(reason="timeout"), attempt=3) is False + + +def test_not_retryable_is_not_recovered(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + assert should_recover_turn(_failed(retryable=False, reason="auth"), attempt=0) is False + assert should_recover_turn(_success(), attempt=0) is False + assert should_recover_turn(None, attempt=0) is False + + +def test_rate_limit_and_billing_are_left_to_the_dispatcher(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + assert should_recover_turn(_failed(reason="rate_limit"), attempt=0) is False + assert should_recover_turn(_failed(reason="billing"), attempt=0) is False + + +def test_upstream_rate_limit_is_a_dispatcher_quota_wall(clear_kanban_env): + """The aggregator's upstream 429 is the same quota-class wall as a direct 429: + not retried in place, and released as the neutral 75 for a worker (main's + ``_TRANSIENT_PROVIDER_REASONS``); a person's run books it as a plain 1.""" + from cli import _single_query_exit_code + + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + assert should_recover_turn(_failed(reason="upstream_rate_limit"), attempt=0) is False + assert _single_query_exit_code(_failed(reason="upstream_rate_limit")) == 75 + clear_kanban_env.delenv("HERMES_KANBAN_TASK", raising=False) + assert _single_query_exit_code(_failed(reason="upstream_rate_limit")) == 1 + + +def test_interrupt_is_never_retry_authority(clear_kanban_env): + """``agent/turn_recovery.py::abort_turn_on_interrupt`` already persisted and + cleared the interrupt — re-entering the model would resurrect cancelled work.""" + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + interrupted = { + "final_response": "interrupted", "messages": [], "api_calls": 4, + "completed": False, "interrupted": True, + "failed": True, "failure_retryable": True, "failure_reason": "timeout", + } + assert should_recover_turn(interrupted, attempt=0) is False + + +def test_terminal_settlement_is_never_retry_authority(clear_kanban_env): + """#87096: the bounded kanban finalizer records ``outcome="timed_out"`` and + releases the claim while the result still reads ``completed=False`` — retrying + would continue work after this exact run was terminalized.""" + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + settled = { + "final_response": "budget exhausted", "messages": [], "api_calls": 60, + "completed": False, "failed": False, + "turn_exit_reason": "max_iterations_reached(60/60)", + } + assert should_recover_turn(settled, attempt=0) is False + # The sharp combination: the turn ALSO carries a retryable failure verdict (the + # loop exhausted its provider retries and then hit the iteration cap, so the + # finalizer terminalized the run). The settlement veto must win over the + # otherwise-eligible failure stamp. + sharp = {**_failed(retryable=True), "turn_exit_reason": "max_iterations_reached(60/60)"} + assert should_recover_turn(sharp, attempt=0) is False + # …and the exit policy still books it honestly rather than as a silent rc=0 + # (no failure_reason on a settled turn → not transient → honest 1). + from cli import _single_query_exit_code + + assert _single_query_exit_code(settled) == 1 + + +def test_incomplete_turn_is_not_retry_authority(clear_kanban_env): + """``partial`` / ``completed=False`` describe an outcome; they are not retry + authority (truncation repair belongs inside the conversation loop, #89289).""" + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + partial = {"partial": True, "completed": False, "final_response": "cut off", + "error": "truncated", "messages": []} + assert should_recover_turn(partial, attempt=0) is False + assert should_recover_turn({"completed": False, "final_response": "x"}, attempt=0) is False + + +# ── live run/claim proof ───────────────────────────────────────────── + + +#: Any "live" lease for these tests (2100-01-01 UTC). +_FAR_FUTURE_EXPIRY = 4_102_444_800 + + +def _make_board(tmp_path, *, status="running", task_pid=None, run_id=1, lock="lk", + run_ended=None, run_pid=None, task_expires=_FAR_FUTURE_EXPIRY, + run_expires=_FAR_FUTURE_EXPIRY): + """A synthetic board whose task carries a LIVE, unexpired task+run lease by + default (current-main dispatcher shape: ``claim_task`` sets ``claim_expires`` + on both rows and ``heartbeat_claim`` mirrors the extension).""" + db = tmp_path / "kanban.db" + conn = sqlite3.connect(db) + conn.execute( + "CREATE TABLE tasks (id TEXT PRIMARY KEY, status TEXT, worker_pid INTEGER, " + "current_run_id INTEGER, claim_lock TEXT, claim_expires INTEGER)" + ) + conn.execute( + "CREATE TABLE task_runs (id INTEGER PRIMARY KEY, ended_at INTEGER, " + "worker_pid INTEGER, claim_expires INTEGER)" + ) + conn.execute( + "INSERT INTO tasks VALUES (?, ?, ?, ?, ?, ?)", + ("t_live", status, task_pid, run_id, lock, task_expires), + ) + if run_id is not None: + conn.execute( + "INSERT INTO task_runs VALUES (?, ?, ?, ?)", (run_id, run_ended, run_pid, run_expires) + ) + conn.commit() + conn.close() + return db + + +def _pin_carrier(monkeypatch, db, *, task="t_live", run_id="1", lock="lk"): + """Pin the full dispatcher-spawn carrier: DB + run id + claim lock.""" + monkeypatch.setenv("HERMES_KANBAN_TASK", task) + monkeypatch.setenv("HERMES_KANBAN_DB", str(db)) + monkeypatch.setenv("HERMES_KANBAN_RUN_ID", run_id) + monkeypatch.setenv("HERMES_KANBAN_CLAIM_LOCK", lock) + + +def test_claim_is_live_when_this_worker_still_owns_the_run(clear_kanban_env, tmp_path): + db = _make_board(tmp_path, task_pid=os.getpid(), run_id=7, lock="lk", run_pid=os.getpid()) + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_live") + clear_kanban_env.setenv("HERMES_KANBAN_DB", str(db)) + clear_kanban_env.setenv("HERMES_KANBAN_RUN_ID", "7") + clear_kanban_env.setenv("HERMES_KANBAN_CLAIM_LOCK", "lk") + assert worker_claim_is_live() is True + + +@pytest.mark.parametrize( + "overrides", + [ + {"status": "ready"}, # task was released back to the board + {"task_pid": 999_999_999}, # another process owns the claim + {"run_ended": 12345}, # run already closed (settled) + {"run_pid": 999_999_999}, # run handed to another pid + {"run_id": None}, # no live run pointer + {"run_id": 2}, # board moved to a different run id + {"lock": "other-lock"}, # claim lock no longer ours + {"task_expires": 1}, # task lease has EXPIRED + {"run_expires": 1}, # run lease has EXPIRED (heartbeat mirror) + {"task_expires": None}, # no provable task lease — NULL is not "unbounded" + {"run_expires": None}, # no provable run lease + ], +) +def test_claim_is_not_live_when_ownership_is_broken(clear_kanban_env, tmp_path, overrides): + board = {"task_pid": os.getpid(), "run_pid": os.getpid(), **overrides} + db = _make_board(tmp_path, **board) + _pin_carrier(clear_kanban_env, db) # the pinned carrier itself is intact + assert worker_claim_is_live() is False + + +def test_claim_check_fails_closed_without_a_board(clear_kanban_env, tmp_path): + """No proof, no retry: a missing board must not authorise model re-entry.""" + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_live") + clear_kanban_env.setenv("HERMES_KANBAN_DB", str(tmp_path / "missing.db")) + assert worker_claim_is_live() is False + clear_kanban_env.delenv("HERMES_KANBAN_DB", raising=False) + clear_kanban_env.setenv("HERMES_KANBAN_BOARD", "no-such-board-xyz") + assert worker_claim_is_live() is False # and no ambient fallback rescues it + + +def test_missing_pinned_carrier_fails_closed(clear_kanban_env, tmp_path): + """Round-2 P1: the dispatcher pins DB + run id + claim lock at spawn. Any + MISSING coordinate means there is no exact authority carrier to re-prove — + the proof fails closed instead of widening to ambient board state or + skipping comparisons.""" + db = _make_board(tmp_path) # a live board with a live lease + _pin_carrier(clear_kanban_env, db) + assert worker_claim_is_live() is True # sanity: the full carrier proves live + + clear_kanban_env.delenv("HERMES_KANBAN_DB", raising=False) + assert worker_claim_is_live() is False # missing DB pin + clear_kanban_env.setenv("HERMES_KANBAN_DB", str(db)) + clear_kanban_env.delenv("HERMES_KANBAN_RUN_ID", raising=False) + assert worker_claim_is_live() is False # missing run-id pin + clear_kanban_env.setenv("HERMES_KANBAN_RUN_ID", "1") + clear_kanban_env.delenv("HERMES_KANBAN_CLAIM_LOCK", raising=False) + assert worker_claim_is_live() is False # missing claim-lock pin + + +def test_missing_db_pin_never_resolves_an_ambient_board(tmp_path, monkeypatch): + """Round-2 P1: the old code reconstructed the board from + ``HERMES_KANBAN_BOARD`` / the default when the pin was absent — the proof + could silently move to whatever board is ambient NOW. With the pin gone the + canonical resolver must not be consulted at all, even when it would return a + fully live board whose row matches this worker (built with the real board + API so this is not a strawman).""" + from hermes_cli import kanban_db as kb + from hermes_cli import kanban_db_connect as kbc + + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + kb.init_db() + with kbc.connect() as conn: + task_id = kb.create_task(conn, title="ambient live task") + claimed = kb.claim_task(conn, task_id, claimer="lk", ttl_seconds=3600) + assert claimed is not None # running, lock+run pointer+unexpired TTL set + monkeypatch.delenv("HERMES_KANBAN_DB", raising=False) + monkeypatch.delenv("HERMES_KANBAN_RUN_ID", raising=False) + monkeypatch.delenv("HERMES_KANBAN_CLAIM_LOCK", raising=False) + monkeypatch.setenv("HERMES_KANBAN_TASK", task_id) + + import agent.kanban_turn_recovery as rec + + consulted: list = [] + real_resolve = kb.kanban_db_path + + def _spy(board=None): + consulted.append(board) + return real_resolve(board=board) + + monkeypatch.setattr(kb, "kanban_db_path", _spy) + assert rec.worker_claim_is_live() is False + assert consulted == [] # no ambient resolution was attempted + + +def test_lease_expired_during_the_backoff_blocks_model_reentry(clear_kanban_env, tmp_path, monkeypatch): + """Round-2 P1 (the reviewer's exact race): failure near TTL expiry → the + pre-backoff proof passes → the 15/45/90s sleep crosses expiry → the + post-backoff re-proof must FAIL on the unexpired-lease check even though + status/run-id/lock/pid/ended_at are all unchanged. No model turn is made.""" + import agent.kanban_turn_recovery as rec + + db = _make_board(tmp_path, task_expires=1_600, run_expires=1_600) + _pin_carrier(clear_kanban_env, db, run_id="1", lock="lk") + clock = iter([1_000, 1_700]) # pre-backoff proof sees a live lease; the re-proof does not + monkeypatch.setattr(rec, "_now", lambda: next(clock)) + turns: list[str] = [] + emitted: list[str] = [] + + attempts = rec.recover_failed_kanban_turns( + lambda nudge: turns.append(nudge), lambda: _failed(), + sleep_fn=lambda s: None, emit=emitted.append, # default claim_check: the real proof + ) + assert attempts == 1 + assert turns == [] # the expired lease vetoed re-entry before the model call + assert "no longer holds a live run/claim" in emitted[-1] + + +# ── the loop ───────────────────────────────────────────────────────── + + +def _loop_ok(result=None): + return (lambda: True) if result is None else result + + +def test_recovery_loop_retries_until_success(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + latest = {"r": _failed()} + turns: list[str] = [] + delays: list[float] = [] + + def turn_fn(nudge: str) -> None: + turns.append(nudge) + latest["r"] = _success() # provider recovered + + attempts = recover_failed_kanban_turns( + turn_fn, lambda: latest["r"], sleep_fn=delays.append, emit=lambda m: None, + claim_check=_loop_ok(), + ) + assert attempts == 1 + assert delays == [15.0] + assert "Do NOT start over" in turns[0] + assert "t_probe" in turns[0] + + +def test_recovery_loop_bounded_when_result_never_changes(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + turns: list[str] = [] + delays: list[float] = [] + attempts = recover_failed_kanban_turns( + lambda nudge: turns.append(nudge), + lambda: _failed(), # stuck result: must NOT loop forever + sleep_fn=delays.append, emit=lambda m: None, claim_check=_loop_ok(), + ) + assert attempts == DEFAULT_MAX_RECOVERY_ATTEMPTS + assert delays == [15.0, 45.0, 90.0] + + +def test_recovery_loop_checks_the_claim_before_every_attempt(clear_kanban_env): + """The live run/claim proof is re-taken for EACH attempt. A run lost while we + were backing off (the dispatcher can reclaim during the 15/45/90s sleep) must + stop the loop WITHOUT re-entering the model — round-5 finding F1.""" + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + clear_kanban_env.setenv("HERMES_KANBAN_TURN_RECOVERY", "3") + checks: list[int] = [] + turns: list[str] = [] + emitted: list[str] = [] + + def claim_check() -> bool: + checks.append(len(checks) + 1) + return len(checks) == 1 # first proof holds; the run is lost during the backoff + + attempts = recover_failed_kanban_turns( + lambda nudge: turns.append(nudge), lambda: _failed(), + sleep_fn=lambda s: None, emit=emitted.append, claim_check=claim_check, + ) + assert checks == [1, 2] # pre-backoff proof + the pre-entry re-proof + assert attempts == 1 # the attempt was authorised … + assert len(turns) == 0 # … but the run was gone before re-entry: no model turn + assert "no longer holds a live run/claim" in emitted[-1] + + +def test_recovery_loop_rechecks_the_claim_after_the_backoff(clear_kanban_env): + """Order pin: check -> sleep -> check -> turn. The re-proof sits immediately + before model re-entry, not only before the backoff (round-5 finding F1).""" + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + events: list[str] = [] + latest = {"r": _failed()} + + def claim_check() -> bool: + events.append("check") + return True + + def sleeper(_seconds: float) -> None: + events.append("sleep") + + def turn_fn(nudge: str) -> None: + events.append("turn") + latest["r"] = _success() + + attempts = recover_failed_kanban_turns( + turn_fn, lambda: latest["r"], sleep_fn=sleeper, emit=lambda m: None, + claim_check=claim_check, + ) + assert attempts == 1 + assert events == ["check", "sleep", "check", "turn"] + + +def test_recovery_emit_receives_status_line(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + emitted: list[str] = [] + latest = {"r": _failed()} + recover_failed_kanban_turns( + lambda nudge: latest.update(r=_success()), lambda: latest["r"], + sleep_fn=lambda s: None, emit=emitted.append, claim_check=_loop_ok(), + ) + assert len(emitted) == 1 + assert "[kanban]" in emitted[0] + assert "attempt 1/3" in emitted[0] + + +def test_nudge_terminal_contract(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_probe") + nudge = build_recovery_nudge(_failed(error="peer closed connection"), attempt=1, max_attempts=3) + assert "t_probe" in nudge + assert "1/3" in nudge + assert "kanban_complete" in nudge + assert "kanban_block" in nudge + assert "Do NOT start over" in nudge + assert "peer closed connection" in nudge + + +# ── shared exit-code policy ────────────────────────────────────────── + + +def test_single_query_exit_code_policy(monkeypatch): + """The shared exit mapping is main's ``cli._single_query_exit_code`` — this PR + composes with it (stripped worker predicate) and must not fork it. Pins the + composed contract as main defines it today: transient provider walls release a + worker as the neutral 75, terminal provider verdicts park it as EX_CONFIG 78, + other failures and unfinished turns are honest 1s, interrupts are 130, + success 0.""" + from cli import _single_query_exit_code + + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + assert _single_query_exit_code(_success()) == 0 + # transient provider walls: neutral 75 for a worker — quota walls and the + # broader transient set (a storm that outlasts in-place recovery is released, + # not counted against the breaker) + for reason in ("rate_limit", "billing", "upstream_rate_limit", "timeout", "overloaded"): + assert _single_query_exit_code(_failed(reason=reason)) == KANBAN_RATE_LIMIT_EXIT_CODE + # terminal provider verdicts (revoked credential, model gone, WAF block) are + # main's EX_CONFIG: the dispatcher parks the card after ONE spawn instead of + # re-spawning into the same wall + for reason in ("auth", "auth_permanent", "model_not_found", "ssl_cert_verification", + "upstream_blocked"): + assert _single_query_exit_code(_failed(reason=reason)) == KANBAN_TERMINAL_PROVIDER_EXIT_CODE + # a non-transient, non-terminal worker failure is an honest 1 + assert _single_query_exit_code(_failed(reason="some_unknown_reason")) == 1 + # unfinished / nothing settled: honest 1 — never a silent rc=0 + assert _single_query_exit_code(None) == 1 + assert _single_query_exit_code({"partial": True, "completed": False}) == 1 + # explicit cancellation: 130 — never retried, never continued + assert _single_query_exit_code({**_failed(), "interrupted": True}) == 130 + + # a person's run: no neutral release — a wall is just a failure + monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False) + assert _single_query_exit_code(_success()) == 0 + assert _single_query_exit_code(_failed(reason="rate_limit")) == 1 + assert _single_query_exit_code(_failed(reason="timeout")) == 1 + assert _single_query_exit_code(None) == 1 + + +# ── call-site behaviour (drive cli.main with a FakeCLI) ────────────── + + +def _install_fake_cli(monkeypatch, chat_script, calls): + """Mirror tests/hermes_cli/test_single_query_session_finalize.py's FakeCLI harness.""" + import cli as cli_mod + + class _Console: + def print(self, *a, **k): + calls.append("query-label") + + class FakeCLI: + def __init__(self, **_kwargs): + self.console = _Console() + self.session_id = "single-query-session" + self.agent = SimpleNamespace(session_id="single-query-session", platform="cli") + + def _claim_active_session(self, surface, *, stderr=False): + return True + + def _show_security_advisories(self): + pass + + def chat(self, query, images=None): + calls.append(("chat", query)) + result = chat_script() if callable(chat_script) else chat_script.pop(0) + self._last_turn_result = result + # Mirror production: chat() returns the settled turn's rendered response — + # for a failed turn that IS the provider error text. + return result.get("final_response", "") if isinstance(result, dict) else "" + + def _print_exit_summary(self, clear_screen=True): + calls.append("summary") + + monkeypatch.setattr(cli_mod, "HermesCLI", FakeCLI) + monkeypatch.setattr(cli_mod.atexit, "register", lambda *a, **k: None) + monkeypatch.setattr(cli_mod, "_finalize_single_query", lambda fake_cli: None) + monkeypatch.setattr(cli_mod, "_collect_query_images", lambda q, img: (q, [])) + monkeypatch.setattr(cli_mod, "_collect_kanban_task_images", lambda imgs: []) + return cli_mod + + +@pytest.fixture +def live_claim(monkeypatch): + """The board proof is a separate contract (unit-tested above); call-site tests + run against a worker that owns its run, unless a test overrides it.""" + import agent.kanban_turn_recovery as rec + + monkeypatch.setattr(rec, "worker_claim_is_live", lambda: True) + return rec + + +def _chats(calls): + return [c for c in calls if isinstance(c, tuple) and c[0] == "chat"] + + +def test_non_quiet_kanban_recovers_in_place(monkeypatch, live_claim): + import agent.kanban_turn_recovery as rec + + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + monkeypatch.setenv("HERMES_KANBAN_TURN_RECOVERY", "2") + monkeypatch.setattr(rec, "RECOVERY_DELAYS_SECONDS", (0.0,)) + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [_failed(), _success()], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 0 # settled successfully + chats = _chats(calls) + assert len(chats) == 2 + assert "Do NOT start over" in chats[1][1] + assert "t_probe" in chats[1][1] + + +def test_non_quiet_kanban_exits_nonzero_after_budget(monkeypatch, live_claim): + import agent.kanban_turn_recovery as rec + + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + monkeypatch.setenv("HERMES_KANBAN_TURN_RECOVERY", "1") + monkeypatch.setattr(rec, "RECOVERY_DELAYS_SECONDS", (0.0,)) + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, _failed, calls) # never recovers + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 75 # exhausted transient wall: released, not counted + assert len(_chats(calls)) == 2 # original + one recovery attempt + + +@pytest.mark.parametrize("reason", ["rate_limit", "billing", "upstream_rate_limit"]) +def test_non_quiet_rate_limit_keeps_the_neutral_exit_code(monkeypatch, live_claim, reason): + """#48000-class boundary: the production ``chat -q`` route must hand a quota + wall (including the aggregator's upstream 429) to the dispatcher as the + neutral EX_TEMPFAIL code, not a counted crash.""" + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [_failed(reason=reason)], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 75 + assert len(_chats(calls)) == 1 # declined in place: the dispatcher owns the cooldown + + +def test_non_quiet_interrupted_turn_is_not_retried(monkeypatch, live_claim): + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + interrupted = {**_failed(), "interrupted": True} + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [interrupted], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 130 # main's interrupt exit: never retried, never continued + assert len(_chats(calls)) == 1 + + +def test_non_quiet_terminal_settlement_is_not_retried(monkeypatch, live_claim): + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + settled = {**_failed(retryable=True), + "turn_exit_reason": "max_iterations_reached(60/60)"} + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [settled], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + # never retried, never continued; the transient stamp books the neutral 75 — + # the #87096 settlement already recorded the durable timed_out outcome. + assert exc_info.value.code == 75 + assert len(_chats(calls)) == 1 + + +def test_non_quiet_partial_is_not_retried(monkeypatch, live_claim): + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + partial = {"partial": True, "completed": False, "final_response": "cut off", + "error": "truncated", "messages": []} + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [partial], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 1 + assert len(_chats(calls)) == 1 + + +def test_non_quiet_skips_recovery_without_a_live_claim(monkeypatch, capsys): + import agent.kanban_turn_recovery as rec + + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + monkeypatch.setattr(rec, "worker_claim_is_live", lambda: False) + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [_failed()], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 75 # denial still exits non-zero; transient → release + assert len(_chats(calls)) == 1 + assert "no longer holds a live run/claim" in capsys.readouterr().err + + +def test_non_quiet_kanban_no_settled_outcome_exits_nonzero(monkeypatch, live_claim): + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [None], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 1 + assert len(_chats(calls)) == 1 + + +def test_non_kanban_failed_run_exits_one_without_recovery(monkeypatch, live_claim): + """No HERMES_KANBAN_TASK: no in-place recovery; main's shared exit contract + books a failed run as 1 (the non-quiet tail now exits unconditionally).""" + monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False) + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [_failed()], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 1 + assert len(_chats(calls)) == 1 # no recovery outside kanban workers + assert "summary" in calls + + +def test_whitespace_task_id_is_not_a_worker_anywhere(monkeypatch, live_claim): + """A whitespace-only HERMES_KANBAN_TASK is not a worker: the exit mapping's + stripped predicate must not hand a quota wall the neutral 75 (raw env + truthiness would).""" + monkeypatch.setenv("HERMES_KANBAN_TASK", " ") + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [_failed(reason="rate_limit")], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 1 # not 75: not a worker, not neutralized + assert len(_chats(calls)) == 1 + + +def _install_quiet_fake_cli(monkeypatch, run_conversation): + """Quiet (-Q) harness: minimal HermesCLI whose agent runs the given function.""" + import cli as cli_mod + + class FakeCLI: + def __init__(self, **_kwargs): + self.provider = "test-provider" + self.model = "test-model" + self.session_id = "quiet-session" + self.conversation_history = [] + self._active_agent_route_signature = "same-route" + self.agent = SimpleNamespace( + session_id="quiet-session", platform="cli", quiet_mode=False, + suppress_status_output=False, stream_delta_callback=object(), + tool_gen_callback=object(), run_conversation=run_conversation, + ) + + def _claim_active_session(self, surface, *, stderr=False): + return True + + def _ensure_runtime_credentials(self): + return True + + def _resolve_turn_agent_config(self, effective_query): + return {"signature": "same-route", "model": None, "runtime": None, "request_overrides": None} + + def _init_agent(self, **kwargs): + return True + + monkeypatch.setattr(cli_mod, "HermesCLI", FakeCLI) + monkeypatch.setattr(cli_mod.atexit, "register", lambda *a, **k: None) + monkeypatch.setattr(cli_mod, "_finalize_single_query", lambda fake_cli: None) + monkeypatch.setattr(cli_mod, "_collect_kanban_task_images", lambda imgs: []) + return cli_mod + + +def test_quiet_kanban_recovers_in_place(monkeypatch, live_claim): + import agent.kanban_turn_recovery as rec + + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + monkeypatch.setenv("HERMES_KANBAN_TURN_RECOVERY", "2") + monkeypatch.setattr(rec, "RECOVERY_DELAYS_SECONDS", (0.0,)) + monkeypatch.delenv("HERMES_KANBAN_GOAL_MODE", raising=False) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return _failed() if len(runs) == 1 else _success() + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 0 + assert len(runs) == 2 + assert "Do NOT start over" in runs[1] + + +def test_quiet_rate_limit_keeps_the_neutral_exit_code(monkeypatch, live_claim): + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + monkeypatch.delenv("HERMES_KANBAN_GOAL_MODE", raising=False) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return _failed(reason="billing") + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 75 + assert len(runs) == 1 + + +def test_quiet_partial_exits_nonzero_without_retrying(monkeypatch, live_claim): + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + monkeypatch.delenv("HERMES_KANBAN_GOAL_MODE", raising=False) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return {"partial": True, "completed": False, "final_response": "cut off", + "error": "truncated"} + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 1 + assert len(runs) == 1 + + +def test_quiet_kanban_no_settled_outcome_exits_nonzero(monkeypatch, live_claim): + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + monkeypatch.delenv("HERMES_KANBAN_GOAL_MODE", raising=False) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return None + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 1 + assert len(runs) == 1 + + +def test_quiet_non_kanban_no_settled_outcome_is_booked_honestly(monkeypatch, live_claim): + monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False) + monkeypatch.delenv("HERMES_KANBAN_GOAL_MODE", raising=False) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return None + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 1 # main's shared contract: unsettled is 1 for every caller + assert len(runs) == 1 + + +# ── goal-mode composition (round-3): a recovery refusal stops ALL model entry ── + + +@pytest.fixture +def goal_spy(monkeypatch): + """Spy on the goal continuation loop at the cli-module seam. The goal loop + itself is pre-existing (its status check sees only run identity, not the + claim lease); the invariant under test here is the CALL-SITE gate.""" + import cli as cli_mod + + calls: list = [] + monkeypatch.setattr(cli_mod, "_run_kanban_goal_loop_q", lambda c, resp: calls.append(resp)) + return calls + + +def _quiet_goal_env(monkeypatch): + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + monkeypatch.setenv("HERMES_KANBAN_GOAL_MODE", "1") + + +def test_goal_mode_does_not_continue_after_recovery_denial(monkeypatch, capsys, goal_spy): + """Round-3 P1: with goal mode ON, a DENIED recovery (claim lost) must reach + the shared exit path without goal continuation — the goal loop's status + check sees only run identity, so continuing would re-enter the model under + an authority this process can no longer prove.""" + import agent.kanban_turn_recovery as rec + + _quiet_goal_env(monkeypatch) + monkeypatch.setattr(rec, "worker_claim_is_live", lambda: False) # recovery denied + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return _failed() + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 75 # timeout stamp → transient release; gate still vetoes + assert len(runs) == 1 # the failed turn only: no recovery nudge, and + assert goal_spy == [] # NO goal continuation after the denial + assert "no longer holds a live run/claim" in capsys.readouterr().err + + +def test_goal_mode_does_not_continue_when_the_lease_expires_during_the_backoff( + clear_kanban_env, tmp_path, monkeypatch, goal_spy +): + """Round-3 P1, the reviewer's exact reproduced fixture, end to end through + ``cli.main``: both leases at 1600, pre-backoff proof at 1000 passes, the + backoff crosses to 1700, the re-proof refuses — and goal mode (which sees a + still-running identity-only status) must NOT start another model turn.""" + import agent.kanban_turn_recovery as rec + + db = _make_board(tmp_path, task_expires=1_600, run_expires=1_600) + _pin_carrier(clear_kanban_env, db, run_id="1", lock="lk") + monkeypatch.setenv("HERMES_KANBAN_GOAL_MODE", "1") + monkeypatch.setattr(rec, "RECOVERY_DELAYS_SECONDS", (0.0,)) + clock = iter([1_000, 1_700]) + monkeypatch.setattr(rec, "_now", lambda: next(clock)) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return _failed() + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 75 # timeout stamp → transient release; gate still vetoes + assert len(runs) == 1 # goal-on: still exactly ONE model call (was 2 before the gate) + assert goal_spy == [] + + +@pytest.mark.parametrize("reason", ["billing", "upstream_rate_limit"]) +def test_goal_mode_does_not_continue_on_a_quota_wall(monkeypatch, goal_spy, reason): + """The neutral 75 must arrive without a sibling continuation making another + quota-bound call first — goal mode is skipped for a quota-wall result.""" + _quiet_goal_env(monkeypatch) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return _failed(reason=reason) + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 75 + assert len(runs) == 1 + assert goal_spy == [] + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + ({**_failed(), "interrupted": True}, 130), # explicit cancellation: 130, never continued + ({**_failed(retryable=True), "turn_exit_reason": "max_iterations_reached(60/60)"}, + 75), # #87096 settlement: transient stamp → neutral release, never continued + ], + ids=["interrupt", "terminal-settlement"], +) +def test_goal_mode_does_not_continue_on_interrupt_or_terminal_settlement( + monkeypatch, goal_spy, result, expected +): + _quiet_goal_env(monkeypatch) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return result + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == expected + assert len(runs) == 1 + assert goal_spy == [] + + +def test_goal_mode_does_not_continue_after_the_recovery_budget_is_exhausted( + monkeypatch, live_claim, goal_spy +): + """Budget exhausted (recovery attempted, still failed): the goal loop must + not take over as an unbounded continuation channel past the recovery cap.""" + import agent.kanban_turn_recovery as rec + + _quiet_goal_env(monkeypatch) + monkeypatch.setenv("HERMES_KANBAN_TURN_RECOVERY", "1") + monkeypatch.setattr(rec, "RECOVERY_DELAYS_SECONDS", (0.0,)) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return _failed() + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 75 # exhausted transient wall: released, not counted + assert len(runs) == 2 # original + the one authorised recovery attempt + assert goal_spy == [] # …and nothing continues past the exhausted budget + + +@pytest.mark.parametrize( + "result", + [ + {"partial": True, "completed": False, "final_response": "cut off", "error": "truncated", + "messages": []}, + ], + ids=["partial"], +) +def test_goal_mode_does_not_continue_on_an_unfinished_nonfailed_turn(monkeypatch, goal_spy, result): + """Pinned semantics for the gate's breadth: an unfinished-but-not-failed turn + (partial / completed=0) also does NOT continue in goal mode — only a settled, + authorized turn continues. Truncation/compression repair is owned inside the + conversation loop (#89289) and by the dispatcher, not by the goal judge.""" + _quiet_goal_env(monkeypatch) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return result + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 1 + assert len(runs) == 1 + assert goal_spy == [] + + +def test_goal_mode_continues_after_a_successful_authorized_recovery( + monkeypatch, live_claim, goal_spy +): + """Positive control: goal continuation is PRESERVED for a settled, + authorized turn — recovery succeeds and the judge-driven loop runs with the + recovered response.""" + import agent.kanban_turn_recovery as rec + + _quiet_goal_env(monkeypatch) + monkeypatch.setenv("HERMES_KANBAN_TURN_RECOVERY", "2") + monkeypatch.setattr(rec, "RECOVERY_DELAYS_SECONDS", (0.0,)) + runs: list = [] + + def run_conversation(*, user_message, conversation_history): + runs.append(user_message) + return _failed() if len(runs) == 1 else _success() + + cli_mod = _install_quiet_fake_cli(monkeypatch, run_conversation) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=True, toolsets="terminal") + + assert exc_info.value.code == 0 + assert len(runs) == 2 # original failed turn + the recovery turn + assert goal_spy == ["done"] # continuation happened, with the recovered response + + +# ── the same gate on the production `-q` route (goal_mode rides `-q`) ── + + +@pytest.fixture +def chat_goal_spy(monkeypatch): + """Spy on the `-q` goal continuation loop. Same invariant as ``goal_spy``: + the gate under test is the CALL-SITE, not the loop's own status check.""" + import cli as cli_mod + + calls: list = [] + monkeypatch.setattr(cli_mod, "_run_kanban_goal_loop_chat", lambda c, resp: calls.append(resp)) + return calls + + +def test_non_quiet_goal_mode_does_not_continue_after_recovery_denial( + monkeypatch, chat_goal_spy +): + """The dispatcher spawns goal_mode cards on the ``-q`` route + (``kanban_db_dispatch.py`` builds ``chat -q`` and sets + ``HERMES_KANBAN_GOAL_MODE=1``), so the round-3 finding has to be closed + there too: a DENIED recovery (claim lost) reaches the shared exit path with + zero further model entry.""" + import agent.kanban_turn_recovery as rec + + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + monkeypatch.setenv("HERMES_KANBAN_GOAL_MODE", "1") + monkeypatch.setattr(rec, "worker_claim_is_live", lambda: False) # recovery denied + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [_failed()], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 75 # transient stamp → released, not counted + assert len(_chats(calls)) == 1 # the failed turn only: no recovery nudge + assert chat_goal_spy == [] # NO goal continuation after the denial + + +def test_non_quiet_goal_mode_continues_after_a_successful_authorized_recovery( + monkeypatch, live_claim, chat_goal_spy +): + """Positive control on the ``-q`` route: a settled, authorized turn still + continues in goal mode — the gate must not swallow the happy path — and the + judge receives the RECOVERED response, not the initial provider error.""" + import agent.kanban_turn_recovery as rec + + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_probe") + monkeypatch.setenv("HERMES_KANBAN_GOAL_MODE", "1") + monkeypatch.setenv("HERMES_KANBAN_TURN_RECOVERY", "2") + monkeypatch.setattr(rec, "RECOVERY_DELAYS_SECONDS", (0.0,)) + calls: list = [] + cli_mod = _install_fake_cli(monkeypatch, [_failed(), _success()], calls) + + with pytest.raises(SystemExit) as exc_info: + cli_mod.main(query="hello", quiet=False, oneshot=True, toolsets="terminal") + + assert exc_info.value.code == 0 + assert len(_chats(calls)) == 2 + # …and the judge must receive the RECOVERED response, not the initial provider + # error (re-review P2): the recovery callback's return value carries through. + assert chat_goal_spy == ["done"]