Files
hermes-agent/tests/agent/test_fallback_trigger_logged.py
kshitijk4poor 6ee6b7cb77 fix(agent): name the cause of a refusal fallback and log every buffered fallback trigger
An HTTP-200 refusal (Anthropic stop_reason=refusal, finish_reason=content_filter)
switched to the fallback without a reason, so the notice read "unavailable
(provider failure)". The same refusal raised as an HTTP 400 (DeepSeek's
"Content Exists Risk") goes through the classifier and reads "content policy
blocked the request". The refusal handler and the content-filter stream path
now pass FailoverReason.content_policy_blocked, which is not a rate-limit
reason, so no cooldown or reset deferral changes.

The refusal WARNING (native stop reason + stop_details) was only written when
no fallback took over, and every other non-exception fallback trigger reported
its cause only through _buffer_diagnostic_status, which _clear_status_buffer
drops once the fallback answers. Each of those sites (eager invalid-response
switch, Nous rate guard, Codex reasoning-only stall, content-filter stream,
Codex app-server turn failure) now logs one WARNING naming the cause when the
switch happens. The refusal line caps stop_details like the refusal text. The existing
"Fallback activated: A -> B" line still names the models.

The credential pool's "no available entries" line now names its provider: an
unrelated pool's line read as the cause of nearby fallbacks.

Fixes #124874
Fixes #152101
2026-11-10 02:26:26 +05:30

96 lines
4.6 KiB
Python

"""A fallback whose trigger is only buffered leaves its cause in the log (#152101).
The "switching to fallback" status is only buffered, and the buffer is dropped once the fallback
answers, so without a log line nothing records why the session left its primary model.
"""
from __future__ import annotations
import logging
from types import SimpleNamespace
from unittest.mock import MagicMock
from agent.error_classifier import FailoverReason
from agent.turn_response_check import retry_invalid_response
from agent.turn_retry_state import TurnRetryState
def _agent(switches: bool) -> MagicMock:
agent = MagicMock()
agent.log_prefix, agent.quiet_mode, agent.api_mode = "", True, "chat_completions"
agent.provider, agent.model = "nous", "anthropic/claude-opus-5.5"
agent._fallback_chain, agent._fallback_index = [{"provider": "custom", "model": "fb"}], 0
agent._try_activate_fallback.return_value = switches
return agent
def _run(agent, caplog):
with caplog.at_level(logging.WARNING, logger="agent.conversation_loop"):
return retry_invalid_response(
agent, response=SimpleNamespace(choices=[], error=None), error_details=["response.choices is empty"],
_retry=TurnRetryState(), thinking_spinner=None, messages=[], api_messages=[], api_kwargs=None,
active_system_prompt=None, conversation_history=None, retry_count=0, max_retries=3,
compression_attempts=0, api_call_count=1, api_request_id="r", api_start_time=0.0, api_duration=0.4,
effective_task_id="t", turn_id="turn",
)
def _switch_lines(caplog) -> list[str]:
return [r.getMessage() for r in caplog.records if "switched to fallback" in r.getMessage()]
def test_invalid_response_fallback_logs_its_trigger(caplog):
verdict = _run(_agent(switches=True), caplog)
assert verdict.action == "break"
assert _switch_lines(caplog) == ["Invalid API response (response.choices is empty) — switched to fallback"]
def test_declined_fallback_logs_no_switch(caplog):
_run(_agent(switches=False), caplog)
assert _switch_lines(caplog) == []
def test_content_filter_stream_fallback_names_reason_and_logs(caplog):
from agent.turn_truncation import _Trunc, _content_filter_fallback
agent = _agent(switches=True)
st = _Trunc(
agent=agent, response=SimpleNamespace(_content_filter_terminated=True), finish_reason="length",
conversation_history=[], api_call_count=1, effective_task_id="t", current_turn_user_idx=None,
result=None, messages=[], length_continue_retries=0, truncated_response_parts=[],
truncated_tool_call_retries=0, retry_count=0, compression_attempts=0,
)
with caplog.at_level(logging.WARNING, logger="agent.conversation_loop"):
verdict = _content_filter_fallback(st, TurnRetryState())
assert verdict is not None and verdict.action == "break"
agent._try_activate_fallback.assert_called_once_with(reason=FailoverReason.content_policy_blocked)
assert _switch_lines(caplog) == ["Content filter terminated the stream — switched to fallback"]
def test_nous_rate_guard_fallback_logs(monkeypatch, caplog):
from agent import nous_rate_guard
from agent.turn_api_call import nous_rate_limit_guard
monkeypatch.setattr(nous_rate_guard, "nous_rate_limit_remaining", lambda **kw: 600)
monkeypatch.setattr("hermes_cli.anon_auth.is_anonymous_agent", lambda agent: False)
monkeypatch.setattr("hermes_cli.anon_auth.apply_model_switch", lambda agent: None)
monkeypatch.setattr("agent.conversation_loop._arm_fallback_restart", lambda *a: "sys")
agent = _agent(switches=True)
with caplog.at_level(logging.WARNING, logger="agent.conversation_loop"):
verdict = nous_rate_limit_guard(
agent, _retry=TurnRetryState(), api_messages=[], messages=[], conversation_history=[],
active_system_prompt="system", retry_count=0, compression_attempts=0, api_call_count=0,
)
assert verdict.action == "break"
assert len(_switch_lines(caplog)) == 1 and _switch_lines(caplog)[0].startswith("Nous rate limit recorded")
def test_codex_app_server_fallback_logs_its_trigger(caplog):
from agent.turn_recovery import activate_codex_app_server_fallback
agent = _agent(switches=True)
agent._has_pending_fallback.return_value = True
with caplog.at_level(logging.WARNING, logger="agent.conversation_loop"):
assert activate_codex_app_server_fallback(agent, {"error": "429 Too Many Requests"}) is True
agent._try_activate_fallback.assert_called_once_with(reason=FailoverReason.rate_limit)
assert _switch_lines(caplog) == ["Codex app-server turn failed (rate_limit: 429 Too Many Requests) — switched to fallback"]