mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 03:12:19 +03:00
f123bb5296
Review finding "one incomplete-response counterexample": on a background-completion turn a reply with finish_reason="tool_calls" and no tool calls ended as notification_no_reply after one call, so the action the model announced was dropped. That finish reason now keeps the recovery ladder, whose re-prompt recovers the follow-up as on main.
378 lines
16 KiB
Python
378 lines
16 KiB
Python
"""Regression tests for the machinery-turn empty-reply quiet exit (#139299).
|
|
|
|
Empty responses on machinery turns (``internal_notification``, ``process_complete``,
|
|
``async_delegation_complete`` display kind) end the turn quietly: no nudge, no
|
|
paid retries, no provider switch, no ``(empty)`` sentinel; the turn is closed with
|
|
the ``[SILENT]`` row so the next human prompt is not merged into the notice.
|
|
Ordinary turns and machinery turns where the model already ran a tool keep the
|
|
full ladder.
|
|
"""
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from agent import turn_empty_response as ter
|
|
|
|
|
|
def _agent():
|
|
a = MagicMock()
|
|
a.model = "test/model"
|
|
a.provider = "test"
|
|
a._current_streamed_assistant_text = ""
|
|
a._has_content_after_think_block.return_value = False
|
|
a._strip_think_blocks.side_effect = lambda t: t or ""
|
|
a._last_content_with_tools = None
|
|
a._thinking_prefill_retries = 2
|
|
a._fallback_chain = None
|
|
a._empty_content_retries = 0
|
|
a._emit_diagnostic_status = MagicMock()
|
|
a._buffer_diagnostic_status = MagicMock()
|
|
a._flush_status_buffer = MagicMock()
|
|
a._build_assistant_message = MagicMock(return_value={"role": "assistant", "content": ""})
|
|
# MagicMock.getattr(foo, "bar", False) returns a truthy child mock unless
|
|
# the attribute is explicitly set to a falsy value. The real AIAgent
|
|
# object never auto-creates these attributes.
|
|
a._post_tool_empty_retried = False
|
|
a._response_was_previewed = False
|
|
return a
|
|
|
|
|
|
def _assistant_msg():
|
|
return SimpleNamespace(reasoning=None, reasoning_content=None, reasoning_details=None)
|
|
|
|
|
|
def _empty_response():
|
|
choice = SimpleNamespace(message=_assistant_msg(), finish_reason="stop")
|
|
choice.message.content = ""
|
|
choice.message.tool_calls = None
|
|
return SimpleNamespace(choices=[choice], model="test/model", usage=None)
|
|
|
|
|
|
def _text_response(text):
|
|
response = _empty_response()
|
|
response.choices[0].message.content = text
|
|
return response
|
|
|
|
|
|
@pytest.fixture
|
|
def loop_agent(monkeypatch):
|
|
from run_agent import AIAgent
|
|
|
|
with (
|
|
patch("model_tools.get_tool_definitions", return_value=[]),
|
|
patch("model_tools.check_toolset_requirements", return_value={}),
|
|
patch("agent.process_bootstrap.OpenAI"),
|
|
):
|
|
agent = AIAgent(
|
|
api_key="test-key-1234567890",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
model="test/model",
|
|
quiet_mode=True,
|
|
skip_context_files=True,
|
|
skip_memory=True,
|
|
)
|
|
agent.client = MagicMock()
|
|
agent.client.chat.completions.create.side_effect = lambda **_kwargs: _empty_response()
|
|
agent._cached_system_prompt = "You are helpful."
|
|
agent._use_prompt_caching = False
|
|
agent.tool_delay = 0
|
|
agent.compression_enabled = False
|
|
agent.save_trajectories = False
|
|
monkeypatch.setattr("model_tools.handle_function_call", MagicMock(return_value="unused"))
|
|
monkeypatch.setattr("agent.retry_utils.jittered_backoff", lambda *_args, **_kwargs: 0)
|
|
monkeypatch.setattr("agent.turn_empty_response.interruptible_backoff_sleep", lambda *_args, **_kwargs: None)
|
|
monkeypatch.setattr(agent, "_persist_session", MagicMock())
|
|
monkeypatch.setattr(agent, "_save_trajectory", MagicMock())
|
|
monkeypatch.setattr(agent, "_cleanup_task_resources", MagicMock())
|
|
return agent
|
|
|
|
|
|
def _recover(agent, messages, **kwargs):
|
|
return ter.recover_empty_response(
|
|
agent,
|
|
_assistant_msg(),
|
|
None,
|
|
"stop",
|
|
final_response="",
|
|
messages=messages,
|
|
api_messages=[],
|
|
conversation_history=[],
|
|
active_system_prompt="sys",
|
|
api_call_count=1,
|
|
turn_exit_reason=None,
|
|
preflight_compression_blocked=False,
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
def _completed_tool_round():
|
|
return [
|
|
{"role": "user", "content": "Run the background task."},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"type": "function",
|
|
"function": {"name": "terminal", "arguments": "{}"},
|
|
}
|
|
],
|
|
},
|
|
{"role": "tool", "content": "Process exited successfully.", "tool_call_id": "call_1"},
|
|
]
|
|
|
|
|
|
class TestNotificationTurnLoop:
|
|
@pytest.mark.parametrize(
|
|
("display_kind", "prompt"),
|
|
[
|
|
("process_complete", "[process completed] Background task finished."),
|
|
("internal_notification", "[kanban] Board updated."),
|
|
],
|
|
)
|
|
def test_machinery_empty_reply_exits_without_nudge(self, loop_agent, display_kind, prompt):
|
|
result = loop_agent.run_conversation(
|
|
prompt,
|
|
conversation_history=_completed_tool_round(),
|
|
persist_user_display_kind=display_kind,
|
|
)
|
|
|
|
assert loop_agent.client.chat.completions.create.call_count == 1
|
|
assert result["final_response"] == ""
|
|
assert result["turn_exit_reason"] == "notification_no_reply"
|
|
provider_messages = loop_agent.client.chat.completions.create.call_args.kwargs["messages"]
|
|
assert all(
|
|
"You just executed tool calls but returned an empty response."
|
|
not in str(message.get("content") or "")
|
|
for message in provider_messages
|
|
)
|
|
|
|
def test_announced_tool_call_without_payload_is_recovered_not_silenced(self, loop_agent):
|
|
"""finish_reason="tool_calls" with no tool calls is a dropped action, not "nothing to add":
|
|
the turn keeps the recovery ladder and the model's follow-up is delivered."""
|
|
def respond(**_kwargs):
|
|
if loop_agent.client.chat.completions.create.call_count == 1:
|
|
dropped = _empty_response()
|
|
dropped.choices[0].finish_reason = "tool_calls"
|
|
return dropped
|
|
return _text_response("did the thing")
|
|
|
|
loop_agent.client.chat.completions.create.side_effect = respond
|
|
result = loop_agent.run_conversation(
|
|
"[process completed] Background task finished.",
|
|
conversation_history=_completed_tool_round(),
|
|
persist_user_display_kind="process_complete",
|
|
)
|
|
|
|
assert loop_agent.client.chat.completions.create.call_count == 2
|
|
assert result["final_response"] == "did the thing"
|
|
assert result.get("turn_exit_reason") != "notification_no_reply"
|
|
|
|
def test_human_turn_empty_reply_still_retries(self, loop_agent):
|
|
loop_agent.run_conversation(
|
|
"What happened?",
|
|
conversation_history=_completed_tool_round(),
|
|
)
|
|
|
|
assert loop_agent.client.chat.completions.create.call_count > 1
|
|
|
|
def test_next_human_turn_is_not_merged_into_the_quiet_notice(self, loop_agent):
|
|
notice = "[process completed] Background task finished."
|
|
first = loop_agent.run_conversation(
|
|
notice,
|
|
conversation_history=_completed_tool_round(),
|
|
persist_user_display_kind="process_complete",
|
|
)
|
|
assert first["turn_exit_reason"] == "notification_no_reply"
|
|
|
|
# The quiet exit must leave the notice answered in the transcript; otherwise the
|
|
# pre-call repair merges the next human prompt into the notice row.
|
|
loop_agent.client.chat.completions.create.side_effect = lambda **_kwargs: _text_response("Sure.")
|
|
second = loop_agent.run_conversation("now do X", conversation_history=first["messages"])
|
|
|
|
wire = loop_agent.client.chat.completions.create.call_args.kwargs["messages"]
|
|
assert wire[-1]["role"] == "user"
|
|
assert wire[-1]["content"] == "now do X"
|
|
assert [m["role"] for m in wire[-3:]] == ["user", "assistant", "user"]
|
|
assert wire[-3]["content"] == notice
|
|
|
|
rows = [m for m in second["messages"] if m.get("role") == "user"]
|
|
assert rows[-1]["content"] == "now do X"
|
|
assert rows[-1].get("display_kind") is None
|
|
assert rows[-2]["content"] == notice
|
|
assert rows[-2].get("display_kind") == "process_complete"
|
|
|
|
|
|
class TestNotificationTurnDetection:
|
|
def test_user_row_with_process_complete_no_tool_is_notification_turn(self):
|
|
messages = [
|
|
{"role": "user", "content": "hello"},
|
|
{"role": "assistant", "content": "hi!"},
|
|
{"role": "user", "content": "[process completed] task done", "display_kind": "process_complete"},
|
|
]
|
|
assert ter._notification_turn_without_tool_round(messages) is True
|
|
|
|
def test_user_row_with_async_delegation_complete_no_tool_is_notification_turn(self):
|
|
messages = [
|
|
{"role": "user", "content": "hello"},
|
|
{"role": "assistant", "content": "hi!"},
|
|
{"role": "user", "content": "[delegate finished]", "display_kind": "async_delegation_complete"},
|
|
]
|
|
assert ter._notification_turn_without_tool_round(messages) is True
|
|
|
|
def test_notification_turn_with_tool_is_not_detected(self):
|
|
messages = [
|
|
{"role": "user", "content": "[process completed]", "display_kind": "process_complete"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "x", "arguments": "{}"}}],
|
|
},
|
|
{"role": "tool", "content": "result", "tool_call_id": "call_1"},
|
|
]
|
|
assert ter._notification_turn_without_tool_round(messages) is False
|
|
|
|
def test_ordinary_user_turn_is_not_detected(self):
|
|
messages = [
|
|
{"role": "user", "content": "ordinary message"},
|
|
]
|
|
assert ter._notification_turn_without_tool_round(messages) is False
|
|
|
|
def test_ordinary_user_turn_with_tools_is_not_detected(self):
|
|
messages = [
|
|
{"role": "user", "content": "run task"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "x", "arguments": "{}"}}],
|
|
},
|
|
{"role": "tool", "content": "result", "tool_call_id": "call_1"},
|
|
]
|
|
assert ter._notification_turn_without_tool_round(messages) is False
|
|
|
|
def test_empty_messages_is_not_detected(self):
|
|
assert ter._notification_turn_without_tool_round([]) is False
|
|
|
|
def test_notification_with_nudge_row_as_user_injection_is_not_detected(self):
|
|
"""The model replied with text, then the user sent a follow-up message:
|
|
that follow-up is the message the next turn answers, not a notification."""
|
|
messages = [
|
|
{"role": "user", "content": "[process completed]", "display_kind": "process_complete"},
|
|
{"role": "assistant", "content": "got it"},
|
|
{"role": "user", "content": "now do X"},
|
|
]
|
|
assert ter._notification_turn_without_tool_round(messages) is False
|
|
|
|
|
|
class TestNotificationQuietExit:
|
|
@pytest.mark.parametrize(
|
|
"display_kind", ["internal_notification", "process_complete", "async_delegation_complete"]
|
|
)
|
|
def test_quiet_exit_closes_the_turn_with_the_silence_row(self, display_kind):
|
|
agent = _agent()
|
|
messages = [{"role": "user", "content": "[notice]", "display_kind": display_kind}]
|
|
verdict = _recover(agent, messages)
|
|
assert verdict.action == "break"
|
|
assert verdict.turn_exit_reason == ter.NOTIFICATION_NO_REPLY_EXIT
|
|
# One closing assistant row, the same marker a deliberate no-reply leaves on a human
|
|
# turn; never the "(empty)" failure sentinel.
|
|
assert [m["role"] for m in messages] == ["user", "assistant"]
|
|
assert messages[-1]["content"] == ter.MACHINERY_NO_REPLY_ROW == "[SILENT]"
|
|
assert not messages[-1].get("_empty_terminal_sentinel")
|
|
assert not messages[-1].get("_empty_recovery_synthetic")
|
|
|
|
def test_process_complete_empty_reply_exits_quietly(self):
|
|
agent = _agent()
|
|
messages = [
|
|
{"role": "user", "content": "[process completed]", "display_kind": "process_complete"},
|
|
]
|
|
verdict = _recover(agent, messages)
|
|
assert verdict.action == "break"
|
|
assert verdict.final_response == ""
|
|
assert verdict.turn_exit_reason == ter.NOTIFICATION_NO_REPLY_EXIT
|
|
|
|
def test_async_delegation_complete_empty_reply_exits_quietly(self):
|
|
agent = _agent()
|
|
messages = [
|
|
{"role": "user", "content": "delegate X returned", "display_kind": "async_delegation_complete"},
|
|
]
|
|
verdict = _recover(agent, messages)
|
|
assert verdict.action == "break"
|
|
assert verdict.final_response == ""
|
|
assert verdict.turn_exit_reason == ter.NOTIFICATION_NO_REPLY_EXIT
|
|
|
|
def test_notification_turn_with_tool_mid_task_not_quiet_exit(self):
|
|
"""A model that acted on a notification with a tool call is mid-task;
|
|
_notification_turn_without_tool_round is False, so the quiet exit path
|
|
is skipped. The post-tool nudge fires instead (verdict is 'continue'
|
|
with the nudge queued as a request-only hint; the transcript is untouched)."""
|
|
agent = _agent()
|
|
agent._last_content_with_tools = "mid-task reply"
|
|
agent._last_content_tools_all_housekeeping = False
|
|
messages = [
|
|
{"role": "user", "content": "[process completed]", "display_kind": "process_complete"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "mid-task reply",
|
|
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "x", "arguments": "{}"}}],
|
|
},
|
|
{"role": "tool", "content": "result", "tool_call_id": "call_1"},
|
|
]
|
|
verdict = _recover(agent, messages)
|
|
# Should not exit via notification_no_reply.
|
|
assert verdict.turn_exit_reason != ter.NOTIFICATION_NO_REPLY_EXIT
|
|
# Post-tool nudge fired: verdict is "continue" and the nudge rides the next request only.
|
|
assert verdict.action == "continue"
|
|
from agent.conversation_loop import _EMPTY_TOOL_RESPONSE_NUDGE
|
|
|
|
assert agent._empty_response_retry_hint == _EMPTY_TOOL_RESPONSE_NUDGE
|
|
assert len(messages) == 3
|
|
|
|
|
|
class TestResponseFiltersNotificationNoReply:
|
|
def test_notification_no_reply_with_empty_text_is_intentional_silence(self):
|
|
from gateway.response_filters import is_intentional_silence_agent_result
|
|
|
|
agent_result = {"failed": False, "turn_exit_reason": ter.NOTIFICATION_NO_REPLY_EXIT}
|
|
assert is_intentional_silence_agent_result(agent_result, "") is True
|
|
assert is_intentional_silence_agent_result(agent_result, " ") is True
|
|
|
|
def test_notification_no_reply_with_non_empty_text_is_not_silence(self):
|
|
from gateway.response_filters import is_intentional_silence_agent_result
|
|
|
|
agent_result = {"failed": False, "turn_exit_reason": ter.NOTIFICATION_NO_REPLY_EXIT}
|
|
assert is_intentional_silence_agent_result(agent_result, "got it") is False
|
|
|
|
def test_notification_no_reply_failed_turn_is_not_silence(self):
|
|
from gateway.response_filters import is_intentional_silence_agent_result
|
|
|
|
agent_result = {"failed": True, "turn_exit_reason": ter.NOTIFICATION_NO_REPLY_EXIT}
|
|
assert is_intentional_silence_agent_result(agent_result, "") is False
|
|
|
|
def test_ordinary_empty_response_exhausted_is_not_notification_silence(self):
|
|
"""An empty_response_exhausted turn with no text and no silence
|
|
marker is not intentional silence -- it falls through to normal
|
|
delivery."""
|
|
from gateway.response_filters import is_intentional_silence_agent_result
|
|
|
|
agent_result = {"failed": False, "turn_exit_reason": "empty_response_exhausted"}
|
|
# "(empty)" is not a LIVE_GATEWAY_SILENT_MARKERS entry; it reaches
|
|
# delivery through the empty-response path, not silence.
|
|
assert is_intentional_silence_agent_result(agent_result, "(empty)") is False
|
|
# But with the [SILENT] marker it IS silence (unrelated to notification path).
|
|
assert is_intentional_silence_agent_result(agent_result, "[SILENT]") is True
|
|
|
|
def test_notification_no_reply_alongside_real_silence_marker(self):
|
|
from gateway.response_filters import is_intentional_silence_agent_result
|
|
|
|
agent_result = {"failed": False, "turn_exit_reason": ter.NOTIFICATION_NO_REPLY_EXIT}
|
|
assert is_intentional_silence_agent_result(agent_result, "[SILENT]") is True
|
|
|
|
def test_notification_no_reply_none_result_is_not_silence(self):
|
|
from gateway.response_filters import is_intentional_silence_agent_result
|
|
|
|
assert is_intentional_silence_agent_result(None, "") is False
|