mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:04:08 +03:00
344de92b59
Four agent flags record that the CURRENT route refused something: - _reasoning_effort_rejected: the route 400'd an enabled reasoning effort, so every later request omits the reasoning fields (teknium1, 2026-09-19,64dceb9c26, #115736); - _reasoning_disable_rejected: the route refused a reasoning disable, so disables are dropped (Teknium, 2026-09-03,f6bd1633f7, #102108); - _reasoning_floor_required: the route requires a minimum effort, so disables become effort low (teknium1, 2026-09-19,2768f31628, #115935); - _stream_options_unsupported: the endpoint 400/422'd stream_options, so streams stop asking for usage (DavidMetcalfe, 2026-06-26,6527af2286, #109143). None was ever reset. After a /model switch, a fallback activation, a primary restore, or a credential rotation that moved the endpoint, the new route kept the old route's verdicts: a model that takes reasoning effort fine had its reasoning stripped for the rest of the session because a different model had rejected it, and a provider that reports streaming usage lost token accounting because a different endpoint had refused stream_options. The voice route already treated the reasoning three as "facts about one route" and saved/cleared/restored them by hand, and the encrypted-reasoning replay verdict already resets on every route change (#61552). New agent/route_verdicts.py holds the per-route flag list and reset_route_verdicts(), which clears them and the encrypted-reasoning replay verdict. It replaces the replay-only reset at the route-change sites: switch_model (agent_runtime_helpers), fallback activation (bind_route_entry), snapshot reinstall (reinstall_runtime_snapshot), and now also _reapply_route_client_config when route_changed (credential-pool rotation and the .env credential refresh, which can point the agent at another base URL). Verdicts are kept with the route rather than thrown away: - the hop that leaves the primary saves its verdicts into the primary's _primary_runtime snapshot and reinstall_primary_runtime puts them back, so a fallback episode does not make the primary re-learn each one with a failed request. Living in that snapshot, they go away when /model rebuilds it, so one primary's verdicts never reach the next. They are stamped with the endpoint that earned them and put back only after the primary's credential pool is rebound and only if it landed on that same endpoint (a pool rotation can move the primary to another base URL); - a /model X --once restore (CLI and TUI) snapshots the primary together with its current verdicts (primary_runtime_for_restore), so the original primary comes back with its own verdicts even when X failed over; - a .env credential adoption whose client rebuild fails rolls the verdicts back with the old route; - a switch_model whose client rebuild fails rolls the verdicts back with the rest of the old route (_SWITCH_SNAPSHOT_FIELDS); - the voice route saves/restores the full list (ROUTE_VERDICT_FIELDS) and drops its own clearing loop, which bind_route_entry now does. That also fixes main losing its encrypted-reasoning replay verdict (and the voice model inheriting main's stream_options verdict) on every voice turn. _oauth_1m_beta_disabled is left alone: the Anthropic OAuth long-context beta is refused per account, not per route, and a switch rebuilds the client anyway. Tests: fallback clears and primary restore returns the primary's own verdicts (test_primary_runtime_restore.py, the #61552 replay test updated to match); a /model switch clears them and a failed switch keeps them (test_switch_model_context.py); rotation to another base URL clears them and same-URL rotation keeps them (test_credential_rotation_route_settings.py); /model --once restores the original primary's verdicts even after X failed over, a /model switch made while on a fallback starts the new primary clean, a primary that rotated to another base URL before failing over does not bring that endpoint's verdicts back to its own, and a failed .env adoption keeps the verdicts (test_route_verdicts_once_restore.py); a voice turn leaves main's stream_options and replay verdicts intact and sends stream_options to the voice endpoint (test_voice_turn_route.py). Removing each piece fails its test.
423 lines
14 KiB
Python
423 lines
14 KiB
Python
"""Tests that switch_model does not inherit stale context_length overrides."""
|
|
|
|
import contextlib
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from hermes_cli.models_local import LMStudioLoadResult
|
|
from run_agent import AIAgent
|
|
from hermes_cli.route_identity import normalize_route_base_url
|
|
from agent.context_compressor import ContextCompressor
|
|
|
|
|
|
class _StubStartupCompressor:
|
|
def __init__(self, *args, **kwargs):
|
|
self.context_length = kwargs.get("config_context_length") or 272_000
|
|
self.config_context_length = kwargs.get("config_context_length")
|
|
self.threshold_tokens = int(self.context_length * 0.95)
|
|
self.threshold_percent = 0.95
|
|
|
|
def get_tool_schemas(self):
|
|
return []
|
|
|
|
def on_session_start(self, *args, **kwargs):
|
|
return None
|
|
|
|
|
|
def test_route_url_normalization_preserves_path_slash_before_query():
|
|
"""A path slash before a query changes OpenAI SDK URL joining."""
|
|
assert normalize_route_base_url(
|
|
"https://example.com/v1/?tenant=large"
|
|
) != normalize_route_base_url("https://example.com/v1?tenant=large")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _make_direct_start_agent(
|
|
cfg: dict, *, model: str, provider: str, base_url: str
|
|
) -> AIAgent:
|
|
with (
|
|
patch("hermes_cli.config.load_config", return_value=cfg), patch("hermes_cli.config.load_config_readonly", return_value=cfg),
|
|
patch("model_tools.get_tool_definitions", return_value=[]),
|
|
patch("model_tools.check_toolset_requirements", return_value={}),
|
|
patch("agent.process_bootstrap.OpenAI"),
|
|
patch("agent.agent_init.ContextCompressor", new=_StubStartupCompressor),
|
|
):
|
|
return AIAgent(
|
|
model=model,
|
|
provider=provider,
|
|
api_key="fake-test-token",
|
|
base_url=base_url,
|
|
quiet_mode=True,
|
|
skip_context_files=True,
|
|
skip_memory=True,
|
|
)
|
|
|
|
|
|
def _make_agent_with_compressor(config_context_length=None) -> AIAgent:
|
|
"""Build a minimal AIAgent with a context_compressor, skipping __init__."""
|
|
agent = AIAgent.__new__(AIAgent)
|
|
|
|
# Primary model settings
|
|
agent.model = "primary-model"
|
|
agent.provider = "openrouter"
|
|
agent.base_url = "https://openrouter.ai/api/v1"
|
|
agent.api_key = "sk-primary"
|
|
agent.api_mode = "chat_completions"
|
|
agent.client = MagicMock()
|
|
agent.quiet_mode = True
|
|
|
|
# Store the initial config_context_length override used at agent construction.
|
|
agent._config_context_length = config_context_length
|
|
|
|
# Context compressor with primary model values
|
|
compressor = ContextCompressor(
|
|
model="primary-model",
|
|
threshold_percent=0.50,
|
|
base_url="https://openrouter.ai/api/v1",
|
|
api_key="sk-primary",
|
|
provider="openrouter",
|
|
quiet_mode=True,
|
|
config_context_length=config_context_length,
|
|
)
|
|
agent.context_compressor = compressor
|
|
|
|
# For switch_model
|
|
agent._primary_runtime = {}
|
|
|
|
return agent
|
|
|
|
|
|
@patch("agent.model_metadata.get_model_context_length", return_value=131_072)
|
|
def test_switch_model_clears_previous_config_context_length(mock_ctx_len):
|
|
"""Switching models must not reuse the previous model.context_length override."""
|
|
agent = _make_agent_with_compressor(config_context_length=32_768)
|
|
|
|
assert agent.context_compressor.model == "primary-model"
|
|
assert agent.context_compressor.context_length == 32_768 # From config override
|
|
|
|
# Switch model
|
|
agent.switch_model("new-model", "openrouter", api_key="sk-new", base_url="https://openrouter.ai/api/v1")
|
|
|
|
# Verify the old config override is not passed to the new model.
|
|
mock_ctx_len.assert_called_once()
|
|
call_kwargs = mock_ctx_len.call_args.kwargs
|
|
assert call_kwargs.get("config_context_length") is None
|
|
|
|
# Verify compressor was updated from the newly resolved model metadata.
|
|
assert agent.context_compressor.model == "new-model"
|
|
assert agent.context_compressor.context_length == 131_072
|
|
|
|
|
|
|
|
|
|
def test_switch_model_reapplies_checked_auxiliary_compression_limit():
|
|
"""A model switch re-probes the auxiliary summariser eagerly: the trigger is clamped to its window
|
|
before the first compaction on the new model, and un-clamped again when a later switch fits (#114707)."""
|
|
agent = _make_agent_with_compressor(config_context_length=200_000)
|
|
agent.compression_enabled = True
|
|
agent._compression_feasibility_checked = False
|
|
agent._aux_compression_context_length_config = None
|
|
agent._custom_providers = []
|
|
agent._compression_warning = None
|
|
agent._print_fn = None
|
|
agent.log_prefix = ""
|
|
agent.suppress_status_output = False
|
|
agent._stream_consumers = []
|
|
agent._executing_tools = False
|
|
agent._mute_post_response = False
|
|
agent.status_callback = None
|
|
agent.tool_progress_callback = None
|
|
agent.tools = []
|
|
agent._emit_status = lambda _message: None
|
|
agent._create_openai_client = lambda *_args, **_kwargs: MagicMock()
|
|
|
|
aux_client = MagicMock(base_url="https://aux.example/v1", api_key="aux-key")
|
|
with (
|
|
patch(
|
|
"agent.model_metadata.get_model_context_length",
|
|
side_effect=[400_000, 80_000],
|
|
),
|
|
patch(
|
|
"agent.auxiliary_client.get_text_auxiliary_client",
|
|
return_value=(aux_client, "auxiliary-model"),
|
|
),
|
|
):
|
|
agent.switch_model(
|
|
"larger-main-model",
|
|
"openrouter",
|
|
api_key="sk-new",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
|
|
assert agent.context_compressor.context_length == 400_000
|
|
assert agent.context_compressor.threshold_tokens == 80_000
|
|
assert agent._compression_feasibility_checked is True
|
|
# A same-runtime window correction (overflow-reported limit) keeps the aux ceiling.
|
|
agent.context_compressor.update_model(
|
|
model="larger-main-model", context_length=1_000_000, base_url="https://openrouter.ai/api/v1",
|
|
api_key="sk-new", provider="openrouter", api_mode="chat_completions",
|
|
)
|
|
assert agent.context_compressor.threshold_tokens == 80_000
|
|
# Switching to a runtime whose aux fits restores the main-model trigger (no one-way ratchet).
|
|
with (
|
|
patch("agent.model_metadata.get_model_context_length", side_effect=[400_000, 400_000]),
|
|
patch("agent.auxiliary_client.get_text_auxiliary_client", return_value=(aux_client, "auxiliary-model")),
|
|
):
|
|
agent.switch_model(
|
|
"other-main-model", "openrouter", api_key="sk-new", base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
assert agent.context_compressor.threshold_tokens == 300_000
|
|
|
|
|
|
def test_switch_model_omitted_base_url_preserves_direct_openai_capability():
|
|
"""A same-provider switch resolves capabilities from the retained URL."""
|
|
agent = _make_agent_with_compressor(config_context_length=None)
|
|
agent.provider = "openai"
|
|
agent.model = "gpt-5.6"
|
|
agent.base_url = "https://api.openai.com/v1"
|
|
agent.runtime_capabilities = {"native_compaction": True}
|
|
agent._create_openai_client = lambda *_args, **_kwargs: MagicMock()
|
|
|
|
with patch("agent.model_metadata.get_model_context_length", return_value=128_000):
|
|
agent.switch_model("gpt-5.6", "openai", api_key="sk-new")
|
|
|
|
assert agent.base_url == "https://api.openai.com/v1"
|
|
assert agent.runtime_capabilities == {"native_compaction": True}
|
|
|
|
|
|
def test_cross_provider_switch_to_default_openai_preserves_native_capability():
|
|
agent = _make_agent_with_compressor(config_context_length=None)
|
|
agent.provider = "openrouter"
|
|
agent.model = "gpt-5.5"
|
|
agent.base_url = "https://openrouter.ai/api/v1"
|
|
agent.runtime_capabilities = {"native_compaction": False}
|
|
agent._create_openai_client = lambda *_args, **_kwargs: MagicMock()
|
|
|
|
with patch("agent.model_metadata.get_model_context_length", return_value=128_000):
|
|
agent.switch_model("gpt-5.6", "openai", api_key="sk-new")
|
|
|
|
assert agent.base_url == "https://api.openai.com/v1"
|
|
assert agent.runtime_capabilities == {"native_compaction": True}
|
|
|
|
|
|
def test_direct_start_model_override_does_not_inherit_profile_context_length():
|
|
"""A CLI ``--model`` startup override must not inherit another model's window."""
|
|
cfg = {
|
|
"model": {
|
|
"default": "kimi-k3",
|
|
"provider": "custom:kimi-coding-1m",
|
|
"base_url": "https://api.kimi.com/coding",
|
|
"context_length": 1_048_576,
|
|
},
|
|
"custom_providers": [
|
|
{
|
|
"name": "kimi-coding-1m",
|
|
"base_url": "https://api.kimi.com/coding",
|
|
"models": {"kimi-k3": {"context_length": 1_048_576}},
|
|
}
|
|
],
|
|
}
|
|
agent = _make_direct_start_agent(
|
|
cfg,
|
|
model="gpt-5.6-sol",
|
|
provider="openai-codex",
|
|
base_url="https://chatgpt.com/backend-api/codex",
|
|
)
|
|
|
|
assert agent.context_compressor.config_context_length is None
|
|
assert agent.context_compressor.context_length == 272_000
|
|
|
|
|
|
def test_direct_start_preserves_context_for_normalized_default_model_alias():
|
|
"""Equivalent vendor-prefixed defaults still own their explicit window."""
|
|
cfg = {
|
|
"model": {
|
|
"default": "openai/gpt-5.6-sol",
|
|
"provider": "openai-codex",
|
|
"base_url": "https://chatgpt.com/backend-api/codex",
|
|
"context_length": 272_000,
|
|
}
|
|
}
|
|
|
|
agent = _make_direct_start_agent(
|
|
cfg,
|
|
model="gpt-5.6-sol",
|
|
provider="openai-codex",
|
|
base_url="https://chatgpt.com/backend-api/codex",
|
|
)
|
|
|
|
assert agent.context_compressor.config_context_length == 272_000
|
|
assert agent.context_compressor.context_length == 272_000
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_lmstudio_switch_uses_destination_context_and_verified_runtime(monkeypatch):
|
|
agent = _make_agent_with_compressor(config_context_length=32_768)
|
|
calls = []
|
|
|
|
def fake_load_config():
|
|
return {}
|
|
|
|
def fake_compatible(_cfg):
|
|
return [{"name": "lmstudio", "base_url": "http://127.0.0.1:1234/v1"}]
|
|
|
|
def fake_provider_context(*, model, base_url, custom_providers):
|
|
assert model == "lmstudio/new-model"
|
|
assert base_url == "http://127.0.0.1:1234/v1"
|
|
return 120_000
|
|
|
|
def fake_lmstudio_load(self, config_context_length=None):
|
|
calls.append(config_context_length)
|
|
return LMStudioLoadResult(100_000)
|
|
|
|
monkeypatch.setattr("hermes_cli.config.load_config", fake_load_config)
|
|
|
|
monkeypatch.setattr("hermes_cli.config.load_config_readonly", fake_load_config)
|
|
monkeypatch.setattr("hermes_cli.config.get_compatible_custom_providers", fake_compatible)
|
|
monkeypatch.setattr("hermes_cli.config.get_custom_provider_context_length", fake_provider_context)
|
|
monkeypatch.setattr(AIAgent, "_ensure_lmstudio_runtime_loaded", fake_lmstudio_load)
|
|
|
|
with patch("agent.model_metadata.get_model_context_length", return_value=100_000) as mock_ctx_len:
|
|
agent.switch_model(
|
|
"lmstudio/new-model",
|
|
"lmstudio",
|
|
api_key="",
|
|
base_url="http://127.0.0.1:1234/v1",
|
|
)
|
|
|
|
assert calls == [120_000]
|
|
call_kwargs = mock_ctx_len.call_args.kwargs
|
|
assert call_kwargs.get("config_context_length") == 100_000
|
|
assert agent._config_context_length == 120_000
|
|
assert agent.context_compressor.context_length == 100_000
|
|
|
|
|
|
def test_later_lmstudio_failure_restores_runtime_capabilities(monkeypatch):
|
|
agent = _make_agent_with_compressor(config_context_length=32_768)
|
|
agent.runtime_capabilities = {"native_compaction": True}
|
|
original_client = agent.client
|
|
|
|
monkeypatch.setattr(
|
|
AIAgent,
|
|
"_ensure_lmstudio_runtime_loaded",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
|
RuntimeError("simulated LM Studio failure")
|
|
),
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="simulated LM Studio failure"):
|
|
agent.switch_model(
|
|
"new-model",
|
|
"openrouter",
|
|
api_key="sk-new",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
|
|
assert agent.model == "primary-model"
|
|
assert agent.provider == "openrouter"
|
|
assert agent.client is original_client
|
|
assert agent.runtime_capabilities == {"native_compaction": True}
|
|
|
|
|
|
@patch("agent.model_metadata.get_model_context_length", return_value=131_072)
|
|
def test_switch_model_starts_the_new_route_without_the_old_routes_rejections(_ctx):
|
|
"""``/model``: a reasoning-effort, reasoning-disable or stream_options rejection learned on the
|
|
old route must not keep stripping those fields from the new route's requests."""
|
|
from agent.route_verdicts import ROUTE_REJECTION_FIELDS
|
|
|
|
agent = _make_agent_with_compressor()
|
|
for name in ROUTE_REJECTION_FIELDS:
|
|
setattr(agent, name, True)
|
|
agent.switch_model("new-model", "openrouter", api_key="sk-new", base_url="https://openrouter.ai/api/v1")
|
|
assert {name: getattr(agent, name) for name in ROUTE_REJECTION_FIELDS} == dict.fromkeys(ROUTE_REJECTION_FIELDS, False)
|
|
|
|
|
|
@patch("agent.model_metadata.get_model_context_length", return_value=131_072)
|
|
def test_failed_switch_model_keeps_the_old_routes_rejections(_ctx):
|
|
"""A switch whose client rebuild fails rolls back onto the old route, verdicts included."""
|
|
from agent.route_verdicts import ROUTE_VERDICT_FIELDS
|
|
|
|
agent = _make_agent_with_compressor()
|
|
for name in ROUTE_VERDICT_FIELDS:
|
|
setattr(agent, name, name != "_codex_reasoning_replay_enabled")
|
|
before = {name: getattr(agent, name) for name in ROUTE_VERDICT_FIELDS}
|
|
with patch.object(type(agent), "_create_openai_client", side_effect=RuntimeError("boom")), \
|
|
contextlib.suppress(Exception):
|
|
agent.switch_model("new-model", "openrouter", api_key="sk-new", base_url="https://openrouter.ai/api/v1")
|
|
assert agent.model != "new-model"
|
|
assert {name: getattr(agent, name) for name in ROUTE_VERDICT_FIELDS} == before
|