mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:04:08 +03:00
fix: step reasoning up to the floor when a route refuses to disable it
Endpoints that understand the reasoning field but refuse the OFF ("Reasoning is
mandatory for this endpoint and cannot be disabled" -- the Nous Portal on
gpt-6-astra) got two different wrong answers:
* auxiliary lanes (title generation sends reasoning_config={"enabled": False})
400'd outright -- the #112781 strip rung only matched "unsupported/unknown
field" wordings, so every session on such a route stayed untitled;
* the main loop dropped the disable and let the route pick its default effort,
the opposite of what a thinking-off user asked for.
Both now step the effort up to the lowest level every reasoning wire accepts
(low) instead. The aux ladder gets a rung ordered before the strip
(agent/auxiliary_reasoning_floor.py: lifts reasoning_effort / extra_body.reasoning
/ _reasoning_config, memoises the (route, model) so the next thinking-off aux call
starts at the floor without the guaranteed 400); the main loop's reasoning_mandatory
recovery records whether the route said mandatory (floor) or unknown-field (drop,
unchanged). error_classifier.is_reasoning_required_rejection separates the two.
Live on the portal (proxy wire capture): title none->400->low->200 titled;
main loop none->400->low->200; second aux call on the route sends low up front.
This commit is contained in:
committed by
Teknium
Unverified
parent
a2978e746b
commit
2768f31628
@@ -29,7 +29,9 @@ from agent.error_classifier import (
|
||||
_OVERLOADED_PATTERNS,
|
||||
UNSUPPORTED_PARAM_MARKERS,
|
||||
is_reasoning_field_rejection,
|
||||
is_reasoning_required_rejection,
|
||||
)
|
||||
from agent.auxiliary_reasoning_floor import remember_reasoning_floor, with_reasoning_floor
|
||||
from agent.auxiliary_structured_output import remember_structured_output_rejection
|
||||
from agent.codex_headers import (
|
||||
CODEX_AUX_BASE_URL as _CODEX_AUX_BASE_URL,
|
||||
@@ -3302,6 +3304,16 @@ def _is_reasoning_field_rejection(exc: Exception) -> bool:
|
||||
return is_reasoning_field_rejection(str(exc))
|
||||
|
||||
|
||||
def _is_reasoning_required_rejection(exc: Exception) -> bool:
|
||||
"""Provider 400 refusing to switch reasoning OFF ("Reasoning is mandatory for this endpoint and cannot
|
||||
be disabled"): the field is understood, only the disable is refused, so the rung steps the effort up to
|
||||
the floor instead of dropping the field (agent/auxiliary_reasoning_floor.py)."""
|
||||
status = getattr(exc, "status_code", None)
|
||||
if status is not None and status not in {400, 422}:
|
||||
return False
|
||||
return is_reasoning_required_rejection(str(exc))
|
||||
|
||||
|
||||
def _without_reasoning_fields(kwargs: dict) -> Optional[dict]:
|
||||
"""Copy *kwargs* without reasoning wire controls (top-level ``reasoning_effort``, the adapter's
|
||||
private ``_reasoning_config`` and every ``extra_body`` reasoning key); None when nothing was
|
||||
@@ -6464,7 +6476,9 @@ def _build_call_kwargs(
|
||||
# OpenAI-compat wire ONCE here, before either path sees the config — the same entry clamp the
|
||||
# main transport applies (#89503); MoA aggregator/reference and aux calls 400'd without it (#112010).
|
||||
from agent.reasoning_effort import clamp_reasoning_config
|
||||
reasoning_config = clamp_reasoning_config(reasoning_config)
|
||||
from agent.auxiliary_reasoning_floor import known_reasoning_floor
|
||||
reasoning_config = clamp_reasoning_config(
|
||||
known_reasoning_floor(reasoning_config, provider_norm, effective_base, model, task))
|
||||
projection = _project_provider_profile(provider, provider_norm, model, effective_base, reasoning_config)
|
||||
kwargs.update(projection.top_level)
|
||||
merged_extra = _merge_aux_extra_body(extra_body, projection, reasoning_config, provider_norm)
|
||||
@@ -7184,7 +7198,8 @@ def _param_rung_accepts(exc: Exception) -> bool:
|
||||
# a temperature-strip retry on max_tokens), and a route-gating 400 after a strip still
|
||||
# reaches the provider-fallback rung.
|
||||
or _is_unsupported_parameter_error(exc, "temperature")
|
||||
or _is_reasoning_field_rejection(exc) or _is_structured_output_rejection(exc)
|
||||
or _is_reasoning_field_rejection(exc) or _is_reasoning_required_rejection(exc)
|
||||
or _is_structured_output_rejection(exc)
|
||||
or _is_model_incompatible_error(exc))
|
||||
|
||||
|
||||
@@ -7238,6 +7253,12 @@ def _parameter_rungs(client: Any, max_tokens: Optional[int]) -> tuple:
|
||||
# (top-level ``reasoning_effort: none``), and strict-schema gateways reject the generic
|
||||
# ``extra_body.reasoning`` fallback outright (#109774); the caller only wanted "no thinking",
|
||||
# so retry with every reasoning field omitted and let the route default apply (#112781).
|
||||
# The endpoint refuses the *disable* rather than the field (Nous Portal gpt-6-astra: "Reasoning is
|
||||
# mandatory ... cannot be disabled"): step the effort up to the floor and remember the route so
|
||||
# the next thinking-off aux call starts there. Ordered before the strip so a floor that still
|
||||
# 400s falls through to it.
|
||||
(_is_reasoning_required_rejection, with_reasoning_floor,
|
||||
"provider requires reasoning; retrying at the floor effort", remember_reasoning_floor),
|
||||
(_is_reasoning_field_rejection, _without_reasoning_fields,
|
||||
"provider rejected the reasoning field; retrying without it (route default applies)", None),
|
||||
(lambda exc: max_tokens is not None and _is_max_tokens_rejection(exc, client), _without_max_tokens,
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Reasoning floor for auxiliary requests: when a route refuses to switch reasoning OFF, step it UP.
|
||||
|
||||
Aux lanes that want speed over thought (title generation: ``max_tokens=64``, JSON body) send the
|
||||
provider's thinking-off encoding — top-level ``reasoning_effort: "none"`` on the custom profile,
|
||||
``extra_body.reasoning: {"enabled": false}`` on OpenRouter-shaped relays, ``_reasoning_config`` on the
|
||||
Anthropic Messages adapters. Some endpoints understand the field but refuse the disable ("Reasoning is
|
||||
mandatory for this endpoint and cannot be disabled" — the Nous Portal on gpt-6-astra); before this
|
||||
module every title call there 400'd and the session stayed untitled.
|
||||
|
||||
The recovery is a *step up*, not a strip: the same request goes out again at the lowest effort every
|
||||
reasoning wire accepts (``low``; ``minimal`` is rejected by o-series and Claude), and the (route, model)
|
||||
pair is memoised so the next disabled-reasoning aux call on it starts at the floor instead of burning
|
||||
the guaranteed 400 first. Dropping the field would also succeed once, but it says nothing about the
|
||||
next call and hands the effort choice back to the provider default (often ``medium`` or higher, the
|
||||
opposite of what a thinking-off caller asked for).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REASONING_FLOOR_EFFORT = "low"
|
||||
|
||||
# (route key, model) pairs that refused a reasoning disable in this process.
|
||||
_FLOORED_ROUTES: set[tuple[str, str]] = set()
|
||||
|
||||
_DISABLED_EFFORTS = {"none", "off", "disabled", "false", "0"}
|
||||
|
||||
|
||||
def _route_key(provider: Optional[str], base_url: Optional[str]) -> str:
|
||||
"""Endpoint host:port when known (a base_url override turns a named provider into ``custom``), else
|
||||
the provider name — the same key shape ``auxiliary_structured_output`` uses."""
|
||||
return (urlparse(base_url or "").netloc or "").lower() or str(provider or "").strip().lower()
|
||||
|
||||
|
||||
def _is_disabled(reasoning_config: Any) -> bool:
|
||||
return isinstance(reasoning_config, dict) and reasoning_config.get("enabled") is False
|
||||
|
||||
|
||||
def floor_reasoning_config(reasoning_config: Any) -> Dict[str, Any]:
|
||||
"""The caller's disabled ``reasoning_config`` lifted to the floor; anything else returned as-is."""
|
||||
if _is_disabled(reasoning_config):
|
||||
return {"enabled": True, "effort": REASONING_FLOOR_EFFORT}
|
||||
return reasoning_config
|
||||
|
||||
|
||||
def with_reasoning_floor(kwargs: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Copy of *kwargs* with every thinking-OFF encoding lifted to ``REASONING_FLOOR_EFFORT``:
|
||||
top-level ``reasoning_effort``, ``extra_body.reasoning`` (OpenRouter shape) and the adapter's private
|
||||
``_reasoning_config``. ``None`` when nothing was disabled, so the ladder never re-sends an unchanged
|
||||
request."""
|
||||
changed = False
|
||||
retry = dict(kwargs)
|
||||
if str(retry.get("reasoning_effort", "")).strip().lower() in _DISABLED_EFFORTS:
|
||||
retry["reasoning_effort"] = REASONING_FLOOR_EFFORT
|
||||
changed = True
|
||||
if _is_disabled(retry.get("_reasoning_config")):
|
||||
retry["_reasoning_config"] = floor_reasoning_config(retry["_reasoning_config"])
|
||||
changed = True
|
||||
extra_body = retry.get("extra_body")
|
||||
if isinstance(extra_body, dict):
|
||||
reasoning = extra_body.get("reasoning")
|
||||
if _is_disabled(reasoning) or (
|
||||
isinstance(reasoning, dict) and str(reasoning.get("effort", "")).strip().lower() in _DISABLED_EFFORTS
|
||||
):
|
||||
retry["extra_body"] = {**extra_body, "reasoning": {"enabled": True, "effort": REASONING_FLOOR_EFFORT}}
|
||||
changed = True
|
||||
return retry if changed else None
|
||||
|
||||
|
||||
def remember_reasoning_floor(
|
||||
provider: Optional[str], base_url: Optional[str], rejected_kwargs: Dict[str, Any], error: BaseException,
|
||||
) -> None:
|
||||
"""Record that this route's ``rejected_kwargs["model"]`` refuses to disable reasoning (the ladder
|
||||
calls this after the stepped-up retry succeeded)."""
|
||||
_FLOORED_ROUTES.add((_route_key(provider, base_url), str(rejected_kwargs.get("model") or "")))
|
||||
|
||||
|
||||
def known_reasoning_floor(
|
||||
reasoning_config: Any, provider: Optional[str], base_url: Optional[str], model: Optional[str],
|
||||
task: Optional[str] = None,
|
||||
) -> Any:
|
||||
"""*reasoning_config* lifted to the floor when this route+model is known to refuse a disable; unchanged
|
||||
otherwise. Runs before the profile projection so every wire shape starts at the floor."""
|
||||
if not _is_disabled(reasoning_config):
|
||||
return reasoning_config
|
||||
if (_route_key(provider, base_url), str(model or "")) not in _FLOORED_ROUTES:
|
||||
return reasoning_config
|
||||
logger.info(
|
||||
"Auxiliary %s: %s (%s) cannot disable reasoning; sending effort=%s up front",
|
||||
task or "call", _route_key(provider, base_url) or "provider", model or "model", REASONING_FLOOR_EFFORT,
|
||||
)
|
||||
return floor_reasoning_config(reasoning_config)
|
||||
@@ -1258,11 +1258,17 @@ def _reasoning_config_for_wire(agent):
|
||||
# The route rejects disables. Resend exactly what the session has
|
||||
# been sending — the user's own config — so the retry lands on the
|
||||
# same provider cache key as every prior request. Only a config that
|
||||
# is itself a disable is dropped (omitted → route default), and that
|
||||
# session has never sent anything else, so nothing warm is lost.
|
||||
# is itself a disable changes, and that session has never sent
|
||||
# anything else, so nothing warm is lost: a route that said the
|
||||
# disable is *mandatory-on* gets the floor effort (closest to what
|
||||
# the user asked for); a relay that does not know the field gets
|
||||
# nothing (route default).
|
||||
if isinstance(cfg, dict) and (
|
||||
cfg.get("enabled") is False or cfg.get("effort") == "none"
|
||||
):
|
||||
if getattr(agent, "_reasoning_floor_required", False):
|
||||
from agent.auxiliary_reasoning_floor import REASONING_FLOOR_EFFORT
|
||||
return {**cfg, "enabled": True, "effort": REASONING_FLOOR_EFFORT}
|
||||
return None
|
||||
return cfg
|
||||
if ephemeral_off:
|
||||
|
||||
@@ -475,6 +475,26 @@ _REASONING_FIELD_TOKEN = re.compile(
|
||||
)
|
||||
|
||||
|
||||
_REASONING_REQUIRED_MARKERS = (
|
||||
"mandatory", "cannot be disabled", "can't be disabled", "must be enabled", "is required",
|
||||
"always enabled", "cannot be turned off",
|
||||
)
|
||||
|
||||
|
||||
def is_reasoning_required_rejection(error_msg: str) -> bool:
|
||||
"""Provider 400 saying the model's reasoning cannot be switched OFF ("Reasoning is mandatory for
|
||||
this endpoint and cannot be disabled", the Nous Portal on gpt-6-astra). The opposite of
|
||||
``is_reasoning_field_rejection``: the field is understood, the *disable* is refused, so the right
|
||||
reaction is to step the effort up to the lowest level rather than drop the field (a dropped field
|
||||
also works, but tells the caller nothing about the next call)."""
|
||||
msg = (error_msg or "").lower()
|
||||
token = _REASONING_FIELD_TOKEN.search(msg)
|
||||
if token is None:
|
||||
return False
|
||||
near = msg[max(0, token.start() - 48):token.end() + 96]
|
||||
return any(m in near for m in _REASONING_REQUIRED_MARKERS)
|
||||
|
||||
|
||||
def is_reasoning_field_rejection(error_msg: str) -> bool:
|
||||
"""Provider 400 rejecting a reasoning wire control by name (``reasoning_effort``, ``reasoning``,
|
||||
``thinking``/``think``): the field token plus either a generic unsupported marker ("Unrecognized
|
||||
|
||||
+14
-2
@@ -609,13 +609,25 @@ def recover_after_classification(
|
||||
):
|
||||
_retry.reasoning_mandatory_retry_attempted = True
|
||||
agent._reasoning_disable_rejected = True
|
||||
# "Reasoning is mandatory ... cannot be disabled" understands the field and refuses only the
|
||||
# OFF: step up to the floor effort (the closest the route allows to what the user asked for)
|
||||
# rather than the route default. A relay that does not know the field at all keeps the
|
||||
# drop (a floor would 400 the same way).
|
||||
from agent.error_classifier import is_reasoning_required_rejection
|
||||
agent._reasoning_floor_required = is_reasoning_required_rejection(str(api_error))
|
||||
try:
|
||||
from hermes_cli.models_reasoning_caps import refresh_reasoning_caps_async
|
||||
refresh_reasoning_caps_async(agent.provider)
|
||||
except Exception:
|
||||
pass
|
||||
_vlines(agent, f"⚠️ {agent.model} rejects disabling reasoning — using the route's default for this session, retrying...")
|
||||
logger.warning("%sReasoning-disable recovery: dropping reasoning disable for %s", agent.log_prefix, agent.model)
|
||||
if agent._reasoning_floor_required:
|
||||
from agent.auxiliary_reasoning_floor import REASONING_FLOOR_EFFORT
|
||||
_vlines(agent, f"⚠️ {agent.model} cannot disable reasoning — using effort={REASONING_FLOOR_EFFORT} for this session, retrying...")
|
||||
logger.warning("%sReasoning-disable recovery: stepping reasoning up to %s for %s",
|
||||
agent.log_prefix, REASONING_FLOOR_EFFORT, agent.model)
|
||||
else:
|
||||
_vlines(agent, f"⚠️ {agent.model} rejects disabling reasoning — using the route's default for this session, retrying...")
|
||||
logger.warning("%sReasoning-disable recovery: dropping reasoning disable for %s", agent.log_prefix, agent.model)
|
||||
return True, recovered_with_pool
|
||||
|
||||
# Provider rejected the image bytes; shrinking can't help, so strip image parts.
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Reasoning-required rejection → step the aux effort up to the floor and remember the route.
|
||||
|
||||
The title lane disables reasoning (``reasoning_config={"enabled": False}``), which the custom profile
|
||||
encodes as top-level ``reasoning_effort: "none"``. Endpoints that understand the field but refuse the
|
||||
disable (Nous Portal on gpt-6-astra: 400 "Reasoning is mandatory for this endpoint and cannot be
|
||||
disabled") used to fail the title outright — the strip rung (#112781) never matched this wording.
|
||||
The recovery is a step UP (``low``), memoised per (route, model) so the next thinking-off aux call
|
||||
on that route starts at the floor without the guaranteed 400.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import auxiliary_reasoning_floor
|
||||
from agent.auxiliary_client import call_llm
|
||||
|
||||
_PORTAL_400 = (
|
||||
"Error code: 400 - {'status': 400, 'message': 'This request is not valid. Check the model name and "
|
||||
"other parameters. Additional info: Reasoning is mandatory for this endpoint and cannot be disabled.'}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_memo():
|
||||
auxiliary_reasoning_floor._FLOORED_ROUTES.clear()
|
||||
yield
|
||||
auxiliary_reasoning_floor._FLOORED_ROUTES.clear()
|
||||
|
||||
|
||||
def _call(client):
|
||||
with (
|
||||
patch("agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("custom", "openai/gpt-6-astra", "http://127.0.0.1:8765/v1", "sk-x", None)),
|
||||
patch("agent.auxiliary_client._get_cached_client", return_value=(client, "openai/gpt-6-astra")),
|
||||
patch("agent.auxiliary_client._validate_llm_response", side_effect=lambda resp, _task, **_kw: resp),
|
||||
patch("agent.auxiliary_client._try_payment_fallback", return_value=None),
|
||||
):
|
||||
return call_llm(
|
||||
task="title_generation", messages=[{"role": "user", "content": "hi"}],
|
||||
extra_body={"response_format": {"type": "json_object"}}, reasoning_config={"enabled": False},
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_required_400_steps_effort_up_to_the_floor_and_remembers_the_route():
|
||||
"""First call: ``none`` → 400 → retry at the floor with everything else intact. Second call on the
|
||||
same route+model: the floor goes out up front, no 400 round-trip."""
|
||||
client = MagicMock()
|
||||
client.base_url = "http://127.0.0.1:8765/v1"
|
||||
client.chat.completions.create.side_effect = [RuntimeError(_PORTAL_400), {"ok": True}, {"ok": True}]
|
||||
|
||||
assert _call(client) == {"ok": True}
|
||||
first, retry = (c.kwargs for c in client.chat.completions.create.call_args_list[:2])
|
||||
assert first["reasoning_effort"] == "none"
|
||||
assert retry["reasoning_effort"] == auxiliary_reasoning_floor.REASONING_FLOOR_EFFORT
|
||||
assert retry["extra_body"]["response_format"] == {"type": "json_object"}
|
||||
assert retry["model"] == first["model"]
|
||||
|
||||
assert _call(client) == {"ok": True}
|
||||
assert client.chat.completions.create.call_count == 3
|
||||
upfront = client.chat.completions.create.call_args_list[2].kwargs
|
||||
assert upfront["reasoning_effort"] == auxiliary_reasoning_floor.REASONING_FLOOR_EFFORT
|
||||
|
||||
|
||||
def test_field_rejection_still_strips_instead_of_stepping_up():
|
||||
"""A relay that does not know the field at all keeps the #112781 behaviour: the field is dropped,
|
||||
nothing is remembered as a floor."""
|
||||
client = MagicMock()
|
||||
client.base_url = "https://relay.example/v1"
|
||||
client.chat.completions.create.side_effect = [
|
||||
RuntimeError("Error code: 400 - Unrecognized request argument supplied: reasoning_effort"), {"ok": True},
|
||||
]
|
||||
assert _call(client) == {"ok": True}
|
||||
retry = client.chat.completions.create.call_args_list[1].kwargs
|
||||
assert "reasoning_effort" not in retry
|
||||
assert not auxiliary_reasoning_floor._FLOORED_ROUTES
|
||||
@@ -84,6 +84,21 @@ class TestReasoningOffOneShotOverride:
|
||||
agent.reasoning_config = {"enabled": False}
|
||||
assert _reasoning_config_for_wire(agent) is None
|
||||
|
||||
def test_mandatory_reasoning_route_steps_a_disable_up_to_the_floor(self):
|
||||
""""Reasoning is mandatory ... cannot be disabled" understands the field: the session's
|
||||
disable becomes the floor effort (closest to what the user asked for), while a config that
|
||||
already reasons still goes out verbatim (cache key preserved)."""
|
||||
from agent.auxiliary_reasoning_floor import REASONING_FLOOR_EFFORT
|
||||
from agent.chat_completion_helpers import _reasoning_config_for_wire
|
||||
|
||||
agent = _AgentStandIn({"enabled": False})
|
||||
agent._reasoning_disable_rejected = True
|
||||
agent._reasoning_floor_required = True
|
||||
assert _reasoning_config_for_wire(agent) == {"enabled": True, "effort": REASONING_FLOOR_EFFORT}
|
||||
|
||||
agent.reasoning_config = {"enabled": True, "effort": "high"}
|
||||
assert _reasoning_config_for_wire(agent) == {"enabled": True, "effort": "high"}
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def loop_agent():
|
||||
|
||||
Reference in New Issue
Block a user