mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:04:08 +03:00
fix(agent): retry once without stream_options when an endpoint rejects it (HTTP 400/422)
Azure AI Foundry serverless (MaaS) endpoints validate the request body
strictly and reject `stream_options: {"include_usage": true}` with 422
`extra_forbidden`. Hermes sent the field on every streaming call, so the
agent was unusable against that endpoint family and the fallback chain
failed too.
When a 400/422 names `stream_options` as an extra/unsupported field and no
delta has been delivered yet, re-open the stream without the field and
remember the rejection on the agent (`_stream_options_unsupported`) so later
turns skip it up front. Streaming itself stays on — this is not the
"stream not supported" case. Usage accounting for such endpoints falls back
to the estimator, as it already does for native Gemini.
Salvage of PR #53271 by @DavidMetcalfe, reshaped onto the `_StreamingCall`
retry loop with per-agent state instead of a process-wide host set; one
end-to-end invariant test through `_interruptible_streaming_api_call`.
Fixes #9705
This commit is contained in:
committed by
Teknium
Unverified
parent
7817af2a16
commit
6527af2286
@@ -627,6 +627,8 @@ _STREAM_STATE: Dict[str, Any] = {
|
||||
"_stream_writer_token": 0,
|
||||
"_stream_writer_tls": threading.local,
|
||||
"_stream_writer_dropped": 0,
|
||||
# Set once a strict endpoint 400/422s on ``stream_options``; later streams omit it (#9705).
|
||||
"_stream_options_unsupported": False,
|
||||
# API-facing user message override when it differs from the persisted transcript (voice).
|
||||
"_persist_user_message_idx": None,
|
||||
"_persist_user_message_override": None,
|
||||
|
||||
@@ -2216,6 +2216,18 @@ _SSE_CONN_PHRASES = ("connection lost", "connection reset", "connection closed",
|
||||
"upstream connect error")
|
||||
|
||||
|
||||
def _rejects_stream_options(exc: BaseException) -> bool:
|
||||
"""A 400/422 whose body names ``stream_options`` as an unknown/extra field: strict
|
||||
OpenAI-compatible endpoints (Azure AI Foundry MaaS, Pydantic ``extra_forbidden``) reject
|
||||
the usage extension outright (#9705). Distinct from "stream not supported", which flips
|
||||
the whole session to non-streaming."""
|
||||
if getattr(exc, "status_code", None) not in (400, 422):
|
||||
return False
|
||||
body = f"{getattr(exc, 'body', '') or ''} {exc}".lower()
|
||||
return "stream_options" in body and any(
|
||||
k in body for k in ("extra", "not supported", "unrecognized", "unexpected", "unknown"))
|
||||
|
||||
|
||||
def _is_sse_connection_error(exc: BaseException) -> bool:
|
||||
from openai import APIError as _APIError
|
||||
if not isinstance(exc, _APIError) or getattr(exc, "status_code", None):
|
||||
@@ -2694,8 +2706,9 @@ class _StreamingCall(StreamingWaitMonitor):
|
||||
return usage, finish_reason
|
||||
|
||||
def _open_chat_stream(self, stream_kwargs: dict[str, Any]):
|
||||
# Native Gemini rejects OpenAI's usage-streaming extension.
|
||||
if not is_native_gemini_base_url(self.agent.base_url):
|
||||
# Native Gemini rejects OpenAI's usage-streaming extension; so do strict endpoints that
|
||||
# already 4xx'd on it this session (``_stream_options_unsupported``, see #9705).
|
||||
if not is_native_gemini_base_url(self.agent.base_url) and not getattr(self.agent, "_stream_options_unsupported", False):
|
||||
stream_kwargs["stream_options"] = {"include_usage": True}
|
||||
request_client = self._attempt_request_client = self.clients.set_client(
|
||||
self.agent._create_request_openai_client(reason="chat_completion_stream_request", api_kwargs=stream_kwargs))
|
||||
@@ -3103,6 +3116,15 @@ class _StreamingCall(StreamingWaitMonitor):
|
||||
_is_sse_conn_err = not _is_timeout and not _is_conn_err and _is_sse_connection_error(e)
|
||||
_is_transient = _is_timeout or _is_conn_err or _is_sse_conn_err or _is_stream_parse_err
|
||||
|
||||
if not self.deltas_were_sent["yes"] and not getattr(self.agent, "_stream_options_unsupported", False) and _rejects_stream_options(e):
|
||||
# Nothing streamed yet: drop the usage extension for this session and re-open.
|
||||
self.agent._stream_options_unsupported = True
|
||||
logger.info("Endpoint rejected stream_options (HTTP %s); retrying without it for this session.",
|
||||
getattr(e, "status_code", None))
|
||||
self._cancel_current_stream_attempt("stream_options_rejected_retry")
|
||||
self.clients.close_once("stream_options_rejected_retry")
|
||||
return True
|
||||
|
||||
if self.deltas_were_sent["yes"]:
|
||||
# Died AFTER tokens were delivered: normally no retry (would duplicate
|
||||
# text). Exception: a tool call in flight — aborting discards it, so
|
||||
|
||||
@@ -260,6 +260,44 @@ class TestStreamingAccumulator:
|
||||
assert call_kwargs["stream"] is True
|
||||
assert "stream_options" not in call_kwargs
|
||||
|
||||
@patch("run_agent.AIAgent._create_request_openai_client")
|
||||
@patch("run_agent.AIAgent._close_request_openai_client")
|
||||
def test_endpoint_rejecting_stream_options_is_retried_without_it(self, mock_close, mock_create):
|
||||
"""Strict OpenAI-compatible endpoints (Azure AI Foundry MaaS) 422 on
|
||||
``stream_options.include_usage``; the call is retried once without the field and
|
||||
the session remembers the rejection (#9705)."""
|
||||
from openai import APIStatusError
|
||||
from run_agent import AIAgent
|
||||
|
||||
body = {"detail": [{"type": "extra_forbidden", "loc": ["body", "stream_options", "include_usage"],
|
||||
"msg": "Extra inputs are not permitted"}]}
|
||||
rejection = APIStatusError("Unprocessable Entity", response=MagicMock(status_code=422), body=body)
|
||||
rejection.status_code = 422
|
||||
calls = []
|
||||
|
||||
def _create(**kwargs):
|
||||
calls.append(kwargs)
|
||||
if "stream_options" in kwargs:
|
||||
raise rejection
|
||||
return iter([_make_stream_chunk(content="ok", finish_reason="stop", model="mistral-small")])
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.chat.completions.create.side_effect = _create
|
||||
mock_create.return_value = mock_client
|
||||
|
||||
agent = AIAgent(api_key="k", base_url="https://hub.services.ai.azure.com/openai/v1",
|
||||
model="mistral-small-2503", provider="custom", quiet_mode=True,
|
||||
skip_context_files=True, skip_memory=True)
|
||||
agent.api_mode = "chat_completions"
|
||||
agent._interrupt_requested = False
|
||||
|
||||
response = agent._interruptible_streaming_api_call({})
|
||||
|
||||
assert response.choices[0].message.content == "ok"
|
||||
assert [("stream_options" in c) for c in calls] == [True, False]
|
||||
assert agent._stream_options_unsupported is True
|
||||
assert agent._disable_streaming is False # streaming itself still works there
|
||||
|
||||
|
||||
|
||||
@patch("run_agent.AIAgent._create_request_openai_client")
|
||||
|
||||
Reference in New Issue
Block a user