fix(agent): retry once without stream_options when an endpoint rejects it (HTTP 400/422)

Azure AI Foundry serverless (MaaS) endpoints validate the request body
strictly and reject `stream_options: {"include_usage": true}` with 422
`extra_forbidden`. Hermes sent the field on every streaming call, so the
agent was unusable against that endpoint family and the fallback chain
failed too.

When a 400/422 names `stream_options` as an extra/unsupported field and no
delta has been delivered yet, re-open the stream without the field and
remember the rejection on the agent (`_stream_options_unsupported`) so later
turns skip it up front. Streaming itself stays on — this is not the
"stream not supported" case. Usage accounting for such endpoints falls back
to the estimator, as it already does for native Gemini.

Salvage of PR #53271 by @DavidMetcalfe, reshaped onto the `_StreamingCall`
retry loop with per-agent state instead of a process-wide host set; one
end-to-end invariant test through `_interruptible_streaming_api_call`.

Fixes #9705
This commit is contained in:
DavidMetcalfe
2026-06-26 14:04:42 -07:00
committed by Teknium
Unverified
parent 7817af2a16
commit 6527af2286
3 changed files with 64 additions and 2 deletions
+2
View File
@@ -627,6 +627,8 @@ _STREAM_STATE: Dict[str, Any] = {
"_stream_writer_token": 0,
"_stream_writer_tls": threading.local,
"_stream_writer_dropped": 0,
# Set once a strict endpoint 400/422s on ``stream_options``; later streams omit it (#9705).
"_stream_options_unsupported": False,
# API-facing user message override when it differs from the persisted transcript (voice).
"_persist_user_message_idx": None,
"_persist_user_message_override": None,
+24 -2
View File
@@ -2216,6 +2216,18 @@ _SSE_CONN_PHRASES = ("connection lost", "connection reset", "connection closed",
"upstream connect error")
def _rejects_stream_options(exc: BaseException) -> bool:
"""A 400/422 whose body names ``stream_options`` as an unknown/extra field: strict
OpenAI-compatible endpoints (Azure AI Foundry MaaS, Pydantic ``extra_forbidden``) reject
the usage extension outright (#9705). Distinct from "stream not supported", which flips
the whole session to non-streaming."""
if getattr(exc, "status_code", None) not in (400, 422):
return False
body = f"{getattr(exc, 'body', '') or ''} {exc}".lower()
return "stream_options" in body and any(
k in body for k in ("extra", "not supported", "unrecognized", "unexpected", "unknown"))
def _is_sse_connection_error(exc: BaseException) -> bool:
from openai import APIError as _APIError
if not isinstance(exc, _APIError) or getattr(exc, "status_code", None):
@@ -2694,8 +2706,9 @@ class _StreamingCall(StreamingWaitMonitor):
return usage, finish_reason
def _open_chat_stream(self, stream_kwargs: dict[str, Any]):
# Native Gemini rejects OpenAI's usage-streaming extension.
if not is_native_gemini_base_url(self.agent.base_url):
# Native Gemini rejects OpenAI's usage-streaming extension; so do strict endpoints that
# already 4xx'd on it this session (``_stream_options_unsupported``, see #9705).
if not is_native_gemini_base_url(self.agent.base_url) and not getattr(self.agent, "_stream_options_unsupported", False):
stream_kwargs["stream_options"] = {"include_usage": True}
request_client = self._attempt_request_client = self.clients.set_client(
self.agent._create_request_openai_client(reason="chat_completion_stream_request", api_kwargs=stream_kwargs))
@@ -3103,6 +3116,15 @@ class _StreamingCall(StreamingWaitMonitor):
_is_sse_conn_err = not _is_timeout and not _is_conn_err and _is_sse_connection_error(e)
_is_transient = _is_timeout or _is_conn_err or _is_sse_conn_err or _is_stream_parse_err
if not self.deltas_were_sent["yes"] and not getattr(self.agent, "_stream_options_unsupported", False) and _rejects_stream_options(e):
# Nothing streamed yet: drop the usage extension for this session and re-open.
self.agent._stream_options_unsupported = True
logger.info("Endpoint rejected stream_options (HTTP %s); retrying without it for this session.",
getattr(e, "status_code", None))
self._cancel_current_stream_attempt("stream_options_rejected_retry")
self.clients.close_once("stream_options_rejected_retry")
return True
if self.deltas_were_sent["yes"]:
# Died AFTER tokens were delivered: normally no retry (would duplicate
# text). Exception: a tool call in flight — aborting discards it, so
+38
View File
@@ -260,6 +260,44 @@ class TestStreamingAccumulator:
assert call_kwargs["stream"] is True
assert "stream_options" not in call_kwargs
@patch("run_agent.AIAgent._create_request_openai_client")
@patch("run_agent.AIAgent._close_request_openai_client")
def test_endpoint_rejecting_stream_options_is_retried_without_it(self, mock_close, mock_create):
"""Strict OpenAI-compatible endpoints (Azure AI Foundry MaaS) 422 on
``stream_options.include_usage``; the call is retried once without the field and
the session remembers the rejection (#9705)."""
from openai import APIStatusError
from run_agent import AIAgent
body = {"detail": [{"type": "extra_forbidden", "loc": ["body", "stream_options", "include_usage"],
"msg": "Extra inputs are not permitted"}]}
rejection = APIStatusError("Unprocessable Entity", response=MagicMock(status_code=422), body=body)
rejection.status_code = 422
calls = []
def _create(**kwargs):
calls.append(kwargs)
if "stream_options" in kwargs:
raise rejection
return iter([_make_stream_chunk(content="ok", finish_reason="stop", model="mistral-small")])
mock_client = MagicMock()
mock_client.chat.completions.create.side_effect = _create
mock_create.return_value = mock_client
agent = AIAgent(api_key="k", base_url="https://hub.services.ai.azure.com/openai/v1",
model="mistral-small-2503", provider="custom", quiet_mode=True,
skip_context_files=True, skip_memory=True)
agent.api_mode = "chat_completions"
agent._interrupt_requested = False
response = agent._interruptible_streaming_api_call({})
assert response.choices[0].message.content == "ok"
assert [("stream_options" in c) for c in calls] == [True, False]
assert agent._stream_options_unsupported is True
assert agent._disable_streaming is False # streaming itself still works there
@patch("run_agent.AIAgent._create_request_openai_client")