Files
Teknium f03d0e561c fix(mcp): routine discovery wakes a parked server once per park streak
_select_new_servers signalled _reconnect_event for every cached server
without a session on every register_mcp_servers() call (each new agent
session, cron tick, managed worker), bypassing the parked backoff and the
single_instance quiet park. A live repro re-running discovery every 5 s
spawned a failing stdio server 72 times in 3 minutes on main.

Discovery now gives a parked server one free wake per streak (the #50170
recovery path); once a revival has failed, or the server is quiet-parked,
the parked backoff decides. Every revival now counts toward the streak,
not only timed self-probes. Explicit reconnects (/reload-mcp,
hermes mcp login) still revive at once. Tests now wait on
tests.waiting.async_wait_until instead of real-time sleeps.
2026-11-19 10:38:40 -08:00

673 lines
38 KiB
Python

#!/usr/bin/env python3
"""MCP (Model Context Protocol) client: connects to the ``mcp_servers`` configured in
~/.hermes/config.yaml (stdio, Streamable HTTP or SSE), discovers their tools and registers them
into the hermes tool registry. The ``mcp`` package is optional (no-op without it).
One background event loop (``_mcp_loop``) in a daemon thread runs each server as a long-lived
Task (``MCPServerTask``) so the transport's anyio cancel scopes enter and exit in one Task; every
``_servers``/loop mutation holds ``_lock``. This module keeps the SDK loader, ``MCPServerTask`` and
all shared state; the ``mcp_tool_*`` siblings read that state back through ``tools.mcp_tool`` at
call time (``_core``) and are imported directly by their callers."""
import asyncio
import contextvars
import importlib
import importlib.util
import inspect
import logging
import os
import threading
import time
from typing import Any, Callable, Dict, List, Optional, Set
logger = logging.getLogger(__name__)
from tools.mcp_tool_common import _DEFAULT_TOOL_TIMEOUT, mcp_field
from tools.mcp_tool_config import _npx_cached_bin, _packaged_npx_entry
from tools.mcp_tool_sampling import ElicitationHandler, SamplingHandler
from tools.mcp_tool_transport import MCPServerTransportMixin
from tools.mcp_tool_server_run import MCPServerRunMixin
from tools.mcp_tool_health import MCPServerHealthMixin
# Wall-clock bound on the fail-open OSV malware preflight before a stdio spawn; just ABOVE
# osv_check._TIMEOUT (10s) so it only bites when a stalled SSL handshake defeats that.
_OSV_MALWARE_CHECK_TIMEOUT_S = 12.0
async def _preflight_stdio_command(server_name: str, command: str, args: list) -> tuple[str, list]:
"""OSV malware preflight (off-loop, wall-clock bound, fail-open on timeout), THEN the
cached-npx swap. The preflight must see the REAL command/args: anything that rewrites argv to a
wrapper or resolved binary has to happen after it, or the check silently inspects the wrapper
and becomes a no-op (``_infer_ecosystem`` keys off the command basename being npx/uvx/pipx)."""
from tools.osv_check import check_package_for_malware
try:
malware_error = await asyncio.wait_for(
asyncio.to_thread(check_package_for_malware, command, args), timeout=_OSV_MALWARE_CHECK_TIMEOUT_S)
except TimeoutError:
logger.warning("MCP server '%s': OSV malware preflight timed out after %.0fs "
"(network slow/unreachable) — proceeding without the check.",
server_name, _OSV_MALWARE_CHECK_TIMEOUT_S)
malware_error = None
if malware_error:
raise ValueError(f"MCP server '{server_name}': {malware_error}")
# npx resolves the package and then FORKS, staying resident as the real server's parent for
# nothing (~48 MB per server, measured). Hermes already supervises the child (shared death
# supervisor), so a cached package is spawned directly; a cache miss leaves npx untouched.
if os.path.basename(command).lower().startswith("npx"):
# A sealed Windows payload's own npx cannot start its node through cmd.exe.
packaged = await asyncio.to_thread(_packaged_npx_entry, command, args)
if packaged:
logger.debug("MCP server '%s': running %s on the packaged node", server_name, packaged[1][0])
return packaged
cached = _npx_cached_bin(args)
if cached:
direct_command, direct_args = cached
logger.debug("MCP server '%s': using cached npx binary %s (skipping the "
"resident `npm exec` parent)", server_name, direct_command)
command, args = direct_command, direct_args
return command, args
# ---- Optional MCP SDK: availability probe now, symbol import on first use ----
_MCP_AVAILABLE = _MCP_HTTP_AVAILABLE = _MCP_NEW_HTTP = _MCP_LEGACY_HTTP = False
_MCP_SAMPLING_TYPES = _MCP_NOTIFICATION_TYPES = _MCP_ELICITATION_TYPES = False
_MCP_MESSAGE_HANDLER_SUPPORTED = _MCP_LOGGING_CALLBACK_SUPPORTED = False
sse_client = None
# Fallback for SDKs without LATEST_PROTOCOL_VERSION (Streamable HTTP arrived with 2025-03-26).
LATEST_PROTOCOL_VERSION = "2025-03-26"
# Newest revision ``ClientSession.initialize()`` speaks; from 2026-07-28 the handshake is a
# per-request envelope so this can be OLDER than LATEST_PROTOCOL_VERSION, and the
# MCP-Protocol-Version header must be seeded from THIS one.
LATEST_HANDSHAKE_VERSION = LATEST_PROTOCOL_VERSION
# Importing ``mcp`` costs ~260ms, so it is deferred to first use (_ensure_mcp_sdk); availability
# is decided now via find_spec so every ``if not _MCP_AVAILABLE`` gate / patch / skipif holds.
try:
_MCP_AVAILABLE = importlib.util.find_spec("mcp") is not None
except Exception:
_MCP_AVAILABLE = False
if not _MCP_AVAILABLE:
logger.debug("mcp package not installed -- MCP tool support disabled")
ClientSession: Any = None
# True once the probe settled: every SDK import either loaded or is genuinely absent from this
# SDK build. A transient failure (a concurrent import's _DeadlockError, a partially initialized
# module, a dependency mid-swap during an update) leaves it False so the next call probes again.
_MCP_SDK_IMPORT_ATTEMPTED = False
_MCP_SDK_IMPORT_LOCK = threading.Lock()
# The ClientSession the probe itself published; any other non-None value is a pre-installed mock.
_MCP_SDK_PROBED_SESSION: Any = None
# Why the HTTP transport did not load (shown in the "requires HTTP transport" error); None if it did.
_MCP_HTTP_IMPORT_ERROR: Optional[str] = None
# Why the core SDK import failed on the last probe (shown in connect errors); None if it loaded.
_MCP_SDK_IMPORT_ERROR: Optional[str] = None
# CPython raises this (a RuntimeError, not an ImportError) when two threads import modules of a
# circular package such as ``mcp.client`` at once. It clears once the other thread finishes, so
# the probe retries it in place a few times before treating the import as failed for this call.
_DeadlockError = getattr(getattr(importlib, "_bootstrap", None), "_DeadlockError", RuntimeError)
_SDK_DEADLOCK_RETRIES, _SDK_DEADLOCK_RETRY_DELAY = 5, 0.02
# Optional SDK type families (module, names, debug message when absent), bound in this order to
# _MCP_SAMPLING_TYPES / _MCP_ELICITATION_TYPES / _MCP_NOTIFICATION_TYPES; an older SDK only
# loses that feature, not MCP.
_OPTIONAL_TYPE_FAMILIES = (
("mcp.types", ("CreateMessageResult", "CreateMessageResultWithTools", "ErrorData", "SamplingCapability",
"SamplingToolsCapability", "TextContent", "ToolUseContent"),
"MCP sampling types not available -- sampling disabled"),
("mcp.types", ("ElicitRequestParams", "ElicitResult"),
"MCP elicitation types not available -- elicitation disabled"),
("mcp.types", ("ServerNotification", "ToolListChangedNotification", "PromptListChangedNotification",
"ResourceListChangedNotification"),
"MCP notification types not available -- dynamic tool discovery disabled"),
)
# Bound by _ensure_mcp_sdk(); module __getattr__ (PEP 562) imports the SDK on first external
# access so mock.patch("tools.mcp_tool.stdio_client") sees a real original, never clobbered.
_MCP_SDK_LAZY_SYMBOLS = frozenset(
{"StdioServerParameters", "stdio_client", "streamablehttp_client", "streamable_http_client"}
| {n for _mod, names, _msg in _OPTIONAL_TYPE_FAMILIES for n in names})
def __getattr__(name: str):
if name in _MCP_SDK_LAZY_SYMBOLS:
_ensure_mcp_sdk()
try:
return globals()[name]
except KeyError:
pass # SDK missing or symbol absent on this SDK build
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
def _load_sdk_names(module: str, names: tuple) -> tuple[Optional[dict], Optional[BaseException], bool]:
"""``(values, None, _)`` for ``names`` from SDK ``module``, else ``(None, error, definitive)``.
``definitive`` means this SDK build lacks the module or name (safe to remember); anything
else -- a dependency missing mid-update, a partially initialized module -- is retried by the
next probe. Nothing is published: the probe binds every value at once after its flags, so a
concurrent caller never observes a half-probed SDK. A racing import's ``_DeadlockError`` is
retried briefly in place, since it clears as soon as the other thread finishes the module."""
for attempt in range(_SDK_DEADLOCK_RETRIES + 1):
partial = None # stays None when the import itself raised
try:
mod = importlib.import_module(module)
# Read before the lookups: a module still initializing may not have bound the name yet,
# and the other thread can finish it before the failure is classified.
partial = bool(getattr(getattr(mod, "__spec__", None), "_initializing", False))
return {n: getattr(mod, n) for n in names}, None, True
except _DeadlockError as exc:
if attempt == _SDK_DEADLOCK_RETRIES:
return None, exc, False
time.sleep(_SDK_DEADLOCK_RETRY_DELAY * (attempt + 1))
except ModuleNotFoundError as exc:
missing = exc.name
return None, exc, missing is not None and (module == missing or module.startswith(missing + "."))
except AttributeError as exc: # definitive only for a name missing from a finished module
return None, exc, partial is False
except ImportError as exc: # partially initialized module / broken dependency: transient
return None, exc, False
return None, None, False # pragma: no cover — the loop always returns
def _describe_import_error(exc: Optional[BaseException]) -> Optional[str]:
return f"{type(exc).__name__}: {exc}" if exc is not None else None
def _ensure_mcp_sdk() -> bool:
"""Import the optional ``mcp`` SDK on first use; return availability. Thread-safe: callers
block on the lock until the holder has published every flag. A transient import failure is
reported for that call and retried on the next one; only a definitive outcome is remembered.
Honors a test-patched ``_MCP_AVAILABLE=False`` (no import) and pre-installed mocks (a
``ClientSession`` the probe did not bind means no re-import)."""
global _MCP_SDK_IMPORT_ATTEMPTED, _MCP_AVAILABLE, _MCP_HTTP_AVAILABLE, _MCP_NEW_HTTP, _MCP_LEGACY_HTTP
global _MCP_SAMPLING_TYPES, _MCP_NOTIFICATION_TYPES, _MCP_ELICITATION_TYPES, sse_client
global _MCP_MESSAGE_HANDLER_SUPPORTED, _MCP_LOGGING_CALLBACK_SUPPORTED, LATEST_HANDSHAKE_VERSION
global _JSONRPC_METHOD_NOT_FOUND, _MCP_SDK_PROBED_SESSION, _MCP_HTTP_IMPORT_ERROR, _MCP_SDK_IMPORT_ERROR
if not _MCP_AVAILABLE:
return False
if _MCP_SDK_IMPORT_ATTEMPTED or (ClientSession is not None and ClientSession is not _MCP_SDK_PROBED_SESSION):
return _MCP_AVAILABLE
with _MCP_SDK_IMPORT_LOCK:
if _MCP_SDK_IMPORT_ATTEMPTED or (ClientSession is not None and ClientSession is not _MCP_SDK_PROBED_SESSION):
return _MCP_AVAILABLE
# A re-probe (an earlier pass was unsettled) only ever upgrades: a value a previous pass
# bound stays bound even if this pass's import of it fails transiently.
reprobe = ClientSession is not None
settled = True
bound: dict = {}
errors: dict = {}
def load(module: str, names: tuple, missing_msg: Optional[str] = None) -> bool:
nonlocal settled
values, exc, definitive = _load_sdk_names(module, names)
if values is not None:
bound.update(values)
return True
errors.setdefault(module, exc)
if not definitive:
settled = False
logger.debug("MCP SDK import of %s failed transiently (%s) -- will retry",
module, _describe_import_error(exc))
elif missing_msg:
logger.debug(missing_msg)
return False
if not (load("mcp", ("ClientSession", "StdioServerParameters"))
and load("mcp.client.stdio", ("stdio_client",))):
if reprobe:
return True # an earlier pass already bound a working core SDK
_MCP_SDK_IMPORT_ERROR = _describe_import_error(errors.get("mcp") or errors.get("mcp.client.stdio"))
if settled:
logger.debug("mcp package not installed -- MCP tool support disabled")
_MCP_AVAILABLE = False
_MCP_SDK_IMPORT_ATTEMPTED = True
else:
logger.warning("MCP SDK import failed (%s) -- MCP servers are unavailable until it "
"imports; retrying on the next connect", _MCP_SDK_IMPORT_ERROR)
return False
# mcp >= 1.24 ships streamable_http_client; 2.0 dropped the deprecated
# streamablehttp_client alias. Either one gives HTTP.
new_http = load("mcp.client.streamable_http", ("streamable_http_client",)) or (reprobe and _MCP_NEW_HTTP)
legacy_http = (load("mcp.client.streamable_http", ("streamablehttp_client",))
or (reprobe and _MCP_LEGACY_HTTP))
http_error = None
if not (new_http or legacy_http):
http_error = (_describe_import_error(errors.get("mcp.client.streamable_http"))
or "the SDK exports no streamable HTTP client")
load("mcp.types", ("LATEST_PROTOCOL_VERSION",),
"mcp.types.LATEST_PROTOCOL_VERSION not available -- using fallback protocol version")
has_handshake = load("mcp.client.session", ("LATEST_HANDSHAKE_VERSION",))
has_sse = load("mcp.client.sse", ("sse_client",),
"mcp.client.sse.sse_client not available -- SSE transport disabled")
type_families = [load(*family) or (reprobe and bool(previous)) for family, previous in
zip(_OPTIONAL_TYPE_FAMILIES,
(_MCP_SAMPLING_TYPES, _MCP_ELICITATION_TYPES, _MCP_NOTIFICATION_TYPES))]
values, _exc, _definitive = _load_sdk_names("mcp.types", ("METHOD_NOT_FOUND",))
# Publish: flags first, then symbols, ClientSession last (it gates the mock fast path).
_MCP_NEW_HTTP, _MCP_LEGACY_HTTP = new_http, legacy_http
_MCP_HTTP_AVAILABLE, _MCP_HTTP_IMPORT_ERROR = new_http or legacy_http, http_error
_MCP_SAMPLING_TYPES, _MCP_ELICITATION_TYPES, _MCP_NOTIFICATION_TYPES = type_families
_MCP_SDK_IMPORT_ERROR = None
if values is not None:
_JSONRPC_METHOD_NOT_FOUND = values["METHOD_NOT_FOUND"]
session = bound.pop("ClientSession")
globals().update(bound)
if not has_handshake and not reprobe:
# Pre-2.x SDKs: newest revision IS the handshake revision.
LATEST_HANDSHAKE_VERSION = LATEST_PROTOCOL_VERSION
if not has_sse and not reprobe:
sse_client = None
_MCP_SDK_PROBED_SESSION = session
globals()["ClientSession"] = session
_MCP_MESSAGE_HANDLER_SUPPORTED = _client_session_accepts("message_handler")
if not _MCP_MESSAGE_HANDLER_SUPPORTED:
logger.debug("MCP SDK does not support message_handler -- dynamic tool discovery disabled")
_MCP_LOGGING_CALLBACK_SUPPORTED = _client_session_accepts("logging_callback")
_MCP_SDK_IMPORT_ATTEMPTED = settled
return True
_SDK_HTTPX_MOD = None
def sdk_httpx():
"""The httpx module the *installed* MCP SDK is built against (mcp 2.0 moved to ``httpx2``).
Every object crossing the SDK boundary (AsyncClient, OAuth Request, exception classes) must
come from the module the SDK itself imports or it fails at the transport layer. Resolved
from the SDK's transport module, else the newest present; ``None`` if neither imports."""
global _SDK_HTTPX_MOD
if _SDK_HTTPX_MOD is not None:
return _SDK_HTTPX_MOD
try:
_transport = importlib.import_module("mcp.client.streamable_http")
found = getattr(_transport, "httpx2", None) or getattr(_transport, "httpx", None)
except (ImportError, _DeadlockError): # absent transport, or a racing import (a RuntimeError)
found = None
if found is not None:
_SDK_HTTPX_MOD = found # only the SDK's own answer is cached
return found
# Fallback: not cached, so the SDK's own module wins once its transport imports.
for fallback in ("httpx2", "httpx"):
try:
return importlib.import_module(fallback)
except ImportError:
pass
return None
def _client_session_accepts(kwarg: str) -> bool:
"""Whether this SDK's ``ClientSession.__init__`` takes ``kwarg`` (older SDKs lack
``message_handler`` and ``logging_callback``)."""
if not _MCP_AVAILABLE:
return False
try:
return kwarg in inspect.signature(ClientSession).parameters
except (TypeError, ValueError):
return False
# MCP logging levels (RFC 5424 syslog severities) -> Python logging levels.
# Port of anomalyco/opencode#34529's serverLog mapping.
_MCP_LOG_LEVEL_MAP = {
"debug": logging.DEBUG, "info": logging.INFO, "notice": logging.INFO,
"warning": logging.WARNING, "error": logging.ERROR, "critical": logging.ERROR,
"alert": logging.ERROR, "emergency": logging.ERROR}
# ---- Reconnect / keepalive tuning ----
_DEFAULT_CONNECT_TIMEOUT = 60 # seconds for initial connection per server
_MAX_RECONNECT_RETRIES = 5
_MAX_INITIAL_CONNECT_RETRIES = 3 # retries for the very first connection attempt
_MAX_BACKOFF_SECONDS = 60
_RECYCLED_RECONNECT_TIMEOUT = 15.0
# Parked servers (tools deregistered) self-probe on this cadence: nothing else can revive them.
_PARKED_RETRY_INTERVAL = 300
# Cap for the parked self-probe backoff: each consecutive still-failed probe doubles the wait from
# _PARKED_RETRY_INTERVAL up to this ceiling, so a server whose failure is provably persistent (a
# duplicate stdio instance holding a single-writer lock) stops re-spawning a doomed child forever.
_PARKED_PROBE_BACKOFF_MAX = 3600
# Bounded wait for a respawned stdio child when a call finds it dead (gateway restarts kill
# every MCP child); bounded so a broken server still parks via run()'s rapid-drop budget.
_STDIO_RESPAWN_WAIT_SEC = 15.0
# Remote clients MUST ping faster than the server's idle-session TTL (short-TTL servers need a
# smaller configured ``keepalive_interval``); stdio only opts in explicitly because local pipes
# have no remote session TTL. The floor stops a tiny interval busy-looping.
_DEFAULT_KEEPALIVE_INTERVAL, _MIN_KEEPALIVE_INTERVAL = 180, 5
# One bounded cancellation cycle at final shutdown so resistant tasks cannot hang exit.
_MCP_LOOP_DRAIN_TIMEOUT = 3.0
# JSON-RPC 2.0 "method not found" (server without optional ``ping``); _ensure_mcp_sdk()
# overrides it from mcp.types once loaded.
_JSONRPC_METHOD_NOT_FOUND = -32601
# nextCursor pagination cap so a forever-cursor cannot spin discovery (50 pages = thousands).
_MCP_LIST_MAX_PAGES = 50
async def _paginate_full_list(list_method, items_attr: str, server_name: str,
cache_meta_out: Optional[dict] = None):
"""Drain a paginated ``list_*`` call by following ``nextCursor``; ``cache_meta_out`` gets the
first page's SEP-2549 hints. Callers must hold the server's ``_rpc_lock``."""
items: list = []
cursor = None
for _ in range(_MCP_LIST_MAX_PAGES):
if not cursor:
result = await list_method()
else:
# mcp 2.0 takes params=PaginatedRequestParams, 1.x takes cursor=.
# Inspect before awaiting: an internal TypeError is not a signature mismatch.
import inspect
try:
signature = inspect.signature(list_method)
except (TypeError, ValueError):
accepts_params = True # Opaque callables use the current SDK convention.
else:
accepts_params = any(
p.kind == inspect.Parameter.VAR_KEYWORD
or (p.name == "params" and p.kind != inspect.Parameter.POSITIONAL_ONLY)
for p in signature.parameters.values()
)
if accepts_params:
import mcp.types as _types # late: keeps the SDK import lazy
_params_cls = getattr(_types, "PaginatedRequestParams", None)
if _params_cls is not None:
result = await list_method(params=_params_cls(cursor=cursor))
else:
result = await list_method(cursor=cursor)
else:
result = await list_method(cursor=cursor)
if cache_meta_out is not None and not items:
for key, snake, camel in (("ttl_ms", "ttl_ms", "ttlMs"), ("cache_scope", "cache_scope", "cacheScope")):
hint = mcp_field(result, snake, camel)
if hint is not None:
cache_meta_out[key] = hint
items.extend(getattr(result, items_attr, None) or [])
cursor = mcp_field(result, "next_cursor", "nextCursor")
# Cursor is an opaque string; anything else (incl. mocks) = last page.
if not isinstance(cursor, str) or not cursor:
break
else:
logger.warning("MCP server '%s': %s pagination exceeded %d pages; truncating at %d items",
server_name, items_attr, _MCP_LIST_MAX_PAGES, len(items))
return items
# ---- Server task -- each MCP server lives in one long-lived asyncio Task ----
class MCPServerTask(MCPServerRunMixin, MCPServerTransportMixin, MCPServerHealthMixin):
"""One MCP server connection in one long-lived asyncio Task (the transport's anyio cancel
scopes must enter/exit in the same Task). Run state machine, transport bring-up and
keepalive/liveness live in the three mixins."""
def __init__(self, name: str):
self.name = name
self.session: Optional[Any] = None
self.tool_timeout: float = _DEFAULT_TOOL_TIMEOUT
self._task: Optional[asyncio.Task] = None
self._ready = asyncio.Event()
self._shutdown_event = asyncio.Event()
# Set -> _run_http/_run_stdio exit cleanly and run() re-enters the transport.
self._reconnect_event = asyncio.Event()
self._tools: list = []
self._registered_tool_names: list[str] = []
self._config: dict = {}
self._error: Optional[Exception] = None
self._sampling: Optional[SamplingHandler] = None
self._elicitation: Optional[ElicitationHandler] = None
self._reconnect_retries: int = 0
# Rapid-drop budget: a session is UNPROVEN until it survives a keepalive interval or a
# successful call; only a proven session clears the budget, so a post-handshake flapper
# still parks.
# Rapid-drop budget (#62212): a freshly (re)established session is UNPROVEN until it demonstrates
# real health — it survived at least one full keepalive interval (keepalive success path) or served
# at least one successful tool call. Only a proven session clears the reconnect budget; a transport
# that flaps right after the handshake keeps getting charged and still reaches the park instead of
# hot-cycling respawns forever.
self._session_proven: bool = False
# Never cleared (unlike _ready): separates first-connect from reconnect failures.
self._ever_connected: bool = False
# Latched when the Streamable HTTP -> SSE fallback connects: reconnects reuse SSE directly.
self._sse_fallback: bool = False
# Status/URL/body of the last HTTP rejection the Streamable HTTP client saw; names the real
# cause when the SDK reports only ``Server returned an error response``.
self._http_rejection: dict = {}
# True from park until proven healthy again; logs the revival once.
self._was_parked: bool = False
# Why the server is parked (the revival_reason handed to _park), None once healthy again.
# Lets cron preflight tell a network-blip park (recovering) from a permanent-error park
# (revoked credentials, dead endpoint) that must not run the job tool-less forever.
self._park_reason: Optional[str] = None
self._last_park_line: Optional[str] = None # last park WARNING text; identical re-parks log at DEBUG
# (token storage, credential stamp) while parked on an OAuth failure; see _arm_auth_park.
self._auth_park: Optional[tuple] = None
# Digest of the resolved inputs the transport last connected with; a cross-profile adopter
# must resolve the same digest in its own scope. None until the transport publishes it.
self._resolved_identity: Optional[str] = None
# True when this stdio server must have exactly ONE live child system-wide (a single-writer
# datastore). A non-owner that finds the resource taken parks quietly instead of self-probing.
self._single_instance: bool = False
# Consecutive park revivals that came back still-broken; drives the parked-probe backoff.
self._parked_probe_failures: int = 0
# True while parked with no timed self-probe (single_instance conflict).
self._park_quiet: bool = False
# In-flight RPC tasks so a deliberate teardown fails them fast; _reconnecting is True
# during that teardown so _track_inflight_rpc turns the cancel into a retryable error.
# In-flight RPC bookkeeping (#48069 salvage): user-visible requests registered while running so a
# reconnect/shutdown teardown can fail them fast instead of orphaning them on a dying transport.
self._inflight_tasks: set = set()
self._reconnecting: bool = False
# Latched by races (teardown-vs-keepalive, auth-lock corruption).
# See #77765, #81051, #84132.
self._suspect_reason: Optional[str] = None
# Teardown that failed in-flight calls => next reconnect is RACE RECOVERY, not a
# budget charge.
self._teardown_race: bool = False
# One-time grace: auth/permanent failure on a PROVEN session gets one suspect+reconnect
# cycle before parking.
self._permanent_grace_used: bool = False
# Children of the current stdio transport: in-flight calls fail FAST when one dies.
# PIDs of the stdio subprocess spawned for the current transport (captured in _run_stdio). Used to
# fail in-flight calls FAST when the child dies instead of waiting out the full tool timeout
# (#81995).
self._stdio_child_pids: set[int] = set()
self._auth_type: str = ""
self._refresh_lock = asyncio.Lock()
# A stdio session is one JSON-RPC stream (a concurrent list_tools can wedge a tool
# call): serialize client-initiated RPCs per server (HTTP too, for ordering).
self._rpc_lock = asyncio.Lock()
self._pending_refresh_tasks: set[asyncio.Task] = set()
# contextvars snapshot inside session.call_tool(): the SDK runs elicitation/create on a
# task that does not inherit HERMES_SESSION_PLATFORM, so the callback replays this.
self._pending_call_context: Optional[contextvars.Context] = None
# Same-call threading.local prompt callbacks (approval/sudo/vault) for the elicitation
# consent hop — asyncio.to_thread carries neither contextvars nor thread locals.
self._pending_call_callbacks: tuple = ()
self._lifecycle_started_at = self._last_tool_call_at = time.monotonic()
self._idle_timeout_seconds = self._max_lifetime_seconds = self._recycled_reason = None
# Handshake InitializeResult: the server's REAL advertised capabilities.
# Captures the ``InitializeResult`` returned by ``await session.initialize()`` so downstream code
# can inspect the server's real advertised capabilities (``.capabilities.resources``,
# ``.capabilities.prompts``) instead of assuming every ``ClientSession`` method attribute
# corresponds to a supported server method. See #18051.
self.initialize_result: Optional[Any] = None
# SEP-2549 cache hints from the last tools/list (ttl_ms, cache_scope).
self._list_cache_meta: dict = {}
# Latched when ``ping`` returns -32601; keepalives then use list_tools. Reset per connect.
self._ping_unsupported: bool = False
# Content types a real Streamable-HTTP endpoint may return on the initial POST/GET;
# anything else on a 2xx means the URL is not an MCP endpoint.
_MCP_CONTENT_TYPES = ("application/json", "text/event-stream")
# ---- Module-level state (every mutation under ``_lock``) ----
#
# Every ledger below is keyed by the CONNECTION KEY from ``tools.mcp_tool_scope``: the bare
# server name outside a multiplexer, ``(owner_scope, name)`` under one. Two profiles naming the
# same server with their own credentials are two connections; a name-keyed ledger let the first
# profile's connection shadow the second's (never connected, silently tool-less — #106005).
_servers: dict[Any, MCPServerTask] = {}
# Profile registry scope per live connection (None outside multiplex) so a multiplexed
# /reload-mcp tears down only its own profile's servers.
_server_scope_keys: dict[Any, Optional[str]] = {}
# Registry scopes that have adopted a live server connection. The owning scope above remains
# authoritative for connection teardown; this set preserves visibility for shared connections.
_server_tool_scopes: dict[Any, set] = {}
_server_connecting: set = set()
_server_connect_errors: dict[Any, str] = {}
# adopter scope -> server names whose shared connection an owner's scoped shutdown tore down;
# drained by the next discovery pass so the adopter is re-registered (see mcp_tool_lifecycle).
_orphaned_adopters: dict[str, set] = {}
# Lazy startup: servers registered from the schema cache without connecting; popped on
# first real connection.
# Keyed by connection key; entries are popped once a real connection is established on first use. See #56832.
_lazy_server_configs: dict[Any, dict] = {}
_lazy_server_fingerprints: dict[Any, str] = {}
_lazy_server_tool_names: dict[Any, list[str]] = {}
# Task-local claim around ``_connect_server``: discovery retains a recoverable parked task
# while standalone probes never publish failed servers into module-global ownership.
_connect_server_claim: contextvars.ContextVar[Optional[Callable[[MCPServerTask], None]]] = (
contextvars.ContextVar("mcp_connect_server_claim", default=None))
# Per-server connect cooldown: a server that fails to spawn never reaches ``_servers``, so
# without it every ``discover_mcp_tools()`` (one per worker session) would respawn it — a
# restart storm whose unreaped children destabilise healthy servers. Exponential-backoff
# deadline honoured by ``register_mcp_servers``; cleared on success.
# Connection-retry cooldown (per-server isolation against restart storms). A single stdio MCP server that
# fails to spawn (bad PATH, ``exec: not found``, crash-on-start) is never recorded in ``_servers`` --
# ``start()`` raises and ``_discover_and_register_server`` aborts before the ``_servers[name] = server``
# line. Without a cooldown, EVERY subsequent ``discover_mcp_tools()`` (one per agent worker session, i.e.
# every few seconds) sees the server as "not connected" and re-spawns it from scratch. That is the restart
# storm in #50394: the failing server is re-attempted on the shared MCP event loop on every worker session,
# the subprocesses pile up unreaped, and the churn destabilises the healthy co-located servers (their tools
# intermittently surface as "Unknown tool"). Fix: after a failed connection attempt, stamp a monotonic
# ``retry_after`` deadline with exponential backoff. ``register_mcp_servers`` skips a server whose cooldown
# has not elapsed, so a chronically failing server is retried on a backoff schedule instead of on every
# worker session -- isolating it from the rest of the bridge. A successful connection clears the state.
_server_connect_retry_after: dict[Any, float] = {} # connection key -> monotonic deadline
_server_connect_failures: dict[Any, int] = {} # connection key -> consecutive failures
_CONNECT_RETRY_BASE_BACKOFF_SEC, _CONNECT_RETRY_MAX_BACKOFF_SEC = 30.0, 600.0
# Per-server circuit breaker: closed -> open (calls short-circuit until the cooldown) ->
# half-open (next call probes). Mutate only via _bump_server_error / _reset_server_error.
# After _CIRCUIT_BREAKER_THRESHOLD consecutive failures, the handler returns a "server unreachable" message
# that tells the model to stop retrying, preventing the 90-iteration burn loop described in #10447. State
# machine: closed — error count below threshold; all calls go through. open — threshold reached;
# calls short-circuit until the cooldown elapses. half-open — cooldown elapsed; the next call is a probe
# that actually hits the session. Probe success → closed. Probe failure → reopens (cooldown re-armed).
# ``_server_breaker_opened_at`` records the monotonic timestamp when the breaker most recently transitioned
# into the open state. Use the ``_bump_server_error`` / ``_reset_server_error`` helpers to mutate this state
# — they keep the count and timestamp in sync.
_server_error_counts: dict[Any, int] = {}
_server_breaker_opened_at: dict[Any, float] = {}
# True while every strike in the current streak was the tool's own error payload (server reachable,
# call rejected); picks the open-breaker wording, since "unreachable" was false for that case (#11113).
_server_errors_all_application: dict[Any, bool] = {}
_CIRCUIT_BREAKER_THRESHOLD, _CIRCUIT_BREAKER_COOLDOWN_SEC = 3, 60.0
# Trust-tier gating (``trust: full | untrusted``): on an untrusted server every write-capable
# call (discovery-time ``readOnlyHint`` not exactly True; malformed fails closed) needs approval
# before the RPC fires. A lying readOnlyHint can only skip approval for calls the operator was
# already warned about, never widen access. Missing trust = full; unrecognized = untrusted (a
# typo must never disable the gate). Classified at CALL time from DISCOVERY data: no schema
# mutation, prompt cache intact. ``_server_trust_levels`` is keyed by the CONSUMING profile's own
# key (its policy for the name, even when it adopted another profile's connection);
# ``_tool_read_only_hints`` by the connection key (the server's own tool annotations).
_server_trust_levels: dict[Any, str] = {}
_tool_read_only_hints: dict[Any, dict[str, bool]] = {}
# Declared per-tool costs + per-chat spend limit (``tools.mcp_tool_costs.CostPolicy``), keyed like
# ``_server_trust_levels``: the consuming profile's policy. Absent = no costs, behaviour unchanged.
_server_cost_policies: dict[Any, Any] = {}
_TRUST_FULL, _TRUST_UNTRUSTED = "full", "untrusted"
def _bump_server_error(server_name: str, *, application: bool = False) -> None:
"""Count a failure; at the threshold (re)stamp the breaker-open time. Keyed by the calling
scope's connection so one profile's failing server never opens another profile's breaker.
*application*: the call completed and the payload was an error (transport is fine)."""
from tools.mcp_tool_scope import _resolve_server_key
key = _resolve_server_key(server_name)
n = _server_error_counts.get(key, 0) + 1
_server_error_counts[key] = n
_server_errors_all_application[key] = application and (n == 1 or _server_errors_all_application.get(key, False))
if n >= _CIRCUIT_BREAKER_THRESHOLD:
_server_breaker_opened_at[key] = time.monotonic()
def _reset_server_error(server_name: str) -> None:
"""Close the breaker on any unambiguous success signal."""
from tools.mcp_tool_scope import _resolve_server_key
key = _resolve_server_key(server_name)
_server_error_counts[key] = 0
_server_breaker_opened_at.pop(key, None)
_server_errors_all_application.pop(key, None)
# Servers opted into parallel tool calls, keyed by the consuming profile's own key (``foo-bar``/
# ``foo_bar`` sanitize alike but must not share policy; neither do two profiles' same-named servers).
_parallel_safe_servers: set = set()
# registry tool name -> raw server name (the generated name is lossy; never re-parse it).
_mcp_tool_server_names: dict[str, str] = {}
# Dedicated event loop in a background daemon thread; _lock guards the loop handles, _servers,
# the status maps and the PID ledgers.
_mcp_loop: Optional[asyncio.AbstractEventLoop] = None
_mcp_thread: Optional[threading.Thread] = None
_lock = threading.Lock()
def _mcp_registry_scope() -> Optional[str]:
"""Registry scope for MCP registrations: the session's own tool scope when one is bound, else a
profile overlay when this process serves profiles, else None. Under ``gateway.multiplex_profiles``
every turn runs scoped; a process that serves a routed profile through the HERMES_HOME override
(dashboard/desktop backend, per-profile cron ticker) is a multiplexer too, even with the flag
off — keying its connections by the bare name would hand one profile's credentialed connection
to every other served profile (#111151). Single-profile processes (no override, or an override
naming their own home) keep bare names."""
from tools.registry import current_session_tool_scope
session_scope = current_session_tool_scope()
if session_scope is not None:
return session_scope
from agent.secret_scope import serves_routed_profile
if not serves_routed_profile():
return None
from tools.registry import registry
return registry.current_scope_key()
def _server_registry_scope(key) -> Optional[str]:
"""Scope owning the connection under *key*'s tools: the one captured at adoption (teardown
runs on the MCP loop without the discovering profile's context), else the current one."""
if key in _server_scope_keys:
return _server_scope_keys[key]
from tools.mcp_tool_scope import _key_scope
return _key_scope(key) or _mcp_registry_scope()
def _server_visible_in_scope(key, scope: Optional[str]) -> bool:
"""Whether the live connection under *key* is visible from ``scope`` without changing its
teardown owner."""
if scope is None:
return True
return (_server_scope_keys.get(key) == scope
or scope in _server_tool_scopes.get(key, ()))
# Cross-process discovery guard: advisory file lock so gateway + CLI + TUI don't all discover.
# See issue #62771.
_LOCK_UNAVAILABLE: Any = object() # sentinel: locking broken/unavailable
_MCP_DISCOVERY_LOCK_PATH: Optional[str] = None # resolved lazily
# A discovery pass (bounded gathers, one 120 s budget per wave) may legitimately
# run past the 120 s a scatter completes in. The waiter must outlast the worst
# legitimate holder (pass ceiling + slack), or it fails over at 120 s, discovers
# unguarded beside a still-connecting holder, and spawns duplicate stdio trees.
# See #117373: the concurrency cap made the old 120 s waiter budget stale.
_MCP_DISCOVERY_PASS_MAX_SEC = 300 # overall pass ceiling
_MCP_DISCOVERY_LOCK_RETRY_DELAY_S = 0.5
# Waiter budget (max_retries * delay) must outlast the pass ceiling: 320 s > 300 s.
_MCP_DISCOVERY_LOCK_MAX_RETRIES = int(
_MCP_DISCOVERY_PASS_MAX_SEC / _MCP_DISCOVERY_LOCK_RETRY_DELAY_S) + 20