mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-20 18:50:12 +03:00
d40d087170
_run_on_mcp_loop wrapped every scheduled coroutine in _wrap_with_home_override and _wrap_with_dashboard_oauth_flow, which re-set the caller's HERMES_HOME override and dashboard OAuth flow inside the task. The comment justifying them said run_coroutine_threadsafe copies the loop thread's context. It copies the calling thread's: it schedules through call_soon_threadsafe, whose Handle captures copy_context() on the caller, and the Task is created inside that copy. Both values are ContextVars (hermes_scope._STATE and mcp_dashboard_oauth._current_dashboard_flow), so they already arrive. A probe that calls _run_on_mcp_loop with no wrappers, from three threads each holding its own override and flow, in coroutine and factory form, and reads both from a child task, saw every caller's values on Python 3.11.11 and 3.14.7. All six call sites schedule through _run_on_mcp_loop. The test that only checked the wrapper's no-op path goes; a new test asserts the dashboard flow reaches the loop. Running the file with scheduling forced into an empty Context fails all three propagation tests.
268 lines
12 KiB
Python
268 lines
12 KiB
Python
"""Background-loop plumbing for tools.mcp_tool: cross-process discovery file lock, scheduling
|
|
coroutines onto the MCP loop from caller threads and the loop's exception handler. Origin state
|
|
(``_lock``, ``_mcp_loop``) is read through ``_core`` so ``mock.patch("tools.mcp_tool.X")`` keeps
|
|
working."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import concurrent.futures
|
|
import contextlib
|
|
import errno
|
|
import logging
|
|
import os
|
|
import threading
|
|
import time
|
|
from typing import Any, Optional
|
|
from tools.mcp_tool_common import _core
|
|
from tools import mcp_tool_lifecycle as _lifecycle
|
|
|
|
logger = logging.getLogger("tools.mcp_tool")
|
|
|
|
|
|
class _LockCookie:
|
|
"""Holds a cross-process file lock; ``release()`` drops it. The file object MUST stay open while
|
|
held: both fcntl and portalocker locks are tied to the descriptor."""
|
|
|
|
def __init__(self, fh: Any) -> None:
|
|
self._fh = fh
|
|
|
|
def release(self) -> None:
|
|
if self._fh is None:
|
|
return
|
|
# Best effort: an unlock/close failure must never propagate out of discovery.
|
|
with contextlib.suppress(Exception):
|
|
if os.name == "posix":
|
|
import fcntl
|
|
fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN)
|
|
else:
|
|
import portalocker
|
|
portalocker.unlock(self._fh)
|
|
with contextlib.suppress(Exception):
|
|
self._fh.close()
|
|
self._fh = None
|
|
|
|
|
|
def _acquire_lock_on_fh(fh: Any) -> bool:
|
|
"""Non-blocking exclusive lock (fcntl on POSIX, portalocker elsewhere). False when another process
|
|
holds it; unexpected errors propagate so the caller can treat locking as unavailable."""
|
|
if os.name == "posix":
|
|
import fcntl
|
|
try:
|
|
fcntl.flock(fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError as e:
|
|
if e.errno in (errno.EACCES, errno.EAGAIN, errno.EWOULDBLOCK):
|
|
return False
|
|
raise
|
|
return True
|
|
import portalocker
|
|
try:
|
|
portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB)
|
|
except portalocker.LockException:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _try_acquire_mcp_discovery_lock() -> Any:
|
|
"""``_LockCookie`` (acquired), ``None`` (held by another process) or ``_LOCK_UNAVAILABLE``
|
|
(locking broken: run discovery unguarded)."""
|
|
# The cached path lives on the ORIGIN module (tests reset ``tools.mcp_tool._MCP_DISCOVERY_LOCK_PATH``).
|
|
# A routed profile (multiplexed gateway) locks under ITS home: the launch profile's lock file
|
|
# would serialize discovery across profiles and never coordinate with B's own single-profile processes.
|
|
from tools import mcp_tool as _origin
|
|
try:
|
|
from hermes_constants import get_hermes_home, get_hermes_home_override
|
|
if get_hermes_home_override() is not None:
|
|
lock_path = str(get_hermes_home() / ".mcp-discovery.lock")
|
|
else:
|
|
if _origin._MCP_DISCOVERY_LOCK_PATH is None:
|
|
_origin._MCP_DISCOVERY_LOCK_PATH = str(get_hermes_home() / ".mcp-discovery.lock")
|
|
lock_path = _origin._MCP_DISCOVERY_LOCK_PATH
|
|
fh = open(lock_path, "w", encoding="utf-8")
|
|
except Exception:
|
|
return _core._LOCK_UNAVAILABLE
|
|
try:
|
|
acquired = _acquire_lock_on_fh(fh)
|
|
except Exception:
|
|
fh.close()
|
|
return _core._LOCK_UNAVAILABLE
|
|
if acquired:
|
|
return _LockCookie(fh)
|
|
fh.close()
|
|
return None
|
|
|
|
|
|
def _mcp_loop_exception_handler(loop, context):
|
|
"""Suppress the benign 'Event loop is closed' RuntimeError httpx finalizers raise against the
|
|
dead loop during shutdown; forward the rest."""
|
|
exc = context.get("exception")
|
|
if not (isinstance(exc, RuntimeError) and "Event loop is closed" in str(exc)):
|
|
loop.default_exception_handler(context)
|
|
|
|
|
|
def _running_loop() -> Optional[asyncio.AbstractEventLoop]:
|
|
"""The MCP loop when it is up, else None (read under ``_lock``)."""
|
|
with _core._lock:
|
|
loop = _core._mcp_loop
|
|
return loop if loop is not None and loop.is_running() else None
|
|
|
|
|
|
def _run_on_mcp_loop(coro_or_factory, timeout: float = 30):
|
|
"""Schedule a coroutine (or zero-arg factory — avoids leaking a never-awaited coroutine when the
|
|
loop is down) on the MCP loop and block until done, polling so user interrupts are honored."""
|
|
from tools.interrupt import is_interrupted
|
|
from agent.async_utils import safe_schedule_threadsafe
|
|
|
|
loop = _running_loop()
|
|
if loop is None:
|
|
if asyncio.iscoroutine(coro_or_factory):
|
|
coro_or_factory.close()
|
|
raise RuntimeError("MCP event loop is not running")
|
|
coro = coro_or_factory() if callable(coro_or_factory) else coro_or_factory
|
|
future = safe_schedule_threadsafe(coro, loop, logger=logger, log_message="MCP scheduling failed")
|
|
if future is None:
|
|
raise RuntimeError("MCP event loop unavailable (failed to schedule)")
|
|
start_time = time.monotonic()
|
|
deadline = None if timeout is None else start_time + timeout
|
|
while True:
|
|
if is_interrupted():
|
|
future.cancel()
|
|
raise InterruptedError("User sent a new message")
|
|
remaining = 0.1 if deadline is None else deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
future.cancel()
|
|
raise TimeoutError(f"MCP call timed out after {time.monotonic() - start_time:.1f}s "
|
|
f"(configured timeout: {float(timeout):.1f}s)")
|
|
try:
|
|
return future.result(timeout=min(0.1, remaining))
|
|
except concurrent.futures.TimeoutError:
|
|
# Aliases builtin TimeoutError, so it also fires for the coroutine's own timeout: a done
|
|
# future must yield its outcome.
|
|
if future.done():
|
|
return future.result()
|
|
|
|
|
|
def _signal_reconnect(server: Any) -> bool:
|
|
"""Ask a server task to rebuild its transport, thread-safely: the event lives on the MCP loop,
|
|
so set via ``call_soon_threadsafe`` when it runs (direct ``.set()`` otherwise). False when the
|
|
server has no reconnect machinery."""
|
|
event = getattr(server, "_reconnect_event", None)
|
|
if event is None:
|
|
return False
|
|
loop = _core._mcp_loop
|
|
if isinstance(event, asyncio.Event) and loop is not None and loop.is_running():
|
|
loop.call_soon_threadsafe(event.set)
|
|
else:
|
|
event.set()
|
|
return True
|
|
|
|
|
|
def reconnect_mcp_server(server_name: str) -> bool:
|
|
"""Ask a currently-live MCP server to rebuild after external re-auth."""
|
|
from tools.mcp_tool_scope import _resolve_server_key
|
|
with _core._lock:
|
|
server = _core._servers.get(_resolve_server_key(server_name))
|
|
return server is not None and _signal_reconnect(server)
|
|
|
|
|
|
def _wait_for_server_session_ready(srv: Any, *, old_session: Any = None, timeout: float = 15.0) -> bool:
|
|
"""Poll until the server exposes a usable, ready session (during a reconnect ``srv.session`` is
|
|
briefly None or stale; retrying blindly burns breaker strikes). With ``old_session`` the observed
|
|
session must differ. Iteration-bounded, not deadline-bounded: tests freeze ``time.monotonic``."""
|
|
iterations = max(1, int(max(float(timeout), 0.0) / 0.25))
|
|
for i in range(iterations):
|
|
session = getattr(srv, "session", None)
|
|
ready = getattr(srv, "_ready", None)
|
|
try:
|
|
is_ready = bool(ready.is_set()) if hasattr(ready, "is_set") else True
|
|
except Exception:
|
|
is_ready = True
|
|
if session is not None and session is not old_session and is_ready:
|
|
return True
|
|
if i < iterations - 1:
|
|
time.sleep(0.25)
|
|
return False
|
|
|
|
|
|
def _signal_reconnect_and_wait(server_name: str, srv: Any, *, op_description: str, timeout: float = 15.0) -> bool:
|
|
"""Request a transport rebuild and wait for the fresh session. ``_ready`` is cleared on the loop
|
|
BEFORE ``_reconnect_event`` is set, else the readiness poll returns at once on the dead session."""
|
|
loop = _core._mcp_loop
|
|
if loop is None or not loop.is_running():
|
|
return False
|
|
|
|
def _request_reconnect() -> None:
|
|
ready, reconnect_event = getattr(srv, "_ready", None), getattr(srv, "_reconnect_event", None)
|
|
if hasattr(ready, "clear"):
|
|
ready.clear()
|
|
if hasattr(reconnect_event, "set"):
|
|
reconnect_event.set()
|
|
|
|
old_session = getattr(srv, "session", None)
|
|
logger.info("MCP server '%s': %s requesting transport reconnect", server_name, op_description)
|
|
loop.call_soon_threadsafe(_request_reconnect)
|
|
return _wait_for_server_session_ready(srv, old_session=old_session, timeout=timeout)
|
|
|
|
|
|
def _ensure_mcp_loop():
|
|
"""Start the background loop thread if not running. The loop/thread handles live on the ORIGIN
|
|
module (tests read and reset ``tools.mcp_tool._mcp_loop``), so they are written there."""
|
|
from tools import mcp_tool as _origin
|
|
with _core._lock:
|
|
if _origin._mcp_loop is not None and _origin._mcp_loop.is_running():
|
|
return
|
|
loop = _origin._mcp_loop = asyncio.new_event_loop()
|
|
loop.set_exception_handler(_mcp_loop_exception_handler)
|
|
_origin._mcp_thread = threading.Thread(target=loop.run_forever, name="mcp-event-loop", daemon=True)
|
|
_origin._mcp_thread.start()
|
|
|
|
|
|
def _stop_mcp_loop(*, only_if_idle: bool = False) -> bool:
|
|
"""Stop the background event loop and join its thread."""
|
|
from tools import mcp_tool as _origin
|
|
with _core._lock:
|
|
if only_if_idle and (_core._servers or _core._server_connecting):
|
|
logger.debug("Leaving MCP event loop running; active servers are registered or connecting")
|
|
return False
|
|
loop, thread = _origin._mcp_loop, _origin._mcp_thread
|
|
_origin._mcp_loop = _origin._mcp_thread = None
|
|
if loop is None:
|
|
return True
|
|
# Drain before stopping: tasks still suspended when the loop closes get resumed by the GC
|
|
# against a closed loop. shutdown_mcp_servers only reaps _servers; everything else ends here.
|
|
future = None
|
|
# Drain before stopping: closing the loop with tasks still suspended leaves their coroutines for the GC,
|
|
# whose finalizer then resumes them to run cleanup against a loop that is already closed -> "Event loop
|
|
# is closed" (#60197).
|
|
if loop.is_running():
|
|
from agent.async_utils import safe_schedule_threadsafe
|
|
|
|
future = safe_schedule_threadsafe(
|
|
_lifecycle._drain_and_stop_mcp_loop(), loop, logger=logger,
|
|
log_message="MCP loop drain: failed to schedule", log_level=logging.WARNING)
|
|
if future is not None:
|
|
try:
|
|
future.result(timeout=_core._MCP_LOOP_DRAIN_TIMEOUT + 1)
|
|
except TimeoutError:
|
|
logger.warning("Timed out waiting for MCP loop drain after %.1fs", _core._MCP_LOOP_DRAIN_TIMEOUT + 1)
|
|
except BaseException as exc:
|
|
logger.warning("Error draining MCP loop tasks: %s", exc)
|
|
elif not loop.is_closed():
|
|
try:
|
|
loop.run_until_complete(_lifecycle._drain_mcp_loop_tasks(timeout=_core._MCP_LOOP_DRAIN_TIMEOUT))
|
|
except BaseException as exc:
|
|
logger.warning("Error draining stopped MCP loop tasks: %s", exc)
|
|
if future is None and loop.is_running(): # drain-and-stop wasn't scheduled: stop it ourselves
|
|
loop.call_soon_threadsafe(loop.stop)
|
|
if thread is not None:
|
|
thread.join(timeout=5)
|
|
if thread.is_alive():
|
|
logger.warning("MCP event loop thread did not stop within 5.0s")
|
|
try:
|
|
loop.close()
|
|
except Exception as exc:
|
|
logger.warning("Unable to close MCP event loop cleanly: %s", exc)
|
|
# The loop is gone, so no session can be in flight: reap active too.
|
|
_lifecycle._kill_orphaned_mcp_children(include_active=True)
|
|
return True
|