mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-20 18:50:12 +03:00
6e9f696ee6
Round-3 review of the gateway create path found three defects and one
structural duplication.
- A config backend outage mid-create read as "nothing was created".
ConfigBackendUnavailable is a SystemExit, so it escaped the handler's
`except Exception` and the dispatcher answered 4064, which the client
read as refused-before-create, even when the profile had already been
published. The handler now catches it (and builds the receipt inside
the same try) and answers 5062. The client no longer treats a bare
4064 as "nothing was created"; only 4061 (no name) is.
- Every `hermes profile create` loaded gateway.platforms (~225 modules,
about +105 ms median) even with no gateway URL, because the
safe_proxy_url_for_log import sat above _where's early return. It now
loads only when a URL is set; a test pins that a URL-less create loads
neither gateway.platforms nor pydantic.
- A newer gateway that added a key under `mirrored` turned a successful
create into "may have created": the strict check forbade extra keys
there. `mirrored` is now a mapping of known value types, so new keys
are kept and never printed.
The receipt had two definitions: the CreateReceipt dataclass and the
contract's ProfileCreateReceipt pydantic model, kept in step by a
field-name test and a hand-written check that skills_seeded["copied"] was
a list. There is now one: the stdlib dataclass ProfileCreateReceipt in
hermes_cli.profile_provision, which the contract uses directly as
ProfilesCreateResult.receipt (pydantic validates dataclasses), so the
generated TS/OpenRPC still describe it. from_wire validates it strictly
with a TypeAdapter (pydantic only on the gateway path). The receipt
carries what the CLI prints, skills_synced (a count) and
skills_opted_out, instead of the raw sync-result dict, which removes the
`copied` check; RECEIPT_FIELDS and the parity test are gone.
Also:
- The refusal chunk limits are imported from gateway_client
(ERROR_DATA_TEXT / ERROR_DATA_ITEMS, now named there) instead of
copied, so lowering the client cap can't silently drop every part.
- A typed refusal with no text is still a refusal ("nothing was
created"), not "may have created".
- POST /api/profiles uses CREATE_REFUSALS.
- The wrapper check reuses profiles.profile_matches_home.
- The developer guide states the wrapper rule the code enforces.
394 lines
20 KiB
Python
394 lines
20 KiB
Python
"""Transport-only RPC client. Closing a viewer never stops its authority."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from collections import deque
|
|
from contextlib import asynccontextmanager, contextmanager, suppress
|
|
import contextvars
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
|
|
# The owner's protocol name; session_contract is dependency-free (no config/plugins load).
|
|
from gateway.session_contract import CANONICAL_GATEWAY_PROTOCOL as GATEWAY_WS_PROTOCOL
|
|
|
|
GATEWAY_WS_TICKET_PREFIX = "hermes-gateway-ticket."
|
|
# Per-request answer budget. Owner verbs are admissions/reads that answer promptly; the one that
|
|
# legitimately runs long passes its own budget (``rpc(..., _timeout=...)``).
|
|
DEFAULT_RPC_TIMEOUT = 30
|
|
# A typed ``--api-key`` on a resume that the owner does not bind (``rebind_launch_key``): every CLI says this.
|
|
RESUME_KEY_MISMATCH = ("--api-key on resume must be the key this session was launched with "
|
|
"(a session launched without --api-key keeps its profile credentials). "
|
|
"A session never switches keys: start a new one (without --resume) to use a different key.")
|
|
# Canonical ``/compress`` answers only after the summary call(s) and the commit: a long session's
|
|
# summary routinely takes more than 30 s (``auxiliary.compression.timeout`` defaults to 120 s per
|
|
# request, ``compression.context_total_ceiling_seconds`` to 600 s). Same budget as Desktop's
|
|
# ``SESSION_COMPRESS_TIMEOUT_MS`` so every surface gives up at the same point.
|
|
COMPRESS_RPC_TIMEOUT = 660
|
|
# A full event queue parks the socket reader; a consumer that takes no event for this long has
|
|
# stalled and is failed. Same bound as the owner's own send deadline (``tui_gateway/ws.py``).
|
|
EVENT_STALL_TIMEOUT = 30
|
|
# Events read past a full queue while an RPC answer is awaited (the owner sends a re-attached
|
|
# session's frames ahead of its ``session.resume`` answer, and a recovery RPC is awaited from the
|
|
# queue's only consumer). Bounded: past either cap the reader waits for the consumer again.
|
|
EVENT_OVERFLOW_FRAMES = 16384
|
|
EVENT_OVERFLOW_BYTES = 8 * 1024 * 1024
|
|
|
|
|
|
def gateway_ws_target(endpoint, ticket):
|
|
"""``(url, subprotocols)`` for dialing the owner's canonical WebSocket with a session ticket.
|
|
|
|
The owner accepts exactly one ticket subprotocol beside ``GATEWAY_WS_PROTOCOL`` and echoes the
|
|
latter; callers check ``ws.subprotocol == GATEWAY_WS_PROTOCOL`` after the upgrade."""
|
|
url = endpoint.api_origin.replace("https:", "wss:").replace("http:", "ws:") + "/api/ws"
|
|
return url, [GATEWAY_WS_PROTOCOL, GATEWAY_WS_TICKET_PREFIX + ticket]
|
|
|
|
|
|
class GatewayClientError(ValueError):
|
|
pass
|
|
|
|
|
|
DISCONNECTED = ("Gateway disconnected; turn outcome is unknown. "
|
|
"Resume the printed session ID to check whether it completed or was interrupted; do not resend the work.")
|
|
# A request refused before it reached any socket: nothing is in doubt, so never DISCONNECTED.
|
|
NOT_SENT = "Gateway disconnected before the request was sent; nothing was sent."
|
|
|
|
|
|
class GatewayRPCError(GatewayClientError):
|
|
"""A received owner refusal, distinct from an ambiguous transport failure.
|
|
|
|
``str(exc)`` is the bounded reason code; ``code`` is the frame's numeric JSON-RPC code (``None``
|
|
when absent or not an int) and ``data`` its bounded, scalar-only ``error.data`` mapping."""
|
|
|
|
def __init__(self, reason, code=None, data=None):
|
|
super().__init__(reason)
|
|
self.code = code
|
|
self.data = data or {}
|
|
|
|
@property
|
|
def unknown_method(self) -> bool:
|
|
"""The owner has no such method (it predates this client): ``-32601``, or only the reason
|
|
name from an owner that sends no code."""
|
|
return self.code == -32601 or str(self) == "unknown_method"
|
|
|
|
|
|
def _bounded_reason(value):
|
|
return value if isinstance(value, str) and value.replace("_", "").isalnum() and len(value) <= 80 else None
|
|
|
|
|
|
#: What ``error.data`` keeps: strings up to this long, lists and key counts up to ``ERROR_DATA_ITEMS``.
|
|
ERROR_DATA_TEXT, ERROR_DATA_ITEMS = 200, 16
|
|
|
|
|
|
def _bounded_error_data(data):
|
|
"""At most 16 short keys whose values are bool/int/None, short strings or short lists of them;
|
|
anything else (nested objects, raw remote diagnostics) is dropped, never forwarded."""
|
|
def scalar(value):
|
|
return value is None or isinstance(value, (bool, int)) or (
|
|
isinstance(value, str) and len(value) <= ERROR_DATA_TEXT)
|
|
if not isinstance(data, dict):
|
|
return {}
|
|
out = {}
|
|
for key, value in data.items():
|
|
if len(out) >= ERROR_DATA_ITEMS or not isinstance(key, str) or len(key) > 64:
|
|
continue
|
|
if scalar(value):
|
|
out[key] = value
|
|
elif isinstance(value, list) and len(value) <= ERROR_DATA_ITEMS and all(scalar(item) for item in value):
|
|
out[key] = list(value)
|
|
return out
|
|
|
|
|
|
def rpc_error(error):
|
|
"""The ``GatewayRPCError`` for one received ``error`` member: only bounded reason codes, never
|
|
raw remote diagnostics. A free-text message (``unknown method: x``) falls back to the bounded
|
|
``data.reason`` the owner sends beside it, then to ``request_failed``."""
|
|
error = error if isinstance(error, dict) else {}
|
|
data = _bounded_error_data(error.get("data"))
|
|
code = error.get("code")
|
|
reason = _bounded_reason(error.get("message", "request_failed")) or _bounded_reason(data.get("reason"))
|
|
return GatewayRPCError(reason or "request_failed",
|
|
code if isinstance(code, int) and not isinstance(code, bool) else None, data)
|
|
|
|
|
|
class GatewayUnavailableError(GatewayClientError):
|
|
"""No ready owner to dial: ``state`` is ``ensure_gateway_runtime``'s verdict (``draining`` while an
|
|
update holds the install, ``starting`` past the deadline, ...). Nothing was submitted."""
|
|
|
|
def __init__(self, message, state, reason_code=None):
|
|
super().__init__(message)
|
|
self.state, self.reason_code = state, reason_code
|
|
|
|
|
|
class EventQueue(asyncio.Queue):
|
|
"""The client's bounded event queue plus an ordered overflow behind it: each ``get`` refills the
|
|
queue from the overflow, so items arrive in read order and ``qsize``/``full`` see only the queue."""
|
|
|
|
def __init__(self, maxsize):
|
|
super().__init__(maxsize)
|
|
self.overflow = deque() # (size, item)
|
|
self.overflow_bytes = 0
|
|
self.moved = asyncio.Event() # an item left the overflow (or an RPC started waiting)
|
|
|
|
def _get(self):
|
|
item = super()._get()
|
|
if self.overflow:
|
|
size, held = self.overflow.popleft()
|
|
self.overflow_bytes -= size
|
|
super()._put(held)
|
|
self.moved.set()
|
|
return item
|
|
|
|
def hold(self, item, size=0):
|
|
"""Queue *item* behind everything already read, never dropping it."""
|
|
if not self.overflow:
|
|
try:
|
|
self.put_nowait(item)
|
|
return
|
|
except asyncio.QueueFull:
|
|
pass
|
|
self.overflow.append((size, item))
|
|
self.overflow_bytes += size
|
|
|
|
|
|
def put_behind(events, item):
|
|
"""Queue a local item (ACP's post-ack recovery) behind every frame read so far; never dropped."""
|
|
if isinstance(events, EventQueue):
|
|
events.hold(item)
|
|
else:
|
|
events.put_nowait(item)
|
|
|
|
|
|
class GatewayClient:
|
|
def __init__(self, websocket):
|
|
self.websocket = websocket
|
|
self.events = EventQueue(maxsize=4096)
|
|
self.pending = {}
|
|
self.sequence = 0
|
|
self.reader = None
|
|
|
|
def _read_on(self):
|
|
"""Keep reading past a full queue: nothing is held, or an awaited answer may sit behind the
|
|
held frames (its waiter may be the queue's only consumer) and the overflow has room."""
|
|
events = self.events
|
|
return not events.overflow or (
|
|
len(events.overflow) < EVENT_OVERFLOW_FRAMES and events.overflow_bytes < EVENT_OVERFLOW_BYTES
|
|
and any(not future.done() for future in self.pending.values()))
|
|
|
|
async def _deliver(self, frame, size):
|
|
self.events.hold(frame, size)
|
|
if self._read_on():
|
|
return
|
|
# Frames the socket already buffered (a CLI resumed from Ctrl-Z) are read without yielding,
|
|
# so the consumer has not had a turn yet: wait for it (or for an RPC that needs the reader).
|
|
# Not wait_for: on 3.11 it swallows a close that lands as the wait completes.
|
|
async with asyncio.timeout(EVENT_STALL_TIMEOUT):
|
|
while not self._read_on():
|
|
self.events.moved.clear()
|
|
await self.events.moved.wait()
|
|
|
|
async def __aenter__(self):
|
|
self.reader = asyncio.create_task(self._read())
|
|
return self
|
|
|
|
async def __aexit__(self, *exc):
|
|
self.reader.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await self.reader
|
|
|
|
async def _read(self):
|
|
from websockets.exceptions import ConnectionClosed
|
|
try:
|
|
async for raw in self.websocket:
|
|
frame = json.loads(raw)
|
|
future = self.pending.get(frame.get("id"))
|
|
if future is not None and not future.done():
|
|
if "error" in frame:
|
|
future.set_exception(rpc_error(frame["error"]))
|
|
else:
|
|
future.set_result(frame.get("result", {}))
|
|
elif "method" in frame:
|
|
await self._deliver(frame, len(raw))
|
|
except (ConnectionClosed, OSError, ValueError, TimeoutError):
|
|
pass
|
|
finally:
|
|
error = GatewayClientError(DISCONNECTED)
|
|
for future in self.pending.values():
|
|
if not future.done():
|
|
future.set_exception(error)
|
|
# A stalled consumer is failed rather than silently losing terminal events: behind
|
|
# whatever is held, or in place of the oldest queued frame when nothing is.
|
|
if self.events.full() and not self.events.overflow:
|
|
self.events.get_nowait()
|
|
self.events.hold(error)
|
|
|
|
async def rpc(self, method, *, _timeout=None, **params):
|
|
"""Send one request and await its answer; ``_timeout`` (seconds, never sent on the wire)
|
|
overrides ``DEFAULT_RPC_TIMEOUT`` for a verb that legitimately takes longer (``math.inf``: no
|
|
deadline; the answer, a hang-up or the caller's cancel ends the wait)."""
|
|
from websockets.protocol import State
|
|
if self.websocket.state is not State.OPEN or (self.reader is not None and self.reader.done()):
|
|
# A closing socket (a keepalive timeout fails it, then waits up to close_timeout for the
|
|
# peer) is never written to, and a reader that ended (closed socket, bad frame, stalled
|
|
# consumer) could not read the answer: refuse before writing, so the caller knows nothing left.
|
|
raise GatewayClientError(NOT_SENT)
|
|
self.sequence += 1
|
|
rid = self.sequence
|
|
future = asyncio.get_running_loop().create_future()
|
|
self.pending[rid] = future
|
|
self.events.moved.set() # a reader waiting on the consumer reads on toward this answer
|
|
from websockets.exceptions import ConnectionClosed
|
|
try:
|
|
try:
|
|
await self.websocket.send(json.dumps({"jsonrpc": "2.0", "id": rid, "method": method, "params": params}))
|
|
except (ConnectionClosed, OSError) as exc:
|
|
# The frame may have left before the socket died: the same unknown outcome as a lost answer.
|
|
raise GatewayClientError(DISCONNECTED) from exc
|
|
budget = DEFAULT_RPC_TIMEOUT if _timeout is None else _timeout
|
|
return await asyncio.wait_for(future, None if budget == float("inf") else budget)
|
|
finally:
|
|
self.pending.pop(rid, None)
|
|
if future.done() and not future.cancelled():
|
|
future.exception() # retrieved, so a failure nobody awaited is never logged as unretrieved
|
|
|
|
|
|
def private_control_query(home: Path, request: bytes, timeout: float = 5) -> bytes:
|
|
"""Send one request line to *home*'s gateway control socket as this OS user; the raw answer.
|
|
|
|
The socket authenticates the peer (same user only), which is what lets a private verb answer."""
|
|
if os.name == "nt":
|
|
from gateway.runtime_bootstrap_windows import query_runtime_control
|
|
return query_runtime_control(home, request, timeout)
|
|
from hermes_cli.gateway_runtime_discovery import connect_private
|
|
deadline = time.monotonic() + timeout
|
|
with connect_private(home, timeout) as peer:
|
|
peer.sendall(request)
|
|
data = bytearray()
|
|
while b"\n" not in data:
|
|
budget = deadline - time.monotonic()
|
|
if budget <= 0:
|
|
raise GatewayClientError("Gateway bootstrap timed out")
|
|
peer.settimeout(budget)
|
|
chunk = peer.recv(4096)
|
|
if not chunk or len(data) + len(chunk) > 65536:
|
|
raise GatewayClientError("Invalid gateway bootstrap response")
|
|
data.extend(chunk)
|
|
return bytes(data)
|
|
|
|
|
|
# An owner that predates ``restart_wait`` refuses the unknown parameter with exactly this answer.
|
|
_UNKNOWN_TICKET_PARAMS = "PermissionError: invalid bootstrap parameters"
|
|
|
|
|
|
def _session_ticket(home: Path, endpoint, *, purpose="interactive", scope=None, subject=None,
|
|
refuse_restart_wait=False) -> str:
|
|
from hermes_cli.gateway_runtime import control_home_for
|
|
from hermes_cli.gateway_runtime_discovery import _identify_response
|
|
# A served secondary's ticket is minted by the multiplexer's socket, bound to the secondary.
|
|
home = control_home_for(home, endpoint)
|
|
# ``subject`` names the principal a same-user caller acts for (the standalone dashboard's
|
|
# authenticated user) instead of the socket UID; the owner's peer gate stays the boundary.
|
|
# ``restart_wait``: the owner refuses the socket at admission if a restart wait has opened by then.
|
|
params = {"profile_id": endpoint.profile_id, "instance_id": endpoint.instance_id, "purpose": purpose,
|
|
**({"scope": scope} if scope else {}), **({"subject": subject} if subject else {}),
|
|
**({"restart_wait": "refuse"} if refuse_restart_wait else {})}
|
|
request = json.dumps({"protocol": 1, "id": 1, "verb": "session-ticket", "params": params}).encode() + b"\n"
|
|
answer = private_control_query(home, request)
|
|
refused = json.loads(answer.split(b"\n", 1)[0]) if refuse_restart_wait else None
|
|
if isinstance(refused, dict) and refused.get("error") == _UNKNOWN_TICKET_PARAMS:
|
|
# An owner older than this client (the code on disk was updated under it) cannot hold the
|
|
# policy; refusing it outright would strand the caller until it restarts. The pre-dial check stands.
|
|
del params["restart_wait"]
|
|
answer = private_control_query(home, json.dumps(
|
|
{"protocol": 1, "id": 1, "verb": "session-ticket", "params": params}).encode() + b"\n")
|
|
grant = _identify_response(answer)
|
|
if any(grant.get(k) != v for k, v in {
|
|
"profile_id": endpoint.profile_id, "instance_id": endpoint.instance_id, "runtime_protocol": 1,
|
|
}.items()) or not isinstance(grant.get("ticket"), str) or not grant["ticket"]:
|
|
raise GatewayClientError("Gateway bootstrap identity changed; retry launch")
|
|
return grant["ticket"]
|
|
|
|
|
|
# A restart's after-turn wait admits clients so a human can answer a parked prompt (#139041). A machine
|
|
# poller held in it (a Kanban worker reading its card's receipt) keeps the turn the wait is waiting
|
|
# on alive, so the restart, and ``hermes update`` behind it, is held until its cap.
|
|
_REFUSE_RESTART_WAIT: contextvars.ContextVar[bool] = contextvars.ContextVar("refuse_restart_wait", default=False)
|
|
|
|
|
|
@contextmanager
|
|
def outside_restart_wait():
|
|
"""``connect_gateway`` calls inside never attach to an owner in its restart's after-turn wait."""
|
|
token = _REFUSE_RESTART_WAIT.set(True)
|
|
try:
|
|
yield
|
|
finally:
|
|
_REFUSE_RESTART_WAIT.reset(token)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def connect_gateway(endpoint=None, subject=None, *, max_size=8 * 1024 * 1024):
|
|
"""Attach to this home's owner. ``endpoint``: an already-discovered ready owner to dial as-is
|
|
(maintenance that must never start a gateway); default ensures one, starting it if needed.
|
|
``subject``: mint the ticket for this principal (the caller's authenticated dashboard user)
|
|
rather than the socket identity; only a peer the owner already trusts may name one.
|
|
``max_size``: the largest frame accepted; a ``session.resume`` answer carries the whole history.
|
|
Inside ``outside_restart_wait()`` an owner in a restart's after-turn wait is refused as
|
|
``draining`` (``restart_wait``) instead of attached, and the ticket carries that policy so the
|
|
owner also refuses the socket when the wait opens between discovery and the dial."""
|
|
from websockets.asyncio.client import connect
|
|
from hermes_constants import get_hermes_home
|
|
from hermes_cli.gateway_runtime import attachable, ensure_gateway_runtime, gateway_profile_home
|
|
from hermes_cli.gateway_runtime_discovery import DiscoveryError
|
|
from urllib.parse import urlsplit
|
|
|
|
remote = "" if endpoint is not None else os.environ.get("HERMES_TUI_GATEWAY_URL", "").strip()
|
|
protocols = None
|
|
if endpoint is not None:
|
|
ticket = await asyncio.to_thread(_session_ticket, gateway_profile_home(get_hermes_home()), endpoint,
|
|
subject=subject)
|
|
url, protocols = gateway_ws_target(endpoint, ticket)
|
|
elif remote:
|
|
parsed = urlsplit(remote)
|
|
if parsed.scheme not in {"ws", "wss"} or not parsed.hostname:
|
|
raise GatewayClientError("Invalid explicit gateway WebSocket URL; no local fallback")
|
|
url = remote
|
|
else:
|
|
# A per-task overlay home is served by the profile its symlinked lock names (#138949).
|
|
home = gateway_profile_home(get_hermes_home())
|
|
# A client's own start: the daemon it may launch ends itself once idle (no client, adapter,
|
|
# cron/kanban work or admission), and the next client starts a fresh one.
|
|
result = await asyncio.to_thread(ensure_gateway_runtime, home, idle_exit=True)
|
|
if not attachable(result) or (_REFUSE_RESTART_WAIT.get() and result.state != "ready"):
|
|
detail = getattr(result, "detail", None)
|
|
if result.reason_code == "runtime_exited" and detail:
|
|
# A multi-line startup report (redacted traceback tail + where the full log is).
|
|
raise GatewayUnavailableError(f"Gateway could not start: {detail}", result.state, result.reason_code)
|
|
detail = f" ({detail})" if detail else ""
|
|
raise GatewayUnavailableError(f"Gateway {result.state}: {result.reason_code or 'not_ready'}{detail}",
|
|
result.state, result.reason_code)
|
|
endpoint = result.endpoint
|
|
try:
|
|
policy: dict = {"refuse_restart_wait": True} if _REFUSE_RESTART_WAIT.get() else {}
|
|
ticket = await asyncio.to_thread(_session_ticket, home, endpoint, **policy)
|
|
except DiscoveryError as exc:
|
|
# The control socket refused the ticket: the verdict discovery gives the same refusal.
|
|
detail = exc.bounded_detail(500)
|
|
raise GatewayUnavailableError(f"Gateway inaccessible: {exc.reason}" + (f" ({detail})" if detail else ""),
|
|
"inaccessible", exc.reason) from exc
|
|
url, protocols = gateway_ws_target(endpoint, ticket)
|
|
try:
|
|
# The gateway is a loopback (or explicitly named) peer, never something to route through the
|
|
# user's HTTP(S) proxy; websockets>=14 reads HTTP_PROXY/HTTPS_PROXY by default and a proxy that
|
|
# cannot reach 127.0.0.1 turns every launch into a 10 s open timeout.
|
|
async with connect(url, subprotocols=protocols, open_timeout=10, max_size=max_size,
|
|
proxy=None) as ws:
|
|
if protocols and ws.subprotocol != GATEWAY_WS_PROTOCOL:
|
|
raise GatewayClientError("Gateway protocol mismatch; update/restart required")
|
|
async with GatewayClient(ws) as client:
|
|
yield client
|
|
except GatewayClientError:
|
|
raise
|
|
except (OSError, TimeoutError) as exc:
|
|
raise GatewayClientError("Gateway connection failed; no local fallback") from exc
|