mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 02:46:38 +03:00
c7a6e433ca
close_all had no production caller, so ending the gateway that serves the Webapp left every SSH tunnel it opened running, with the remote serve it owns still alive. The Webapp lifespan now runs it after the store watcher stops.
244 lines
10 KiB
Python
244 lines
10 KiB
Python
"""HTTP lifespan composition; standalone backend services remain opt-in owners."""
|
|
|
|
from contextlib import asynccontextmanager
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import threading
|
|
|
|
from fastapi import FastAPI
|
|
from hermes_cli.pty_session import run_reaper
|
|
|
|
logger = logging.getLogger("hermes_cli.web_server")
|
|
|
|
|
|
def _unlink_pty_markers(app: FastAPI) -> None:
|
|
"""Second pass after ``PTY_REGISTRY.close_all()``: drop channel markers left in app state,
|
|
including stale paths from sessions reaped earlier (``PtySession.close()`` only removes
|
|
markers owned by live registry sessions)."""
|
|
from hermes_cli.web_server import _get_pty_active_session_files
|
|
|
|
for marker in set(_get_pty_active_session_files(app).values()):
|
|
try:
|
|
marker.unlink(missing_ok=True)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
@asynccontextmanager
|
|
async def standalone_lifespan(app: "FastAPI"):
|
|
from hermes_cli.web_server import (
|
|
_eager_reconcile_own_session_db, _warm_gateway_module,
|
|
_start_desktop_cron_ticker, _dashboard_selftest_loop,
|
|
_auto_archive_ticker_loop, _terminate_desktop_managed_gateway,
|
|
)
|
|
from hermes_cli.web_server_chat import PTY_REGISTRY
|
|
|
|
app.state.event_channels = {} # dict[str, set]
|
|
app.state.event_lock = asyncio.Lock()
|
|
app.state.pty_active_session_files = {} # dict[str, Path]
|
|
# Serializes chat-argv resolution so concurrent /api/pty connections don't
|
|
# overlap ``npm install`` / ``npm run build``. Locks live on app.state (not
|
|
# module globals) so they bind to the running loop, not the import-time one.
|
|
app.state.chat_argv_lock = asyncio.Lock()
|
|
|
|
# Bring state.db schema current BEFORE the first session-list poll
|
|
# (#79531/#80037): a store left behind by `hermes update` otherwise 500s
|
|
# every poll until its owner completes schema reconciliation.
|
|
# Off-thread so a locked store never delays the socket (Desktop
|
|
# ready-probe times out at 10s, GH-73083). NOT a daemon, and joined at
|
|
# shutdown: its sqlite connection must be closed by the thread that is
|
|
# stepping it. A daemon copy that outlived the lifespan had its
|
|
# connection closed from the main thread mid-probe (pytest's leaked-DB
|
|
# sweep) and segfaulted the interpreter. The worker is time-bounded by
|
|
# SessionDB's lock patience, so the join cannot hang shutdown.
|
|
eager_reconcile_thread = threading.Thread(
|
|
target=_eager_reconcile_own_session_db,
|
|
name="statedb-eager-reconcile",
|
|
)
|
|
eager_reconcile_thread.start()
|
|
|
|
# Import hermes_cli.gateway *before* the yield: on Windows + 3.11 the
|
|
# import holds the GIL, so run_in_executor still froze the loop 15-22s and
|
|
# the Desktop's 10s ready-probe timed out (GH-73083).
|
|
_warm_gateway_module()
|
|
|
|
# Snapshot the checkout revision so lazy-import paths (model picker) can
|
|
# refuse with "restart required" after `hermes update` replaced the code
|
|
# (#86207); the update flow does not reliably restart the dashboard.
|
|
from gateway.code_skew import record_boot_fingerprint
|
|
|
|
record_boot_fingerprint()
|
|
|
|
# Hosted Bot rooms belong to the backend process. Recovery may need a
|
|
# contended state.db migration, so keep it off the pre-yield path: Group
|
|
# Chat must degrade on its own rather than block every Desktop feature.
|
|
from tui_gateway import methods_groups as _hosted_groups
|
|
import tui_gateway.server
|
|
|
|
try:
|
|
tui_gateway.server.install_tui_message_injector()
|
|
except Exception:
|
|
logger.warning("TUI message injector did not install", exc_info=True)
|
|
|
|
hosted_room_start_cancel = threading.Event()
|
|
|
|
def _start_hosted_rooms() -> None:
|
|
try:
|
|
_hosted_groups.start_hosted_room_service()
|
|
except Exception:
|
|
logger.exception("Hosted Group Chat recovery failed during backend startup")
|
|
finally:
|
|
if hosted_room_start_cancel.is_set():
|
|
_hosted_groups.stop_hosted_room_service(timeout=1.0)
|
|
|
|
hosted_room_start_thread = threading.Thread(
|
|
target=_start_hosted_rooms,
|
|
daemon=True,
|
|
name="hosted-room-startup",
|
|
)
|
|
hosted_room_start_thread.start()
|
|
|
|
# Desktop-spawned backends fire cron jobs themselves, since the app has no
|
|
# gateway running the scheduler. Server `hermes dashboard` is unaffected —
|
|
# it relies on its own gateway.
|
|
from hermes_cli.process_identity import is_desktop_owned_backend
|
|
cron_stop: threading.Event | None = None
|
|
cron_thread: threading.Thread | None = None
|
|
desktop_owned = is_desktop_owned_backend()
|
|
if desktop_owned:
|
|
# Reap an orphaned gateway from an abnormal previous exit (reparented to
|
|
# launchd, still holding the platform WebSocket) before forking a fresh
|
|
# one that would race the same credential (#77276). Runs
|
|
# unconditionally; protection of a healthy standalone gateway lives
|
|
# INSIDE the reaper (registration probed with cleanup_stale=False).
|
|
# Startup grace: spare a gateway still claiming gateway.pid/lock (#122533).
|
|
try:
|
|
from hermes_cli.dashboard_procs import _REAP_MIN_AGE_SECONDS
|
|
from hermes_cli.gateway import _reap_unsupervised_gateway_orphans
|
|
|
|
_reap_unsupervised_gateway_orphans(min_age_s=_REAP_MIN_AGE_SECONDS)
|
|
except Exception:
|
|
logger.exception("Desktop startup: orphan gateway reap failed")
|
|
|
|
cron_stop = threading.Event()
|
|
cron_thread = threading.Thread(
|
|
target=_start_desktop_cron_ticker,
|
|
args=(cron_stop,),
|
|
daemon=True,
|
|
name="desktop-cron-ticker",
|
|
)
|
|
cron_thread.start()
|
|
|
|
# Reap idle/dead keep-alive PTY sessions (30-min TTL). The registry is process-wide and
|
|
# close_all() refuses new spawns, so a re-entered lifespan reopens it first.
|
|
PTY_REGISTRY.reopen()
|
|
pty_reaper_task = asyncio.create_task(run_reaper(PTY_REGISTRY))
|
|
# Periodic authenticated self-test feeding the ``dashboard`` component on /api/status.
|
|
selftest_task = asyncio.create_task(_dashboard_selftest_loop())
|
|
# Live auto-archive timer, independent of list requests.
|
|
auto_archive_task = asyncio.create_task(_auto_archive_ticker_loop())
|
|
|
|
# Managed local runtime (local_runtime.enabled): bring llama-server back so a
|
|
# restart doesn't strand a llamacpp main model. Off-thread and best-effort;
|
|
# failure falls back to cloud providers like a cold start. Server only —
|
|
# models load on first inference (an empty router holds no VRAM).
|
|
def _boot_local_runtime():
|
|
try:
|
|
from hermes_cli.config import load_config
|
|
from hermes_cli.local_runtime.bootstrap import ensure_local_runtime
|
|
|
|
ensure_local_runtime(load_config())
|
|
except Exception as exc:
|
|
logging.getLogger(__name__).warning("local runtime boot failed: %s", exc)
|
|
|
|
threading.Thread(target=_boot_local_runtime, daemon=True, name="local-runtime-boot").start()
|
|
|
|
# Records who carries inference for setup.status / free_tier.status and broadcasts `setup.ready`
|
|
# (no identity is created at boot). Off-thread; the desktop's first setup.status waits on the
|
|
# record (bounded) instead.
|
|
from hermes_cli.free_tier_bootstrap import start_background_bootstrap
|
|
|
|
start_background_bootstrap()
|
|
|
|
try:
|
|
yield
|
|
finally:
|
|
try:
|
|
tui_gateway.server.clear_tui_message_injector()
|
|
except Exception:
|
|
logger.debug("TUI message injector clear skipped", exc_info=True)
|
|
hosted_room_start_cancel.set()
|
|
_hosted_groups.stop_hosted_room_service(timeout=5.0)
|
|
hosted_room_start_thread.join(timeout=1.0)
|
|
if cron_stop is not None:
|
|
cron_stop.set()
|
|
pty_reaper_task.cancel()
|
|
selftest_task.cancel()
|
|
auto_archive_task.cancel()
|
|
eager_reconcile_thread.join()
|
|
await PTY_REGISTRY.close_all()
|
|
|
|
# PtySession.close() removes markers owned by live registry sessions.
|
|
# This second pass cleans any channel markers left in app state,
|
|
# including stale paths from sessions reaped earlier.
|
|
_unlink_pty_markers(app)
|
|
|
|
# Stop the managed llama-server with its parent (an orphan pins VRAM).
|
|
try:
|
|
from hermes_cli.local_runtime.bootstrap import shutdown_local_runtime
|
|
|
|
shutdown_local_runtime()
|
|
except Exception:
|
|
# Shutdown boundary: a supervisor that fails to stop must not skip the
|
|
# managed-gateway teardown below; log it so the orphan is diagnosable.
|
|
logger.warning("managed local runtime shutdown failed", exc_info=True)
|
|
if desktop_owned:
|
|
_terminate_desktop_managed_gateway()
|
|
|
|
|
|
@asynccontextmanager
|
|
async def http_lifespan(app: FastAPI):
|
|
"""The gateway already owns schema, rooms, scheduler and model services."""
|
|
from contextlib import AsyncExitStack
|
|
|
|
from hermes_cli.web_server import _dashboard_selftest_loop
|
|
from hermes_cli.web_server_chat import PTY_REGISTRY
|
|
from hermes_cli.web_server_surface import policy
|
|
|
|
app.state.event_channels = {}
|
|
app.state.event_lock = asyncio.Lock()
|
|
app.state.pty_active_session_files = {}
|
|
app.state.chat_argv_lock = asyncio.Lock()
|
|
PTY_REGISTRY.reopen() # a prior lifespan's close_all() refuses spawns until reopened
|
|
tasks = [
|
|
asyncio.create_task(run_reaper(PTY_REGISTRY), name="gateway-api-pty-reaper"),
|
|
asyncio.create_task(_dashboard_selftest_loop(), name="gateway-api-selftest"),
|
|
]
|
|
try:
|
|
async with AsyncExitStack() as surface:
|
|
if policy(app.state).host_terminal:
|
|
from hermes_cli.web_host_terminal_sessions import host_terminal_lifespan
|
|
await surface.enter_async_context(host_terminal_lifespan(app))
|
|
if policy(app.state).gateway_connections:
|
|
from hermes_cli import gateway_connections_ssh
|
|
from hermes_cli.gateway_connections_watch import watch_store
|
|
# Registered first so it runs last: the watcher stops, then every SSH tunnel this
|
|
# listener opened (and the remote serve it owns) is closed, never orphaned.
|
|
surface.push_async_callback(asyncio.to_thread, gateway_connections_ssh.close_all)
|
|
await surface.enter_async_context(watch_store())
|
|
yield
|
|
finally:
|
|
for task in tasks:
|
|
task.cancel()
|
|
await asyncio.gather(*tasks, return_exceptions=True)
|
|
await PTY_REGISTRY.close_all()
|
|
_unlink_pty_markers(app)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def app_lifespan(app: FastAPI):
|
|
lifespan = http_lifespan if getattr(app.state, "gateway_runner", None) is not None else standalone_lifespan
|
|
async with lifespan(app):
|
|
yield
|