Files
Teknium c7a6e433ca fix(webapp): close saved-gateway SSH tunnels when the Webapp listener stops
close_all had no production caller, so ending the gateway that serves the
Webapp left every SSH tunnel it opened running, with the remote serve it
owns still alive. The Webapp lifespan now runs it after the store watcher
stops.
2026-11-07 12:01:04 -08:00

244 lines
10 KiB
Python

"""HTTP lifespan composition; standalone backend services remain opt-in owners."""
from contextlib import asynccontextmanager
import asyncio
import logging
import os
import threading
from fastapi import FastAPI
from hermes_cli.pty_session import run_reaper
logger = logging.getLogger("hermes_cli.web_server")
def _unlink_pty_markers(app: FastAPI) -> None:
"""Second pass after ``PTY_REGISTRY.close_all()``: drop channel markers left in app state,
including stale paths from sessions reaped earlier (``PtySession.close()`` only removes
markers owned by live registry sessions)."""
from hermes_cli.web_server import _get_pty_active_session_files
for marker in set(_get_pty_active_session_files(app).values()):
try:
marker.unlink(missing_ok=True)
except OSError:
pass
@asynccontextmanager
async def standalone_lifespan(app: "FastAPI"):
from hermes_cli.web_server import (
_eager_reconcile_own_session_db, _warm_gateway_module,
_start_desktop_cron_ticker, _dashboard_selftest_loop,
_auto_archive_ticker_loop, _terminate_desktop_managed_gateway,
)
from hermes_cli.web_server_chat import PTY_REGISTRY
app.state.event_channels = {} # dict[str, set]
app.state.event_lock = asyncio.Lock()
app.state.pty_active_session_files = {} # dict[str, Path]
# Serializes chat-argv resolution so concurrent /api/pty connections don't
# overlap ``npm install`` / ``npm run build``. Locks live on app.state (not
# module globals) so they bind to the running loop, not the import-time one.
app.state.chat_argv_lock = asyncio.Lock()
# Bring state.db schema current BEFORE the first session-list poll
# (#79531/#80037): a store left behind by `hermes update` otherwise 500s
# every poll until its owner completes schema reconciliation.
# Off-thread so a locked store never delays the socket (Desktop
# ready-probe times out at 10s, GH-73083). NOT a daemon, and joined at
# shutdown: its sqlite connection must be closed by the thread that is
# stepping it. A daemon copy that outlived the lifespan had its
# connection closed from the main thread mid-probe (pytest's leaked-DB
# sweep) and segfaulted the interpreter. The worker is time-bounded by
# SessionDB's lock patience, so the join cannot hang shutdown.
eager_reconcile_thread = threading.Thread(
target=_eager_reconcile_own_session_db,
name="statedb-eager-reconcile",
)
eager_reconcile_thread.start()
# Import hermes_cli.gateway *before* the yield: on Windows + 3.11 the
# import holds the GIL, so run_in_executor still froze the loop 15-22s and
# the Desktop's 10s ready-probe timed out (GH-73083).
_warm_gateway_module()
# Snapshot the checkout revision so lazy-import paths (model picker) can
# refuse with "restart required" after `hermes update` replaced the code
# (#86207); the update flow does not reliably restart the dashboard.
from gateway.code_skew import record_boot_fingerprint
record_boot_fingerprint()
# Hosted Bot rooms belong to the backend process. Recovery may need a
# contended state.db migration, so keep it off the pre-yield path: Group
# Chat must degrade on its own rather than block every Desktop feature.
from tui_gateway import methods_groups as _hosted_groups
import tui_gateway.server
try:
tui_gateway.server.install_tui_message_injector()
except Exception:
logger.warning("TUI message injector did not install", exc_info=True)
hosted_room_start_cancel = threading.Event()
def _start_hosted_rooms() -> None:
try:
_hosted_groups.start_hosted_room_service()
except Exception:
logger.exception("Hosted Group Chat recovery failed during backend startup")
finally:
if hosted_room_start_cancel.is_set():
_hosted_groups.stop_hosted_room_service(timeout=1.0)
hosted_room_start_thread = threading.Thread(
target=_start_hosted_rooms,
daemon=True,
name="hosted-room-startup",
)
hosted_room_start_thread.start()
# Desktop-spawned backends fire cron jobs themselves, since the app has no
# gateway running the scheduler. Server `hermes dashboard` is unaffected —
# it relies on its own gateway.
from hermes_cli.process_identity import is_desktop_owned_backend
cron_stop: threading.Event | None = None
cron_thread: threading.Thread | None = None
desktop_owned = is_desktop_owned_backend()
if desktop_owned:
# Reap an orphaned gateway from an abnormal previous exit (reparented to
# launchd, still holding the platform WebSocket) before forking a fresh
# one that would race the same credential (#77276). Runs
# unconditionally; protection of a healthy standalone gateway lives
# INSIDE the reaper (registration probed with cleanup_stale=False).
# Startup grace: spare a gateway still claiming gateway.pid/lock (#122533).
try:
from hermes_cli.dashboard_procs import _REAP_MIN_AGE_SECONDS
from hermes_cli.gateway import _reap_unsupervised_gateway_orphans
_reap_unsupervised_gateway_orphans(min_age_s=_REAP_MIN_AGE_SECONDS)
except Exception:
logger.exception("Desktop startup: orphan gateway reap failed")
cron_stop = threading.Event()
cron_thread = threading.Thread(
target=_start_desktop_cron_ticker,
args=(cron_stop,),
daemon=True,
name="desktop-cron-ticker",
)
cron_thread.start()
# Reap idle/dead keep-alive PTY sessions (30-min TTL). The registry is process-wide and
# close_all() refuses new spawns, so a re-entered lifespan reopens it first.
PTY_REGISTRY.reopen()
pty_reaper_task = asyncio.create_task(run_reaper(PTY_REGISTRY))
# Periodic authenticated self-test feeding the ``dashboard`` component on /api/status.
selftest_task = asyncio.create_task(_dashboard_selftest_loop())
# Live auto-archive timer, independent of list requests.
auto_archive_task = asyncio.create_task(_auto_archive_ticker_loop())
# Managed local runtime (local_runtime.enabled): bring llama-server back so a
# restart doesn't strand a llamacpp main model. Off-thread and best-effort;
# failure falls back to cloud providers like a cold start. Server only —
# models load on first inference (an empty router holds no VRAM).
def _boot_local_runtime():
try:
from hermes_cli.config import load_config
from hermes_cli.local_runtime.bootstrap import ensure_local_runtime
ensure_local_runtime(load_config())
except Exception as exc:
logging.getLogger(__name__).warning("local runtime boot failed: %s", exc)
threading.Thread(target=_boot_local_runtime, daemon=True, name="local-runtime-boot").start()
# Records who carries inference for setup.status / free_tier.status and broadcasts `setup.ready`
# (no identity is created at boot). Off-thread; the desktop's first setup.status waits on the
# record (bounded) instead.
from hermes_cli.free_tier_bootstrap import start_background_bootstrap
start_background_bootstrap()
try:
yield
finally:
try:
tui_gateway.server.clear_tui_message_injector()
except Exception:
logger.debug("TUI message injector clear skipped", exc_info=True)
hosted_room_start_cancel.set()
_hosted_groups.stop_hosted_room_service(timeout=5.0)
hosted_room_start_thread.join(timeout=1.0)
if cron_stop is not None:
cron_stop.set()
pty_reaper_task.cancel()
selftest_task.cancel()
auto_archive_task.cancel()
eager_reconcile_thread.join()
await PTY_REGISTRY.close_all()
# PtySession.close() removes markers owned by live registry sessions.
# This second pass cleans any channel markers left in app state,
# including stale paths from sessions reaped earlier.
_unlink_pty_markers(app)
# Stop the managed llama-server with its parent (an orphan pins VRAM).
try:
from hermes_cli.local_runtime.bootstrap import shutdown_local_runtime
shutdown_local_runtime()
except Exception:
# Shutdown boundary: a supervisor that fails to stop must not skip the
# managed-gateway teardown below; log it so the orphan is diagnosable.
logger.warning("managed local runtime shutdown failed", exc_info=True)
if desktop_owned:
_terminate_desktop_managed_gateway()
@asynccontextmanager
async def http_lifespan(app: FastAPI):
"""The gateway already owns schema, rooms, scheduler and model services."""
from contextlib import AsyncExitStack
from hermes_cli.web_server import _dashboard_selftest_loop
from hermes_cli.web_server_chat import PTY_REGISTRY
from hermes_cli.web_server_surface import policy
app.state.event_channels = {}
app.state.event_lock = asyncio.Lock()
app.state.pty_active_session_files = {}
app.state.chat_argv_lock = asyncio.Lock()
PTY_REGISTRY.reopen() # a prior lifespan's close_all() refuses spawns until reopened
tasks = [
asyncio.create_task(run_reaper(PTY_REGISTRY), name="gateway-api-pty-reaper"),
asyncio.create_task(_dashboard_selftest_loop(), name="gateway-api-selftest"),
]
try:
async with AsyncExitStack() as surface:
if policy(app.state).host_terminal:
from hermes_cli.web_host_terminal_sessions import host_terminal_lifespan
await surface.enter_async_context(host_terminal_lifespan(app))
if policy(app.state).gateway_connections:
from hermes_cli import gateway_connections_ssh
from hermes_cli.gateway_connections_watch import watch_store
# Registered first so it runs last: the watcher stops, then every SSH tunnel this
# listener opened (and the remote serve it owns) is closed, never orphaned.
surface.push_async_callback(asyncio.to_thread, gateway_connections_ssh.close_all)
await surface.enter_async_context(watch_store())
yield
finally:
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
await PTY_REGISTRY.close_all()
_unlink_pty_markers(app)
@asynccontextmanager
async def app_lifespan(app: FastAPI):
lifespan = http_lifespan if getattr(app.state, "gateway_runner", None) is not None else standalone_lifespan
async with lifespan(app):
yield