mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 03:12:19 +03:00
feat(bot-desktop): memory gate and idle auto-stop for small hosts
Hosted instances come in 4 GB and 8 GB packages; a headed desktop plus the browser a human opens during a takeover costs 1.1-1.5 GB on top of the gateway (measured in the official image: gateway 304 MiB, +Xvnc/Xfce 520, +Chromium on one page 1,073). Left unguarded, the loser of that squeeze is Chromium mid-login or the gateway itself. - runtime.start() refuses while the host, or its cgroup when tighter, has less than bot_desktop.min_free_memory_mb free (default 1536); status() carries `blocker` plus memory_available_mb/memory_limit_mb so the pane shows the reason in place of Start and the CLI prints it. A running screen is never reported blocked: the gate guards the allocation. - A screen nobody used for bot_desktop.idle_stop_minutes (default 30) is stopped by the gateway's display watcher and comes back on the next use. "Use" = a computer_use dispatch, a browser/cua spawn onto the screen, an attached viewer (stamped from the RFB bridge at most once a minute), or start itself; a human holding the lease is never timed out. Both knobs are 0-disableable; the contract gains the three status fields.
This commit is contained in:
@@ -344,8 +344,8 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) {
|
||||
<div className="grid min-h-48 place-items-center p-6 text-center">
|
||||
<div className="flex flex-col items-center gap-2">
|
||||
<div className="text-sm font-medium">{t.screen.stoppedTitle}</div>
|
||||
<div className="text-xs text-muted-foreground">{t.screen.stoppedBody}</div>
|
||||
<Button disabled={busy} onClick={() => void start()} size="sm">
|
||||
<div className="text-xs text-muted-foreground">{status.blocker ?? t.screen.stoppedBody}</div>
|
||||
<Button disabled={busy || Boolean(status.blocker)} onClick={() => void start()} size="sm">
|
||||
{busy ? <GlyphSpinner /> : <Codicon name="play" />}
|
||||
{t.screen.start}
|
||||
</Button>
|
||||
|
||||
@@ -953,6 +953,9 @@ export interface DisplayStatus {
|
||||
geometry: string
|
||||
install_command?: string | null
|
||||
browser?: string | null
|
||||
blocker?: string | null
|
||||
memory_available_mb?: number | null
|
||||
memory_limit_mb?: number | null
|
||||
lease: DisplayLease
|
||||
profile_key: string
|
||||
}
|
||||
@@ -987,6 +990,9 @@ export interface DisplayStopResult {
|
||||
geometry: string
|
||||
install_command?: string | null
|
||||
browser?: string | null
|
||||
blocker?: string | null
|
||||
memory_available_mb?: number | null
|
||||
memory_limit_mb?: number | null
|
||||
lease: DisplayLease
|
||||
profile_key: string
|
||||
stopped: boolean
|
||||
@@ -1007,6 +1013,9 @@ export interface DisplayObserveResult {
|
||||
geometry: string
|
||||
install_command?: string | null
|
||||
browser?: string | null
|
||||
blocker?: string | null
|
||||
memory_available_mb?: number | null
|
||||
memory_limit_mb?: number | null
|
||||
lease: DisplayLease
|
||||
profile_key: string
|
||||
ticket: string
|
||||
@@ -3905,6 +3914,9 @@ export interface DisplayStatusPayload {
|
||||
geometry: string
|
||||
install_command?: string | null
|
||||
browser?: string | null
|
||||
blocker?: string | null
|
||||
memory_available_mb?: number | null
|
||||
memory_limit_mb?: number | null
|
||||
lease: DisplayLease
|
||||
profile_key: string
|
||||
}
|
||||
|
||||
@@ -10738,6 +10738,42 @@
|
||||
"default": null,
|
||||
"title": "Browser"
|
||||
},
|
||||
"blocker": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Blocker"
|
||||
},
|
||||
"memory_available_mb": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Memory Available Mb"
|
||||
},
|
||||
"memory_limit_mb": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Memory Limit Mb"
|
||||
},
|
||||
"lease": {
|
||||
"$ref": "#/components/schemas/DisplayLease"
|
||||
},
|
||||
@@ -10865,6 +10901,42 @@
|
||||
"default": null,
|
||||
"title": "Browser"
|
||||
},
|
||||
"blocker": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Blocker"
|
||||
},
|
||||
"memory_available_mb": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Memory Available Mb"
|
||||
},
|
||||
"memory_limit_mb": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Memory Limit Mb"
|
||||
},
|
||||
"lease": {
|
||||
"$ref": "#/components/schemas/DisplayLease"
|
||||
},
|
||||
@@ -10976,6 +11048,42 @@
|
||||
"default": null,
|
||||
"title": "Browser"
|
||||
},
|
||||
"blocker": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Blocker"
|
||||
},
|
||||
"memory_available_mb": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Memory Available Mb"
|
||||
},
|
||||
"memory_limit_mb": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Memory Limit Mb"
|
||||
},
|
||||
"lease": {
|
||||
"$ref": "#/components/schemas/DisplayLease"
|
||||
},
|
||||
@@ -11118,6 +11226,42 @@
|
||||
"default": null,
|
||||
"title": "Browser"
|
||||
},
|
||||
"blocker": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Blocker"
|
||||
},
|
||||
"memory_available_mb": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Memory Available Mb"
|
||||
},
|
||||
"memory_limit_mb": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Memory Limit Mb"
|
||||
},
|
||||
"lease": {
|
||||
"$ref": "#/components/schemas/DisplayLease"
|
||||
},
|
||||
|
||||
@@ -2281,6 +2281,14 @@ DEFAULT_CONFIG = {
|
||||
# host. Off by default so installing TigerVNC for other reasons never yields a screen nobody asked
|
||||
# for; Hermes Desktop's Screen pane offers Start and this toggle.
|
||||
"auto_start": False,
|
||||
# Refuse to start the screen while the host (or its container cgroup) has less than this free.
|
||||
# Xvnc + Xfce idle at ~220 MB and a takeover's browser adds 0.5-1 GB; on a small instance the
|
||||
# loser is Chromium mid-login or the gateway itself. 0 disables the check.
|
||||
"min_free_memory_mb": 1536,
|
||||
# Stop a screen nobody has used (no computer_use action, browser spawn, viewer or takeover) for this
|
||||
# long; it restarts on the next use. Idle Xvnc + Xfce hold ~220 MB, an abandoned browser far more.
|
||||
# 0 keeps screens up until stopped.
|
||||
"idle_stop_minutes": 30,
|
||||
},
|
||||
"computer_use": {
|
||||
# cua-driver's upstream PostHog telemetry defaults ON; Hermes sets
|
||||
|
||||
@@ -29,6 +29,9 @@ def _screen_status(args) -> int:
|
||||
print(f" control: {who} rfb socket: {st.socket}")
|
||||
print(" View it: Hermes Desktop → Bots → this bot → Screen")
|
||||
return 0
|
||||
if st.blocker:
|
||||
print(f"Bot Desktop [{st.profile}]: installed, not running. {st.blocker}")
|
||||
return 1
|
||||
print(f"Bot Desktop [{st.profile}]: installed, not running. Start: hermes computer-use screen start")
|
||||
return 1
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ from hermes_cli.web_server_chat import _ws_request_is_allowed
|
||||
_log = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
_ACTIVITY_STAMP_S = 60.0
|
||||
_READ_CHUNK = 64 * 1024
|
||||
_CLOSE_CONTROL_TAKEN = 4000
|
||||
_CLEAN_CLOSE = frozenset({1000, 1001})
|
||||
@@ -140,12 +141,25 @@ async def _bridge(ws: WebSocket, info: dict) -> None:
|
||||
rfb_filter = RfbClientFilter(_may_send_input)
|
||||
|
||||
viewer_closed = asyncio.Event()
|
||||
activity_file = Path(profile_home) / "bot-desktop" / "activity"
|
||||
stamped = {"at": 0.0}
|
||||
|
||||
def _stamp_activity() -> None:
|
||||
# An attached viewer is use: the idle auto-stop must not take a screen someone is watching.
|
||||
if loop.time() - stamped["at"] < _ACTIVITY_STAMP_S:
|
||||
return
|
||||
stamped["at"] = loop.time()
|
||||
try:
|
||||
activity_file.touch()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
async def rfb_to_ws() -> None:
|
||||
while True:
|
||||
chunk = await reader.read(_READ_CHUNK)
|
||||
if not chunk:
|
||||
return
|
||||
_stamp_activity()
|
||||
await ws.send_bytes(chunk) # awaiting the send is the backpressure toward Xvnc
|
||||
|
||||
async def ws_to_rfb() -> None:
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Bot Desktop idle auto-stop: a screen nobody used past ``bot_desktop.idle_stop_minutes`` is stopped,
|
||||
one a human holds or that saw recent use is not."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
from tools.bot_desktop import lease, runtime
|
||||
|
||||
|
||||
def _running(monkeypatch, tmp_path):
|
||||
sd = tmp_path / "bd"
|
||||
sd.mkdir()
|
||||
(sd / "env").write_text("DISPLAY=:20\n")
|
||||
monkeypatch.setattr(runtime, "state_dir", lambda: sd)
|
||||
monkeypatch.setattr(runtime, "_launcher_pid", lambda: 4242)
|
||||
monkeypatch.setattr(runtime, "idle_stop_seconds", lambda: 600.0)
|
||||
stops = []
|
||||
monkeypatch.setattr(runtime, "stop", lambda: stops.append(True) or True)
|
||||
return sd, stops
|
||||
|
||||
|
||||
def _age(path, seconds):
|
||||
old = time.time() - seconds
|
||||
os.utime(path, (old, old))
|
||||
|
||||
|
||||
def test_idle_screen_is_stopped_but_a_used_or_held_one_is_not(tmp_path, monkeypatch):
|
||||
sd, stops = _running(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(lease, "get", lambda profile_key=None: lease.Lease(holder=lease.AGENT))
|
||||
|
||||
_age(sd / "env", 5) # just started, never stamped: ages from publish
|
||||
assert runtime.stop_if_idle() is False and stops == []
|
||||
|
||||
_age(sd / "env", 3600)
|
||||
runtime.touch_activity() # a computer_use action / browser spawn a moment ago
|
||||
assert runtime.stop_if_idle() is False and stops == []
|
||||
|
||||
_age(sd / "activity", 3600)
|
||||
monkeypatch.setattr(lease, "get", lambda profile_key=None: lease.Lease(holder=lease.HUMAN, viewer_id="v1"))
|
||||
assert runtime.stop_if_idle() is False and stops == [], "a human mid-login is never timed out"
|
||||
|
||||
monkeypatch.setattr(lease, "get", lambda profile_key=None: lease.Lease(holder=lease.AGENT))
|
||||
assert runtime.stop_if_idle() is True and stops == [True]
|
||||
|
||||
|
||||
def test_idle_stop_disabled_by_zero(tmp_path, monkeypatch):
|
||||
sd, stops = _running(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(runtime, "idle_stop_seconds", lambda: 0.0)
|
||||
_age(sd / "env", 10 ** 6)
|
||||
assert runtime.stop_if_idle() is False and stops == []
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Bot Desktop memory gate: the screen is refused, with a reason clients can show, while the host
|
||||
cannot spare ``bot_desktop.min_free_memory_mb``; a host we cannot read is never treated as small."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.bot_desktop import resources, runtime
|
||||
|
||||
|
||||
def test_start_refuses_and_status_explains_when_memory_is_short(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path / "bd")
|
||||
monkeypatch.setattr(runtime, "missing_binaries", lambda: [])
|
||||
monkeypatch.setattr(runtime, "_launcher_pid", lambda: None)
|
||||
monkeypatch.setattr(resources, "min_free_mb", lambda: 1536)
|
||||
monkeypatch.setattr(resources, "memory_info", lambda: resources.MemoryInfo(available_mb=900, limit_mb=4096))
|
||||
spawned = []
|
||||
monkeypatch.setattr(runtime, "_spawn_and_wait", lambda *a, **k: spawned.append(a))
|
||||
|
||||
st = runtime.status()
|
||||
assert st.running is False and st.blocker and "900 MB" in st.blocker and "4096 MB" in st.blocker
|
||||
assert st.memory_available_mb == 900 and st.memory_limit_mb == 4096
|
||||
with pytest.raises(RuntimeError, match="Not enough free memory"):
|
||||
runtime.start()
|
||||
assert spawned == [], "the launcher must not be spawned on a host that cannot hold it"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("info", [
|
||||
resources.MemoryInfo(available_mb=None, limit_mb=None), # unreadable: not evidence of a small host
|
||||
resources.MemoryInfo(available_mb=1536, limit_mb=2048), # exactly the threshold is enough
|
||||
])
|
||||
def test_unknown_or_sufficient_memory_never_blocks(monkeypatch, info):
|
||||
monkeypatch.setattr(resources, "min_free_mb", lambda: 1536)
|
||||
assert resources.memory_blocker(info) is None
|
||||
|
||||
|
||||
def test_running_screen_is_not_reported_blocked_by_later_pressure(tmp_path, monkeypatch):
|
||||
"""The gate guards the allocation; once the desktop is up, memory pressure is the browser's problem,
|
||||
not a reason to tell the pane its running screen is blocked."""
|
||||
monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path / "bd")
|
||||
monkeypatch.setattr(runtime, "missing_binaries", lambda: [])
|
||||
monkeypatch.setattr(runtime, "_launcher_pid", lambda: 4242)
|
||||
monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":20"})
|
||||
monkeypatch.setattr(resources, "min_free_mb", lambda: 1536)
|
||||
monkeypatch.setattr(resources, "memory_info", lambda: resources.MemoryInfo(available_mb=100, limit_mb=4096))
|
||||
st = runtime.status()
|
||||
assert st.running is True and st.blocker is None
|
||||
|
||||
|
||||
def test_memory_info_takes_the_tighter_of_cgroup_and_host(tmp_path, monkeypatch):
|
||||
"""A 16 GB cgroup limit on a 4 GB box is still a 4 GB box; a 2 GB limit on a 64 GB box is 2 GB."""
|
||||
v2 = tmp_path / "cg"; v2.mkdir()
|
||||
monkeypatch.setattr(resources, "_CGROUP_V2", v2)
|
||||
monkeypatch.setattr(resources, "_CGROUP_V1", tmp_path / "nope")
|
||||
meminfo = tmp_path / "meminfo"
|
||||
monkeypatch.setattr(resources, "_MEMINFO", meminfo)
|
||||
meminfo.write_text("MemTotal: 4194304 kB\nMemAvailable: 3145728 kB\n")
|
||||
(v2 / "memory.max").write_text(str(16 * 1024 ** 3)); (v2 / "memory.current").write_text(str(1024 ** 3))
|
||||
assert resources.memory_info().available_mb == 3072
|
||||
(v2 / "memory.max").write_text(str(2 * 1024 ** 3))
|
||||
info = resources.memory_info()
|
||||
assert info.available_mb == 1024 and info.limit_mb == 2048
|
||||
(v2 / "memory.max").write_text("max") # no limit: host numbers
|
||||
assert resources.memory_info() == resources.MemoryInfo(available_mb=3072, limit_mb=4096)
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Host resources the screen needs before it starts: memory.
|
||||
|
||||
A headed desktop is a big allocation relative to a small gateway host: Xvnc + Xfce idle at ~220 MiB
|
||||
and the browser a human opens during a takeover adds 500 MiB to 1 GiB (measured in the official image:
|
||||
gateway 304 MiB → +Xfce 520 MiB → +Chromium on one page 1,073 MiB). On a 4 GB container the loser
|
||||
of that squeeze is Chromium, killed mid-login, or the gateway itself. So ``runtime.start()`` refuses
|
||||
while the host cannot spare ``bot_desktop.min_free_memory_mb`` and ``display.status`` says why, which
|
||||
the pane shows in place of the Start button.
|
||||
|
||||
"Available" is the tighter of two numbers: what the cgroup still allows (a container limit; v2
|
||||
``memory.max``/``memory.current``, v1 ``memory.limit_in_bytes``/``memory.usage_in_bytes``) and what
|
||||
the kernel reports free for the whole machine (``MemAvailable``) — a limit of 8 GB on a 4 GB host is
|
||||
not 8 GB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
_CGROUP_V2 = Path("/sys/fs/cgroup")
|
||||
_CGROUP_V1 = Path("/sys/fs/cgroup/memory")
|
||||
_MEMINFO = Path("/proc/meminfo")
|
||||
_MIB = 1024 * 1024
|
||||
DEFAULT_MIN_FREE_MB = 1536
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryInfo:
|
||||
available_mb: Optional[int] # None when nothing readable (non-Linux, odd sandboxes)
|
||||
limit_mb: Optional[int] # the cgroup limit when there is one, else the machine's MemTotal
|
||||
|
||||
|
||||
def _read_int(path: Path) -> Optional[int]:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8").strip()
|
||||
except OSError:
|
||||
return None
|
||||
return int(text) if text.isdigit() else None # "max" → None
|
||||
|
||||
|
||||
def _meminfo() -> dict[str, int]:
|
||||
try:
|
||||
lines = _MEMINFO.read_text(encoding="utf-8").splitlines()
|
||||
except OSError:
|
||||
return {}
|
||||
out: dict[str, int] = {}
|
||||
for line in lines:
|
||||
key, _, rest = line.partition(":")
|
||||
parts = rest.split()
|
||||
if parts and parts[0].isdigit():
|
||||
out[key] = int(parts[0]) * 1024 # kB
|
||||
return out
|
||||
|
||||
|
||||
def _cgroup_limit_and_usage() -> tuple[Optional[int], Optional[int]]:
|
||||
limit = _read_int(_CGROUP_V2 / "memory.max")
|
||||
usage = _read_int(_CGROUP_V2 / "memory.current")
|
||||
if limit is None and usage is None:
|
||||
limit = _read_int(_CGROUP_V1 / "memory.limit_in_bytes")
|
||||
usage = _read_int(_CGROUP_V1 / "memory.usage_in_bytes")
|
||||
if limit is not None and limit >= 1 << 60: # v1 "unlimited" is a huge sentinel
|
||||
limit = None
|
||||
return limit, usage
|
||||
|
||||
|
||||
def memory_info() -> MemoryInfo:
|
||||
info = _meminfo()
|
||||
host_available = info.get("MemAvailable")
|
||||
host_total = info.get("MemTotal")
|
||||
limit, usage = _cgroup_limit_and_usage()
|
||||
candidates = [x for x in (host_available, (limit - usage) if limit is not None and usage is not None else None)
|
||||
if x is not None]
|
||||
available = min(candidates) if candidates else None
|
||||
limit_bytes = limit if limit is not None else host_total
|
||||
return MemoryInfo(available_mb=None if available is None else max(0, available // _MIB),
|
||||
limit_mb=None if limit_bytes is None else limit_bytes // _MIB)
|
||||
|
||||
|
||||
def min_free_mb() -> int:
|
||||
from hermes_cli.config import load_config_readonly
|
||||
cfg = load_config_readonly().get("bot_desktop") or {}
|
||||
try:
|
||||
return max(0, int(cfg.get("min_free_memory_mb", DEFAULT_MIN_FREE_MB)))
|
||||
except (TypeError, ValueError):
|
||||
return DEFAULT_MIN_FREE_MB
|
||||
|
||||
|
||||
def memory_blocker(info: Optional[MemoryInfo] = None) -> Optional[str]:
|
||||
"""Why the screen must not start now, or None. Unknown memory is not a blocker: a host we cannot
|
||||
read is not a host we know to be small."""
|
||||
need = min_free_mb()
|
||||
if need == 0:
|
||||
return None
|
||||
info = info or memory_info()
|
||||
if info.available_mb is None or info.available_mb >= need:
|
||||
return None
|
||||
limit = f" of {info.limit_mb} MB" if info.limit_mb else ""
|
||||
return (f"Not enough free memory for a desktop: {info.available_mb} MB available{limit}, "
|
||||
f"{need} MB needed (bot_desktop.min_free_memory_mb)")
|
||||
@@ -123,6 +123,9 @@ class DesktopStatus:
|
||||
geometry: str
|
||||
install_command: Optional[str]
|
||||
browser: Optional[str] # headed Chromium the dock's Browser icon and agent-browser share; None = no headed browser
|
||||
blocker: Optional[str] = None # why start() would refuse right now (memory); None = may start
|
||||
memory_available_mb: Optional[int] = None
|
||||
memory_limit_mb: Optional[int] = None
|
||||
|
||||
def as_dict(self) -> Dict[str, object]:
|
||||
return dict(self.__dict__)
|
||||
@@ -303,6 +306,7 @@ def desktop_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]:
|
||||
env = dict(os.environ if base_env is None else base_env)
|
||||
published = published_env()
|
||||
if published:
|
||||
touch_activity() # a browser / cua-driver spawn is the agent using its screen
|
||||
env.update(published)
|
||||
env.pop("WAYLAND_DISPLAY", None) # X11 desktop; a leaked Wayland socket flips GTK/Chromium backends
|
||||
from tools.bot_desktop.browser import env_for_agent
|
||||
@@ -315,7 +319,10 @@ def ensure_started_for_tool() -> None:
|
||||
host that has NO display and the packages installed gets its screen started on first use, so a headless
|
||||
gateway works the first time instead of answering "no DISPLAY is set". Failure is not an error here;
|
||||
the tool's own "no display" diagnosis is the right message then."""
|
||||
if published_env() or not _should_auto_start(os.environ):
|
||||
if published_env():
|
||||
touch_activity()
|
||||
return
|
||||
if not _should_auto_start(os.environ):
|
||||
return
|
||||
try:
|
||||
start()
|
||||
@@ -333,6 +340,60 @@ def _should_auto_start(env: Dict[str, str]) -> bool:
|
||||
return bool(cfg.get("auto_start", False))
|
||||
|
||||
|
||||
# ---- idle auto-stop -------------------------------------------------------------------------------
|
||||
# A desktop nobody is using still holds ~220 MiB (plus whatever browser was left open). Every use —
|
||||
# a computer_use action, a browser spawn onto the screen, a viewer attached, a human takeover — stamps
|
||||
# ``activity``; the gateway's display watcher stops a screen idle past ``bot_desktop.idle_stop_minutes``
|
||||
# unless a human holds it. The next use starts it again (auto_start or the pane's Start).
|
||||
DEFAULT_IDLE_STOP_MINUTES = 30
|
||||
|
||||
|
||||
def touch_activity() -> None:
|
||||
path = state_dir() / "activity"
|
||||
try:
|
||||
path.touch()
|
||||
os.utime(path, None)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def idle_seconds() -> Optional[float]:
|
||||
"""Seconds since the last stamped use; None when the screen never recorded one (falls back to the
|
||||
env file's publish time so a screen started and then forgotten still ages)."""
|
||||
for name in ("activity", "env"):
|
||||
try:
|
||||
return max(0.0, time.time() - (state_dir() / name).stat().st_mtime)
|
||||
except OSError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def idle_stop_seconds() -> float:
|
||||
from hermes_cli.config import load_config_readonly
|
||||
cfg = load_config_readonly().get("bot_desktop") or {}
|
||||
try:
|
||||
minutes = float(cfg.get("idle_stop_minutes", DEFAULT_IDLE_STOP_MINUTES))
|
||||
except (TypeError, ValueError):
|
||||
minutes = DEFAULT_IDLE_STOP_MINUTES
|
||||
return max(0.0, minutes) * 60
|
||||
|
||||
|
||||
def stop_if_idle() -> bool:
|
||||
"""Stop this profile's screen when it has been idle past the limit and no human holds it. True when
|
||||
it was stopped."""
|
||||
limit = idle_stop_seconds()
|
||||
if limit <= 0 or _launcher_pid() is None:
|
||||
return False
|
||||
idle = idle_seconds()
|
||||
if idle is None or idle < limit:
|
||||
return False
|
||||
from tools.bot_desktop import lease as _bd_lease
|
||||
if _bd_lease.get().holder == _bd_lease.HUMAN:
|
||||
return False
|
||||
logger.info("Bot Desktop for profile %s idle for %.0f min; stopping", _profile_name(), idle / 60)
|
||||
return stop()
|
||||
|
||||
|
||||
def published_env() -> Dict[str, str]:
|
||||
"""Variables the launcher wrote once Xfce's private bus existed; empty when the desktop is down."""
|
||||
if _launcher_pid() is None:
|
||||
@@ -361,21 +422,28 @@ def geometry() -> str:
|
||||
|
||||
def status(profile: Optional[str] = None) -> DesktopStatus:
|
||||
from tools.bot_desktop import browser as _bd_browser
|
||||
from tools.bot_desktop import resources
|
||||
missing: list[str] = missing_binaries() if is_supported_host() else list(REQUIRED_BINARIES)
|
||||
pid = _launcher_pid()
|
||||
env = published_env()
|
||||
running = pid is not None and bool(env.get("DISPLAY"))
|
||||
mem = resources.memory_info() if is_supported_host() else resources.MemoryInfo(None, None)
|
||||
return DesktopStatus(
|
||||
profile=profile or _profile_name(),
|
||||
supported=is_supported_host(),
|
||||
installed=not missing,
|
||||
missing=missing,
|
||||
running=pid is not None and bool(env.get("DISPLAY")),
|
||||
running=running,
|
||||
pid=pid,
|
||||
display=env.get("DISPLAY"),
|
||||
socket=str(rfb_socket_path()) if rfb_socket_path() else None,
|
||||
geometry=geometry(),
|
||||
install_command=install_command() if missing else None,
|
||||
browser=_bd_browser.executable() if is_supported_host() else None,
|
||||
# A running screen is never "blocked": the check guards the allocation, not the session.
|
||||
blocker=None if running or missing or not is_supported_host() else resources.memory_blocker(mem),
|
||||
memory_available_mb=mem.available_mb,
|
||||
memory_limit_mb=mem.limit_mb,
|
||||
)
|
||||
|
||||
|
||||
@@ -408,6 +476,9 @@ def start(*, wait_seconds: float = 15.0) -> DesktopStatus:
|
||||
with _flocked(sd / "start.lock"):
|
||||
if _launcher_pid() is not None and published_env().get("DISPLAY"):
|
||||
return status()
|
||||
from tools.bot_desktop import resources
|
||||
if (blocker := resources.memory_blocker()) is not None:
|
||||
raise RuntimeError(blocker)
|
||||
if _launcher_pid() is None:
|
||||
_reap_orphaned_server(sd)
|
||||
_ALLOC_LOCK.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
@@ -457,6 +528,7 @@ def _spawn_and_wait(sd: Path, wait_seconds: float) -> DesktopStatus:
|
||||
raise RuntimeError(f"Bot Desktop launcher exited with {proc.returncode}:\n{tail}")
|
||||
if env_file.exists() and (sd / "rfb.sock").exists():
|
||||
logger.info("Bot Desktop for profile %s up on :%s", _profile_name(), num)
|
||||
touch_activity()
|
||||
return status()
|
||||
time.sleep(0.1)
|
||||
# Giving up must take the launch down: left alone, the launcher publishes DISPLAY and rfb.sock a moment
|
||||
@@ -488,6 +560,6 @@ def _stop_locked(sd: Path) -> bool:
|
||||
return reaped
|
||||
# The launcher runs in its own session; killing the group takes Xvnc, dbus and Xfce with it.
|
||||
_kill_group_then_wait(pid, pid, grace=5.0)
|
||||
(sd / "launcher.pid").unlink(missing_ok=True)
|
||||
(sd / "env").unlink(missing_ok=True)
|
||||
for name in ("launcher.pid", "env", "activity"):
|
||||
(sd / name).unlink(missing_ok=True)
|
||||
return True
|
||||
|
||||
@@ -48,6 +48,9 @@ class DisplayStatus(Result):
|
||||
geometry: str
|
||||
install_command: str | None = None
|
||||
browser: str | None = None
|
||||
blocker: str | None = None # why display.start would refuse now (host memory); the pane shows it instead of Start
|
||||
memory_available_mb: int | None = None
|
||||
memory_limit_mb: int | None = None
|
||||
lease: DisplayLease
|
||||
profile_key: str
|
||||
|
||||
|
||||
@@ -70,6 +70,27 @@ def _poll_runtime_files() -> None:
|
||||
_broadcast_global_event("display.status", payload)
|
||||
|
||||
|
||||
_IDLE_CHECK_S = 30.0
|
||||
_last_idle_check = 0.0
|
||||
|
||||
|
||||
def _poll_idle_screens() -> None:
|
||||
"""Every _IDLE_CHECK_S: stop screens idle past ``bot_desktop.idle_stop_minutes`` (runtime.stop_if_idle
|
||||
holds the rules). The stop moves the runtime files, so _poll_runtime_files broadcasts display.status."""
|
||||
global _last_idle_check
|
||||
if time.monotonic() - _last_idle_check < _IDLE_CHECK_S:
|
||||
return
|
||||
_last_idle_check = time.monotonic()
|
||||
from hermes_constants import reset_hermes_home_override, set_hermes_home_override
|
||||
from tools.bot_desktop import runtime as _bd_runtime
|
||||
for home in _watched_lease_homes():
|
||||
token = set_hermes_home_override(home)
|
||||
try:
|
||||
_bd_runtime.stop_if_idle()
|
||||
finally:
|
||||
reset_hermes_home_override(token)
|
||||
|
||||
|
||||
def _poll_lease_files() -> None:
|
||||
"""One pass: read a home's lease only when its file mtime moved; broadcast when the epoch did."""
|
||||
from hermes_constants import hermes_home_key
|
||||
@@ -112,6 +133,7 @@ def _ensure_lease_watcher() -> None:
|
||||
while True:
|
||||
try:
|
||||
_poll_lease_files()
|
||||
_poll_idle_screens()
|
||||
_poll_runtime_files()
|
||||
except Exception: # noqa: BLE001 - a torn read must not kill the watcher
|
||||
logger.debug("lease watcher poll failed", exc_info=True)
|
||||
|
||||
Reference in New Issue
Block a user