From 5f02a8c19e869541cb4f081d64ad7198e2bf2d28 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:48:59 -0700 Subject: [PATCH] fix(config): a same-size config.yaml rewrite in one timestamp tick is no longer served stale Every config parse cache (load_config, read_raw_config, load_user_config_effective, read_config_doc_readonly, the managed overlay) keyed on the stat tuple (mtime_ns, size, ino, ctime_ns). On kernels without multigrain timestamps a same-length in-place rewrite (port 3 -> 5, k-old -> k-new) in the tick of an earlier read keeps all four, so the older parse was served for the rest of the process: a rotated api_key never reached a follow-profile session and the route stayed on the old endpoint. That is the CI flake in test_rotated_key_reaches_follow_and_held_pick_on_a_fresh_owner_and_routes_are_never_mixed (got port 3, expected 5). Caches now key on config_cache_version(): the backend's version plus a content digest while the mtime is racy (younger than utils.RACY_MTIME_WINDOW_NS, the same "racily clean" rule the change watcher and roster already use), read before the parse it guards. Settled files cost no extra read; config_version() itself is unchanged, so change watchers and the parse-failure record see no spurious change when a file settles. --- hermes_cli/commands_completion.py | 4 +- hermes_cli/config.py | 29 ++++---- hermes_cli/config_backend.py | 29 +++++++- hermes_cli/managed_scope.py | 8 +-- hermes_cli/stale_modules.py | 2 +- .../test_config_same_tick_rewrite.py | 67 +++++++++++++++++++ utils.py | 15 ++++- .../docs/developer-guide/config-backend.md | 9 ++- 8 files changed, 136 insertions(+), 27 deletions(-) create mode 100644 tests/hermes_cli/test_config_same_tick_rewrite.py diff --git a/hermes_cli/commands_completion.py b/hermes_cli/commands_completion.py index 3a0990d43e..15e77b4998 100644 --- a/hermes_cli/commands_completion.py +++ b/hermes_cli/commands_completion.py @@ -34,9 +34,9 @@ def _personalities_from_cli_config() -> dict[str, Any]: from hermes_cli.personality import available_personalities try: from hermes_cli.config import get_config_path - from hermes_cli.config_backend import config_version + from hermes_cli.config_backend import config_cache_version cfg_path = get_config_path() - sig = (str(cfg_path), *config_version(cfg_path)) + sig = (str(cfg_path), *config_cache_version(cfg_path)) except Exception: sig = (None, None, None, None, None) if _personalities_memo is None or _personalities_memo[0] != sig: diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 5fc93b38a1..8c7e8f8937 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -47,7 +47,7 @@ from hermes_constants import get_hermes_home, get_process_hermes_home from utils import atomic_replace, fast_safe_load, file_signature, mkstemp_beside from hermes_cli.config_issues import ConfigIssue, _issue from hermes_cli.config_backend import ( - config_exists, config_version, probe_config_readable, read_config_doc, + config_cache_version, config_exists, config_version, file_cache_version, probe_config_readable, read_config_doc, write_config_document) from hermes_cli.config_read_errors import ( _CONFIG_PARSE_FAILURES, _FIX_PERMS, _FIX_YAML, FailedConfigRead, _backups_dir_display, @@ -178,9 +178,9 @@ _LAST_EXPANDED_CONFIG_BY_PATH: dict[str, Any] = {} # _normalize_* + _expand_env_vars (~13 ms/call). save_config() + migrate_config() write via # the config writer seam, which produces a fresh inode, so stat() sees a new signature and the next load # repopulates automatically — no explicit invalidation hook. See #58514. -_LOAD_CONFIG_CACHE: dict[str, tuple[int, ...]] = {} -# path -> (mtime_ns, size, ino, ctime_ns, raw yaml dict) for read_raw_config() (no defaults merged in). -_RAW_CONFIG_CACHE: dict[str, tuple[int, ...]] = {} +_LOAD_CONFIG_CACHE: dict[str, tuple[Any, ...]] = {} +# path -> (mtime_ns, size, ino, ctime_ns, racy digest, raw yaml dict) for read_raw_config() (no defaults merged in). +_RAW_CONFIG_CACHE: dict[str, tuple[Any, ...]] = {} # Env var names written to .env that aren't in OPTIONAL_ENV_VARS (managed by setup/provider # flows directly). Also the set reload_env() may remove from os.environ. @@ -1835,7 +1835,7 @@ def _read_raw_config_impl(*, want_deepcopy: bool) -> dict[str, Any]: # every cached read for the duration). A lost race just falls through to the locked re-check. try: config_path = get_config_path() - cache_key = config_version(config_path) + cache_key = config_cache_version(config_path) hit = _raw_config_cache_hit(str(config_path), cache_key) if hit is not None: return copy.deepcopy(hit) if want_deepcopy else hit @@ -1845,7 +1845,7 @@ def _read_raw_config_impl(*, want_deepcopy: bool) -> dict[str, Any]: with _CONFIG_LOCK: config_path = get_config_path() try: - cache_key = config_version(config_path) + cache_key = config_cache_version(config_path) except FileNotFoundError: return {} except OSError as e: @@ -2139,23 +2139,22 @@ def apply_terminal_config_to_env( return target -def _load_config_cache_sig(config_path: Path) -> tuple[Optional[tuple[int, int, int, int]], Optional[tuple[int, ...]]]: - """Return ``(user_sig, cache_sig)`` for ``_LOAD_CONFIG_CACHE``. - The managed config file's signature is folded in ((0, 0, 0, 0) = none) so editing it invalidates +def _load_config_cache_sig(config_path: Path) -> tuple[Optional[tuple[Any, ...]], Optional[tuple[Any, ...]]]: + """Return ``(user_sig, cache_sig)`` for ``_LOAD_CONFIG_CACHE`` (racy-clean guarded, see ``config_cache_version``). + The managed config file's signature is folded in (zeros = none) so editing it invalidates the merged result. ``cache_sig`` is None only when neither file exists (nothing to cache on).""" try: - user_sig: Optional[tuple[Any, ...]] = config_version(config_path) + user_sig: Optional[tuple[Any, ...]] = config_cache_version(config_path) except FileNotFoundError: user_sig = None managed_dir = managed_scope.get_managed_dir() try: - mst = (managed_dir / "config.yaml").stat() if managed_dir else None # config-reader: ok — managed overlay, not a user layer - managed_sig = file_signature(mst) if mst else (0, 0, 0, 0) + managed_sig = file_cache_version(managed_dir / "config.yaml") if managed_dir else (0, 0, 0, 0, "") # config-reader: ok — managed overlay except OSError: - managed_sig = (0, 0, 0, 0) - if user_sig is None and managed_sig == (0, 0, 0, 0): + managed_sig = (0, 0, 0, 0, "") + if user_sig is None and managed_sig == (0, 0, 0, 0, ""): return None, None - return user_sig, (*(user_sig or (0, 0, 0, 0)), *managed_sig) + return user_sig, (*(user_sig or (0, 0, 0, 0, "")), *managed_sig) def _last_known_good_fallback(config_path: Path, path_key: str, cache_sig, exc: Exception) -> Optional[dict[str, Any]]: diff --git a/hermes_cli/config_backend.py b/hermes_cli/config_backend.py index b4b9303795..9136fe99a3 100644 --- a/hermes_cli/config_backend.py +++ b/hermes_cli/config_backend.py @@ -54,7 +54,11 @@ class ConfigBackend(Protocol): ... def version(self, home: Path) -> tuple[Any, ...]: - """Cheap cache signature; changes whenever the user layer does.""" + """Cheap change signature; changes whenever the user layer does.""" + ... + + def cache_version(self, home: Path) -> tuple[Any, ...]: + """``version`` plus whatever a cache needs to tell apart writes ``version`` cannot see.""" ... def exists(self, home: Path) -> bool: ... @@ -79,6 +83,12 @@ class FileBackend: from utils import file_signature return file_signature(st) + @classmethod + def cache_version_path(cls, path: Path) -> tuple[Any, ...]: + version = cls.version_path(path) + from utils import racy_content_digest + return (*version, racy_content_digest(path, version[0])) # version[0] is st_mtime_ns + @staticmethod def read_path(path: Path) -> Any: from utils import fast_safe_load @@ -112,6 +122,11 @@ class FileBackend: def version(self, home: Path) -> tuple[int, int, int, int]: return self.version_path(self.config_path(home)) + def cache_version(self, home: Path) -> tuple[Any, ...]: + version = self.version(home) + from utils import racy_content_digest + return (*version, racy_content_digest(self.config_path(home), version[0])) + def exists(self, home: Path) -> bool: return self.config_path(home).exists() @@ -173,6 +188,18 @@ def config_version(config_path: PathLike) -> tuple[Any, ...]: return backend.version(home) +def config_cache_version(config_path: PathLike) -> tuple[Any, ...]: + """Signature for a cache of a config file's parse: :func:`config_version` can stay equal across a + same-tick same-size rewrite, which would pin the older parse for the process lifetime.""" + backend, home = _route(config_path) + return backend.cache_version(home) + + +def file_cache_version(path: PathLike) -> tuple[Any, ...]: + """:func:`config_cache_version` of a file that is not a home's user layer (the managed overlay).""" + return FileBackend.cache_version_path(Path(path)) + + def config_version_or_none(config_path: PathLike) -> Optional[tuple[Any, ...]]: """:func:`config_version`, or None when the config is absent or cannot be stat'ed.""" try: diff --git a/hermes_cli/managed_scope.py b/hermes_cli/managed_scope.py index ea88affc66..3cc274f1e4 100644 --- a/hermes_cli/managed_scope.py +++ b/hermes_cli/managed_scope.py @@ -16,14 +16,14 @@ from pathlib import Path from typing import Dict, Optional -# Stale-module bridge: this module binds ``utils.file_signature`` at import time, so a fresh +# Stale-module bridge: this module binds ``utils`` names at import time, so a fresh # import in a post-pull updater process (pre-handoff purge keeps root modules cached) dies # unless the stale ``utils`` is dropped first. See hermes_cli.stale_modules. from hermes_cli.stale_modules import drop_stale_root_modules drop_stale_root_modules() -from utils import fast_safe_load, file_signature +from utils import fast_safe_load, file_signature, racy_content_digest logger = logging.getLogger(__name__) @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) _DEFAULT_MANAGED_DIR = Path("/etc/hermes") _CACHE_LOCK = threading.Lock() -# path_key -> (*file_signature, parsed) +# path_key -> (*file_signature, racy digest, parsed) _CONFIG_CACHE: dict[str, tuple] = {} _ENV_CACHE: dict[str, tuple] = {} @@ -77,7 +77,7 @@ def _cached_read(path: Path, cache: dict[str, tuple], parse): st = path.stat() except OSError: return None # absent - key = file_signature(st) + key = (*file_signature(st), racy_content_digest(path, st.st_mtime_ns)) path_key = str(path) with _CACHE_LOCK: hit = cache.get(path_key) diff --git a/hermes_cli/stale_modules.py b/hermes_cli/stale_modules.py index 2132da71ab..a5971a4d8f 100644 --- a/hermes_cli/stale_modules.py +++ b/hermes_cli/stale_modules.py @@ -25,7 +25,7 @@ from typing import Mapping, Sequence # exist on the on-disk copy after this release. Extend when a new root-level # symbol would otherwise break the pre-handoff upgrade path. _ROOT_MODULE_REQUIRED_ATTRS: dict[str, tuple[str, ...]] = { - "utils": ("file_signature",), + "utils": ("file_signature", "racy_content_digest"), } diff --git a/tests/hermes_cli/test_config_same_tick_rewrite.py b/tests/hermes_cli/test_config_same_tick_rewrite.py new file mode 100644 index 0000000000..d9b73e0050 --- /dev/null +++ b/tests/hermes_cli/test_config_same_tick_rewrite.py @@ -0,0 +1,67 @@ +"""Config caches see a same-size in-place rewrite made in the timestamp tick of the cached read. + +Kernels without multigrain timestamps (CI runners, older hosts) stamp mtime/ctime from a coarse +clock, so ``config.yaml`` rewritten in place to a same-length value (``port 3`` -> ``port 5``) in the +tick of an earlier read keeps mtime, ctime, size and inode: the stat signature every config cache +keys on is unchanged, and the older parse was served for the rest of the process. This host may +have fine-grained timestamps, so ``coarse_clock`` reports both writes in one frozen tick, which is +what such a kernel returns.""" +import json +import os +import time + +import pytest + + +def _write(path, port): + path.write_text(json.dumps({'model': {'provider': 'custom', 'base_url': f'http://127.0.0.1:{port}/v1'}})) + + +def _readers(): + from hermes_cli.config import load_config, load_config_readonly, read_raw_config + from hermes_cli.config_backend import read_config_doc_readonly + from hermes_cli.config_effective import load_user_config_effective + from hermes_cli.config import get_config_path + path = get_config_path() + return { + 'load_config': lambda: load_config(), + 'load_config_readonly': lambda: load_config_readonly(), + 'read_raw_config': lambda: read_raw_config(), + 'load_user_config_effective': lambda: load_user_config_effective(path), + 'read_config_doc_readonly': lambda: read_config_doc_readonly(path), + } + + +@pytest.fixture +def coarse_clock(monkeypatch): + """Every stat signature reports one frozen mtime/ctime tick, like a coarse filesystem clock.""" + import utils + real, tick = utils.file_signature, time.time_ns() + monkeypatch.setattr(utils, 'file_signature', lambda st: (tick, *real(st)[1:3], tick)) + + +def test_same_size_rewrite_in_one_tick_reaches_every_config_reader(coarse_clock): + from hermes_cli.config import get_config_path + path = get_config_path() + path.parent.mkdir(parents=True, exist_ok=True) + readers = _readers() + _write(path, 3) + assert {name: read()['model']['base_url'] for name, read in readers.items()} == dict.fromkeys( + readers, 'http://127.0.0.1:3/v1') + _write(path, 5) + assert {name: read()['model']['base_url'] for name, read in readers.items()} == dict.fromkeys( + readers, 'http://127.0.0.1:5/v1') + + +def test_an_unchanged_file_stays_one_parse_while_fresh_and_once_settled(coarse_clock): + from hermes_cli.config import get_config_path + path = get_config_path() + path.parent.mkdir(parents=True, exist_ok=True) + readonly = {k: v for k, v in _readers().items() if k in ('load_config_readonly', 'read_config_doc_readonly')} + _write(path, 3) + for read in readonly.values(): # inside the racy window: the content digest matches, still a hit + assert read() is read() + settled = time.time_ns() - 10_000_000_000 + os.utime(path, ns=(settled, settled)) + for read in readonly.values(): + assert read() is read() diff --git a/utils.py b/utils.py index f4b4867c47..6538220418 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,7 @@ """Shared utility functions for hermes-agent.""" import errno +import hashlib import json import logging import os @@ -60,6 +61,17 @@ def mtime_is_racy(mtime_ns: int) -> bool: return time.time_ns() - mtime_ns < RACY_MTIME_WINDOW_NS +def racy_content_digest(path: str | Path, mtime_ns: int) -> str: + """``""`` once ``mtime_ns`` has settled; while it is racy, a digest of the file's bytes. A cache + keyed on a stat signature appends it, because two same-size in-place writes in one timestamp tick + share mtime, ctime, size and inode (kernels without multigrain timestamps). Read before the parse + it guards: a write in between then costs one re-parse instead of pinning the older content.""" + if not mtime_is_racy(mtime_ns): + return "" + with open(path, "rb") as f: + return hashlib.blake2b(f.read(), digest_size=16).hexdigest() + + def _preserve_file_mode(path: Path) -> int | None: """Permission bits of *path* if it exists, else ``None``.""" try: @@ -640,7 +652,8 @@ def load_yaml_file_readonly(path: str | Path) -> Any: Returns the cached object itself — callers must never mutate it. Parse errors propagate and are not cached; a missing file raises ``FileNotFoundError`` like ``open`` does.""" path = Path(path) - sig = file_signature(path.stat()) + st = path.stat() + sig = (*file_signature(st), racy_content_digest(path, st.st_mtime_ns)) key = str(path) with _YAML_FILE_CACHE_LOCK: cached = _YAML_FILE_CACHE.get(key) diff --git a/website/docs/developer-guide/config-backend.md b/website/docs/developer-guide/config-backend.md index 8c3afdf949..218563a23f 100644 --- a/website/docs/developer-guide/config-backend.md +++ b/website/docs/developer-guide/config-backend.md @@ -19,14 +19,17 @@ already reaches it. |---|---| | `read_config_doc(path)` | Parsed user layer (raises like `open` + `fast_safe_load`) | | `read_config_doc_readonly(path)` | Signature-cached read; never mutate the result | -| `config_version(path)` | Cache signature (the file's stat tuple) | +| `config_version(path)` | Change signature (the file's stat tuple) | +| `config_cache_version(path)` / `file_cache_version(path)` | Signature for a cache of the parse: the stat tuple plus a content digest while the mtime is younger than `utils.RACY_MTIME_WINDOW_NS` | | `config_exists(path)` | Whether the layer exists | | `probe_config_readable(path)` | Raise `OSError` while the layer cannot be read (no parse) | | `write_config_document(path, doc)` / `write_config_key(path, key, value)` | Writes; callers normally use `atomic_config_write` / `atomic_config_replace` | Only `/config.yaml` is a user layer. Any other file passed to these helpers (an import -source, a fixture) is read and written where it is. Caches keyed on `config_version` miss -whenever the backend's layer changes. The managed scope (`/etc/hermes`) stays an overlay on top +source, a fixture) is read and written where it is. A cache of a parse keys on +`config_cache_version`, not `config_version`: on kernels without multigrain timestamps a same-size +in-place rewrite in the tick of a read keeps mtime, ctime, size and inode, so the stat tuple alone +pins the older parse for the life of the process (a rotated `api_key` is the usual casualty). The managed scope (`/etc/hermes`) stays an overlay on top of whatever user layer the backend returns. ## Lints