mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-19 21:37:12 +03:00
be3d6affc8
hermes_state.py becomes the stand-in plugins import, the rest of the legacy store and its test folder go, and the per-process modules the backend imported from main move under persistence/sqlite/_state/. The backend checks PID namespaces and lease holders through process_identity and its WAL setup through sqlite_support, and its error module takes the shared predicates and classification from storage_errors, so neither helper set has a second copy. The backend, sqlite_support.py and the tests that pin those modules reach their final imports, and the backend-import guard allows no file outside persistence/. The modules no caller imports any more go with it: the agent's onboarding rules (now in session_rules) and gateway/session_admission.py. The stand-in's SessionDB releases its store on close() and, used again afterwards, opens it again with a warning, as the old class reopened its connection, unless the file at its path was replaced or the store is latched corrupt; a read-only one raises. A close during a call releases the store when the call returns. A named source wins over a source list in its session listing and count. An agent given a writable stand-in SessionDB as session_db= and no persistence= takes the store handle that SessionDB forwards to as its persistence, so its turn flush, token counts, compression and session row writes go through the storage interface, as an agent given the old SessionDB wrote through it. The backend-import guard also holds the backend's remaining reads of the process or task home, each with its reason, and refuses any other and any import of the stand-in inside persistence/sqlite/. Plugin context engines get the agent's store as persistence= beside the deprecated session_db=, in each hook whose signature accepts that keyword, including the rebind after /new, /resume and /branch, and the context engine guide and the plugin guide's replacement table name it. A test fails once the running release reaches the stand-in's removal release, listing what goes with the stand-in. release_unheld_media and collect_unheld_api_images take a store only: their SessionDB branches, which no caller reaches, go with the gateway's own copies of the admission and transcript queries.
615 lines
37 KiB
Python
615 lines
37 KiB
Python
"""Durable transcript persistence for ``AIAgent`` (mixin; MRO-resolved from ``run_agent``): SQLite flush
|
|
with intrinsic ``_DB_PERSISTED_MARKER`` dedup, ephemeral-scaffolding filtering, explicit
|
|
trajectory export."""
|
|
import hashlib
|
|
|
|
import logging
|
|
import re
|
|
from contextlib import nullcontext
|
|
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from agent.context_compressor import (
|
|
COMPRESSED_SUMMARY_METADATA_KEY, _DB_PERSISTED_MARKER, ContextCompressor, _newest_checkpoint_carrier,
|
|
_strip_persistence_markers, drop_shadowed_checkpoints,
|
|
)
|
|
from session_rules.turns import user_originated_turn_view, sanitize_context
|
|
from agent.lazy_forward import forward as _forward, forward_static as _forward_static
|
|
|
|
from agent.trajectory import save_trajectory as _save_trajectory_to_file
|
|
from agent.message_metadata import (
|
|
ABSORBED_MESSAGE_UIDS, DB_ROW_SNAPSHOT, MERGED_TURN_PREFIX, MESSAGE_UID, REPAIR_BOOKKEEPING_FIELDS,
|
|
RESTORED_FROM_STORE, TOOL_CALL_UID, TOOL_CALL_UIDS, copy_identity_fields, tool_call_uid_from_history,
|
|
)
|
|
from session_rules.turns import message_uid_or_none, _durable_content, _is_ephemeral_scaffolding
|
|
from agent.transcript_repair import sync_flushed_message_markers
|
|
from persistence.conversation.messages import NewMessage, RowDisposition
|
|
from session_rules.turns import _EPHEMERAL_SCAFFOLDING_FLAGS, _IMAGE_PART_TYPES
|
|
|
|
|
|
logger = logging.getLogger("run_agent") # origin module's name: log records / caplog filters unchanged
|
|
|
|
# Reasoning/codex fields are role-gated (assistant-only) inside _insert_message_rows.
|
|
_ROW_REASONING_KEYS = ("reasoning", "reasoning_content", "reasoning_details", "codex_reasoning_items", "codex_message_items")
|
|
_PERSIST_AFTER_ADMISSION_INTERRUPT = "_persist_after_admission_interrupt"
|
|
|
|
|
|
# `_DB_PERSISTED_MARKER` (agent.context_compressor) is the intrinsic "already written to SQLite" marker: an
|
|
# id(msg) set can alias a freed dict's address onto a new message, a key on the dict cannot. The `_` prefix is
|
|
# mandatory (wire sanitizers strip `_` keys). CONTRACT: the marker asserts the dict's CONTENT is durable as
|
|
# written — any in-place mutation that must persist MUST pop it (turn_finalizer, context_compressor).
|
|
|
|
|
|
def _safe_session_filename_component(session_id: str) -> str:
|
|
"""Path-safe component for a (possibly untrusted ``X-Hermes-Session-Id``) ID: non ``[A-Za-z0-9_-]`` → ``_``,
|
|
capped, plus a content hash when changed so distinct IDs cannot collide."""
|
|
raw = str(session_id or "").strip()
|
|
sanitized = re.sub(r"[^\w-]", "_", raw).strip("._")[:96] or "session"
|
|
if raw and sanitized == raw:
|
|
return sanitized
|
|
return f"{sanitized}_{hashlib.sha256(raw.encode('utf-8', errors='surrogatepass')).hexdigest()[:12]}"
|
|
|
|
|
|
def _override_replaces_content(msg: dict, content: Any, override: Any) -> bool:
|
|
"""May the persist override replace ``content``? A plain-text override must not replace native image/audio
|
|
blocks (a list override is the clean multimodal payload and does), nor a message MERGED with a compaction
|
|
summary (overwriting would drop the summary)."""
|
|
return (
|
|
override is not None
|
|
and not msg.get(COMPRESSED_SUMMARY_METADATA_KEY)
|
|
and (not isinstance(content, list) or isinstance(override, list))
|
|
)
|
|
|
|
|
|
def _content_with_turn_override(msg: dict, content: Any, override: Any) -> Any:
|
|
"""Replace only the absorbed current turn, leaving unanswered history intact. The join matches
|
|
``_merge_consecutive_users``; a row that no longer holds the prefix takes the plain override."""
|
|
prefix = msg.get(MERGED_TURN_PREFIX)
|
|
if isinstance(content, str) and isinstance(override, str) and isinstance(prefix, str):
|
|
if content == prefix or content.startswith(prefix + "\n\n"):
|
|
return prefix + "\n\n" + override if override else prefix
|
|
return override
|
|
|
|
|
|
def durable_user_row_content(agent, msg: dict, content: Any, api_content: Any) -> tuple[Any, Any]:
|
|
"""``(content, api_content)`` as the current turn's user row is written: the persist override is the
|
|
clean transcript, the live content is what the wire sent — so when they differ and nothing else was
|
|
injected, the live bytes ARE the sidecar. Shared by the flush and the turn-start stamp so the stamp
|
|
matches the row the flush wrote."""
|
|
override = getattr(agent, "_persist_user_message_override", None)
|
|
if _override_replaces_content(msg, content, override):
|
|
if api_content is None and isinstance(content, str) and content != override:
|
|
api_content = content
|
|
content = _content_with_turn_override(msg, content, override)
|
|
return content, api_content
|
|
|
|
|
|
def _summary_display_kind(msg: dict) -> Any:
|
|
"""Standalone handoffs are hidden so they never occupy the active user slot in retry/undo dispatch;
|
|
merge-into-tail carriers keep their prior visibility."""
|
|
if (
|
|
msg.get(COMPRESSED_SUMMARY_METADATA_KEY)
|
|
and user_originated_turn_view(msg) is None
|
|
and (
|
|
ContextCompressor.classify_summary_content(msg.get("content")) == "standalone"
|
|
or not msg.get("_compressed_summary_has_user_turn")
|
|
)
|
|
):
|
|
return "hidden"
|
|
return msg.get("display_kind")
|
|
|
|
|
|
def _persist_lock(agent):
|
|
"""Close and turn-start persistence can run on separate CLI threads: one critical section.
|
|
|
|
``__init__`` always creates ``_session_persist_lock``; only ``object.__new__``-built test stubs lack it
|
|
(they run unlocked, matching the historical ``if persist_lock is None`` branch).
|
|
"""
|
|
lock = getattr(agent, "_session_persist_lock", None)
|
|
return nullcontext() if lock is None else lock
|
|
|
|
|
|
def adopt_unanswered_turn(history: list[dict[str, Any]], query: Any, agent: Any) -> bool:
|
|
"""Re-stage the transcript's unanswered tail row as THIS turn's user message; True when adopted.
|
|
|
|
A dispatcher's re-run of a failed delivery turn resumes the DM its first attempt already persisted
|
|
instead of appending it again. Rows loaded from the store are born durable (``_rows_to_conversation``),
|
|
so handing the tail row back as ``agent._pending_cli_user_message`` makes ``_stage_turn_user_message``
|
|
reuse it as this turn's user dict and the flush writes no second row. The one user is the peer-DM
|
|
lane's re-run (``resume_unanswered_turn`` in ``gateway.platforms.api_server``, #115325): in process it
|
|
calls this on the agent it just built for the re-run; under the session authority the admission
|
|
carries ``resume_unanswered`` and ``gateway.run_turn_runner`` calls it on the admitted history
|
|
when its tail is that bare user row (a tool suffix there keeps the completed work in view).
|
|
The owner-side Bot Chat retry (``gateway.session_bot._maybe_retry``) does not adopt: its failed
|
|
turn is already closed by the durable failed-turn boundary, which this function declines by design.
|
|
|
|
The DM is not always the literal tail: a turn that died mid-way persisted its tool scaffolding — assistant
|
|
``tool_calls`` rows and their ``tool`` results — behind the DM before the failure text was built, and the
|
|
dispatcher retries that too. The DM is still unanswered while nothing after it is a plain assistant reply,
|
|
so it is adopted and the failed attempt's scaffolding leaves the in-memory transcript: the re-run starts
|
|
the turn over from the DM (the rows stay in the DB as the record of the failed attempt; the re-run's
|
|
answer lands after them as a valid continuation). Anything else declines — no user row at the tail, or a
|
|
different text there — so a person's deliberate re-send of the same text is never swallowed.
|
|
"""
|
|
idx = next((i for i in range(len(history) - 1, -1, -1)
|
|
if isinstance(history[i], dict) and history[i].get("role") == "user"), None)
|
|
if idx is None or history[idx].get("content") != query:
|
|
return False
|
|
if not all(isinstance(row, dict) and (row.get("role") == "tool" or (row.get("role") == "assistant" and row.get("tool_calls")))
|
|
for row in history[idx + 1:]):
|
|
return False
|
|
tail = history[idx]
|
|
del history[idx:]
|
|
tail[_DB_PERSISTED_MARKER] = True
|
|
agent._pending_cli_user_message = tail
|
|
return True
|
|
|
|
|
|
# --- flush phases (module-level so the flush also works bound onto duck-typed agents) ---
|
|
|
|
def _db_flush_seed_ids(agent) -> set:
|
|
"""One-shot ``_flushed_db_message_ids`` seed (same session, after a non-empty flush); the scan translates
|
|
it to markers and the flush clears it."""
|
|
current_session_id = getattr(agent, "session_id", None)
|
|
same_session = getattr(agent, "_flushed_db_message_session_id", None) == current_session_id
|
|
seed_ids = getattr(agent, "_flushed_db_message_ids", None) if same_session and agent._last_flushed_db_idx != 0 else None
|
|
agent._flushed_db_message_session_id = current_session_id
|
|
return seed_ids if isinstance(seed_ids, set) else set()
|
|
|
|
|
|
def _db_flush_store_uids(agent) -> set:
|
|
"""``message_uid``s this agent saw durable in its CURRENT session (flushed by it, or scanned as already-durable
|
|
history): the provenance that lets the write path skip a marker-stripped copy of one of them (#134799)."""
|
|
session_id = getattr(agent, "session_id", None)
|
|
scope = getattr(agent, "_db_store_uid_scope", None)
|
|
if not (isinstance(scope, tuple) and scope[0] == session_id):
|
|
scope = agent._db_store_uid_scope = (session_id, set())
|
|
return scope[1]
|
|
|
|
|
|
def _db_flush_scan_start(agent, messages: list[dict]) -> int:
|
|
"""Skip the identity-matched, still-marked prefix of the previous flush's snapshot."""
|
|
scan_start = 0
|
|
for prev, cur in zip(getattr(agent, "_db_flush_scan_prefix", None) or (), messages):
|
|
if cur is not prev or not cur.get(_DB_PERSISTED_MARKER):
|
|
break
|
|
scan_start += 1
|
|
return scan_start
|
|
|
|
|
|
def _db_flush_row(agent, msg: dict, is_current_turn_user: bool) -> dict[str, Any]:
|
|
"""Build the session-db row for ``msg``, applying the persist override to THIS row only."""
|
|
role = msg.get("role", "unknown")
|
|
content = msg.get("content")
|
|
# api_content sidecar: exact bytes sent to the API when they differ from clean content (replay parity).
|
|
api_content = msg.get("api_content") if isinstance(msg.get("api_content"), str) else None
|
|
timestamp = msg.get("timestamp")
|
|
if is_current_turn_user and role == "user":
|
|
content, api_content = durable_user_row_content(agent, msg, content, api_content)
|
|
ov_timestamp = getattr(agent, "_persist_user_message_timestamp", None)
|
|
timestamp = timestamp if ov_timestamp is None else ov_timestamp
|
|
if api_content == content:
|
|
api_content = None
|
|
# get_messages_as_conversation replays rows through sanitize_context().strip(); capture the sent bytes
|
|
# when they would differ (compared in wire form).
|
|
if (
|
|
api_content is None and role in ("user", "assistant") and isinstance(content, str) and content
|
|
and sanitize_context(content).strip() != content.strip()
|
|
):
|
|
api_content = content
|
|
# Key order is the divert-JSONL wire order (divert_session_transcript_jsonl).
|
|
row = {
|
|
"role": role, "content": _durable_content(content),
|
|
"tool_name": msg.get("tool_name") or (msg.get("name") if role == "tool" else None),
|
|
"tool_calls": msg["tool_calls"] if isinstance(msg.get("tool_calls"), list) else None,
|
|
"tool_call_id": msg.get("tool_call_id"), "effect_disposition": msg.get("effect_disposition"),
|
|
"token_count": msg.get("token_count"), "finish_reason": msg.get("finish_reason"),
|
|
**{k: msg.get(k) for k in _ROW_REASONING_KEYS},
|
|
"_compressed_summary": bool(msg.get(COMPRESSED_SUMMARY_METADATA_KEY)),
|
|
"timestamp": timestamp, "api_content": api_content,
|
|
"display_kind": _summary_display_kind(msg), "display_metadata": msg.get("display_metadata"),
|
|
# Load-bearing for restart drain-window recovery dedup.
|
|
"platform_message_id": msg.get("platform_message_id") or msg.get("message_id"),
|
|
"observed": bool(msg.get("observed")),
|
|
}
|
|
if isinstance(msg.get("_row_id"), int):
|
|
row["_row_id"] = msg["_row_id"]
|
|
# The merge witness rides on the survivor's row (an owned column: a row-addressed rewrite of the
|
|
# survivor writes it too), so a restart sees which rows the composite folded.
|
|
copy_identity_fields(msg, row)
|
|
if isinstance(msg.get(DB_ROW_SNAPSHOT), str):
|
|
row[DB_ROW_SNAPSHOT] = msg[DB_ROW_SNAPSHOT]
|
|
return row
|
|
|
|
|
|
def sync_rewritten_tool_rows(agent, batch: list[dict], contents_before: list[Any]) -> int:
|
|
"""Carry an end-of-batch rewrite of already-flushed tool results (the turn budget's spill
|
|
stubs) onto their durable rows; returns the rows rewritten.
|
|
|
|
Each result was flushed as it landed, so its row holds the full body. Replay reads the
|
|
store, so a row left stale reloads the spilled bytes next turn: the request grows by
|
|
everything the budget removed and its cached prefix diverges at the first spilled row.
|
|
The full body survives as the archived twin ``rewrite_pruned_rows`` keeps, beside the
|
|
spill file. A store without the in-place writer, or a row another writer moved, keeps
|
|
today's row.
|
|
"""
|
|
from agent.agent_init_session import agent_interface
|
|
session_db, session_id = getattr(agent, "_session_db", None), getattr(agent, "session_id", None)
|
|
store = agent_interface(agent)
|
|
rewrite = getattr(session_db, "rewrite_pruned_rows", None)
|
|
if getattr(agent, "_persist_disabled", False) or not session_id or (store is None and not callable(rewrite)):
|
|
return 0
|
|
changed = [
|
|
(msg, before) for msg, before in zip(batch, contents_before)
|
|
if msg.get(_DB_PERSISTED_MARKER) and msg.get("role") == "tool" and msg.get("content") != before]
|
|
if not changed:
|
|
return 0
|
|
rows = [(_db_flush_row(agent, {**msg, "content": before}, False), _db_flush_row(agent, msg, False))
|
|
for msg, before in changed]
|
|
from persistence.core.errors import PruneRowUnresolved, TurnLeaseLost
|
|
lease = dict(turn_lease_holder=getattr(agent, "_active_session_turn_lease_holder", None),
|
|
turn_lease_ttl_seconds=getattr(agent, "_active_session_turn_lease_ttl_seconds", 300.0) or 300.0)
|
|
try:
|
|
with _persist_lock(agent):
|
|
if store is not None:
|
|
from agent.conversation_compression_archive import held_message
|
|
from persistence.conversation.messages import PrunedRowChange
|
|
result = store.messages.rewrite_pruned_rows(session_id, [
|
|
PrunedRowChange(held_message(original), NewMessage.from_live(replacement),
|
|
NewMessage.from_live(original))
|
|
for original, replacement in rows], **lease)
|
|
for (_original, replacement), snapshot in zip(rows, result.snapshots):
|
|
if snapshot is not None:
|
|
replacement[DB_ROW_SNAPSHOT] = snapshot
|
|
written = result.rewritten
|
|
else:
|
|
written = rewrite(session_id, rows, **lease)
|
|
except (PruneRowUnresolved, TurnLeaseLost) as exc: # another writer owns the row or turn
|
|
logger.info("Tool-result rewrite not persisted; the stored row keeps the full body: %s", exc)
|
|
return 0
|
|
except Exception as exc: # health: allow BLE001 -- a store failure must not fail the turn; the row keeps its body
|
|
logger.warning("Tool-result rewrite not persisted: %s", exc)
|
|
return 0
|
|
for (msg, _before), (_original, replacement) in zip(changed, rows):
|
|
if isinstance(replacement.get(DB_ROW_SNAPSHOT), str):
|
|
msg[DB_ROW_SNAPSHOT] = replacement[DB_ROW_SNAPSHOT]
|
|
return int(written)
|
|
|
|
|
|
def _db_flush_collect(agent, messages: list[dict], conversation_history: Optional[list[dict]],
|
|
replay_history: bool = False):
|
|
"""Scan for un-flushed messages; returns ``(rows, msgs)`` to write in one transaction. ``replay_history``
|
|
(session-row heal) writes the history prefix again instead of stamping it as already durable."""
|
|
seed_ids = _db_flush_seed_ids(agent)
|
|
history_ids = {id(item) for item in (conversation_history or []) if isinstance(item, dict)}
|
|
ov_idx = getattr(agent, "_persist_user_message_idx", None)
|
|
# Also match the staged CLI dict by identity — the close safety-net may flush a shortened snapshot whose
|
|
# turn index refers to the full history.
|
|
pending_cli_message = getattr(agent, "_pending_cli_user_message", None)
|
|
batch_rows: list[dict[str, Any]] = []
|
|
batch_msgs: list[dict] = []
|
|
tool_uid_owners: dict = {} # tool_call_uid_from_history memo; the scanned dicts outlive this loop
|
|
# Durable-uid re-insert guard: a row whose uid this agent already saw durable is marked RESTORED_FROM_STORE;
|
|
# resolve_and_repair_transcript_batch skips it only if that uid is still ACTIVE, inside the write
|
|
# transaction, and never for a snapshot-carrying dict (the CAS rewrite owns its live edit).
|
|
store_uids = _db_flush_store_uids(agent)
|
|
for msg_idx in range(_db_flush_scan_start(agent, messages), len(messages)):
|
|
msg = messages[msg_idx]
|
|
# Append-only flush: a mid-turn persist of scaffolding would commit a synthetic turn the end-of-turn
|
|
# drop cannot un-write. Skip regardless of position.
|
|
if not isinstance(msg, dict) or _is_ephemeral_scaffolding(msg):
|
|
continue
|
|
uid = message_uid_or_none(msg)
|
|
if msg.get(_DB_PERSISTED_MARKER):
|
|
if uid is not None:
|
|
store_uids.add(uid)
|
|
continue
|
|
# Already durable (history copy or caller-seeded): stamp so future flushes skip it.
|
|
is_history = id(msg) in history_ids
|
|
if (
|
|
(is_history and not replay_history) or id(msg) in seed_ids
|
|
) and not msg.get(_PERSIST_AFTER_ADMISSION_INTERRUPT):
|
|
msg[_DB_PERSISTED_MARKER] = True
|
|
if uid is not None:
|
|
store_uids.add(uid)
|
|
continue
|
|
if getattr(agent, "_mute_notification_reply", False) and not is_history:
|
|
# Only new rows, never the cached history prefix. Keep evidence/model
|
|
# context intact while transcript pollers omit unsolicited presentation.
|
|
msg["display_kind"] = "hidden"
|
|
msg["display_metadata"] = {**(msg.get("display_metadata") or {}), "notification_category": "diagnostic"}
|
|
if msg.get("role") == "tool" and not msg.get(TOOL_CALL_UID):
|
|
# A result whose call was flushed in an earlier batch: pair it with the uid that assistant row
|
|
# minted (same-batch pairing happens inside the insert).
|
|
if (tool_uid := tool_call_uid_from_history(messages, msg_idx, tool_uid_owners)) is not None:
|
|
msg[TOOL_CALL_UID] = tool_uid
|
|
row = _db_flush_row(agent, msg, ov_idx == msg_idx or msg is pending_cli_message)
|
|
if uid is not None and uid in store_uids:
|
|
row[RESTORED_FROM_STORE] = True
|
|
batch_rows.append(row)
|
|
batch_msgs.append(msg)
|
|
return batch_rows, batch_msgs
|
|
|
|
|
|
def _new_message(row: dict[str, Any]) -> NewMessage:
|
|
"""A ``_db_flush_row`` row as the storage interface's ``NewMessage``."""
|
|
return NewMessage(
|
|
role=row["role"], content=row["content"], timestamp=row["timestamp"], tool_calls=tuple(row["tool_calls"] or ()),
|
|
tool_name=row["tool_name"], tool_call_id=row["tool_call_id"], display_kind=row["display_kind"],
|
|
compressed_summary=row["_compressed_summary"], api_content=row["api_content"],
|
|
display_metadata=row["display_metadata"], platform_message_id=row["platform_message_id"],
|
|
observed=row["observed"], effect_disposition=row["effect_disposition"], finish_reason=row["finish_reason"],
|
|
**{key: row[key] for key in _ROW_REASONING_KEYS}, repair_target=row.get("_row_id"),
|
|
db_row_snapshot=row.get(DB_ROW_SNAPSHOT), message_uid=row.get(MESSAGE_UID),
|
|
absorbed_message_uids=tuple(row.get(ABSORBED_MESSAGE_UIDS) or ()), tool_call_uids=row.get(TOOL_CALL_UIDS),
|
|
tool_call_uid=row.get(TOOL_CALL_UID), token_count=row["token_count"],
|
|
restored_from_store=row.get(RESTORED_FROM_STORE) is True)
|
|
|
|
|
|
def _db_flush_write(agent, batch_rows: list[dict[str, Any]], batch_msgs: list[dict], messages: list[dict]) -> None:
|
|
"""One transaction for the turn's new rows: on failure nothing lands and no markers are stamped."""
|
|
if not batch_rows:
|
|
return
|
|
from agent.agent_init_session import agent_interface
|
|
guards = {
|
|
"turn_lease_holder": getattr(agent, "_active_session_turn_lease_holder", None),
|
|
"turn_lease_ttl_seconds": getattr(agent, "_active_session_turn_lease_ttl_seconds", 300.0) or 300.0,
|
|
}
|
|
if (store := agent_interface(agent)) is not None:
|
|
result = store.messages.append_batch(agent.session_id, [_new_message(row) for row in batch_rows], **guards)
|
|
# Stamp each row with the state the write gave it, as the store's own writer stamps the dicts it takes. A
|
|
# restored row the store skipped (no row address, no snapshot) is only marked durable, as that writer marks it.
|
|
for row, outcome in zip(batch_rows, result.outcomes):
|
|
if (outcome.disposition is RowDisposition.REPAIRED and row.get(RESTORED_FROM_STORE) is True
|
|
and row.get("_row_id") is None and row.get(DB_ROW_SNAPSHOT) is None):
|
|
continue
|
|
row.update((key, value) for key, value in outcome.as_row_annotation().items() if value is not None)
|
|
else:
|
|
agent._session_db.append_messages_batch(
|
|
session_id=agent.session_id, messages=batch_rows,
|
|
compression_lock_holder=getattr(agent, "_active_compression_lock_holder", None), **guards)
|
|
sync_flushed_message_markers(batch_msgs, batch_rows)
|
|
_db_flush_store_uids(agent).update(uid for msg in batch_msgs if (uid := message_uid_or_none(msg)) is not None)
|
|
if _newest_checkpoint_carrier(batch_msgs, "codex_reasoning_items") >= 0:
|
|
# The insert already rewrote the older rows (SessionDB._drop_shadowed_checkpoint_rows); mirror it on
|
|
# the live transcript so forks/compaction built from memory carry one checkpoint too. Markers stay:
|
|
# the rows are durable exactly as the dicts now read.
|
|
drop_shadowed_checkpoints(messages)
|
|
|
|
|
|
def _db_flush_adopt_compression_tip(agent) -> bool:
|
|
"""Adopt the live continuation of a compression-closed session. Same-id tip = no continuation; a tip
|
|
whose row is missing or already ended is not adopted either."""
|
|
from agent.agent_init_session import agent_interface
|
|
old_id, store = agent.session_id, agent_interface(agent)
|
|
try:
|
|
tip = store.lineage.tip(old_id) if store is not None else agent._session_db.get_compression_tip(old_id)
|
|
except Exception as tip_exc:
|
|
logger.warning("compression tip lookup failed for %s: %s", old_id, tip_exc)
|
|
return False
|
|
if not tip or tip == old_id:
|
|
return False
|
|
try:
|
|
tip_row = store.sessions.get(tip) if store is not None else agent._session_db.get_session(tip)
|
|
except Exception:
|
|
tip_row = None
|
|
if tip_row is None or (tip_row.ended_at if store is not None else tip_row.get("ended_at")) is not None:
|
|
return False
|
|
logger.warning("Adopted live compression tip %s for closed session %s; retrying flush once", tip, old_id)
|
|
agent.session_id, agent._flushed_db_message_ids, agent._last_flushed_db_idx = tip, set(), 0
|
|
agent._compression_adoption_failed = False
|
|
return True
|
|
|
|
|
|
def _db_flush_session_row_gone(agent, session_id: Optional[str]) -> bool:
|
|
"""True only when ``session_id``'s row is confirmed absent; a failed lookup is not proof."""
|
|
from agent.agent_init_session import agent_interface
|
|
store = agent_interface(agent)
|
|
try:
|
|
return (store.sessions.get(session_id) if store is not None else agent._session_db.get_session(session_id)) is None
|
|
except Exception as lookup_exc:
|
|
logger.warning("session row lookup failed for %s: %s", session_id, lookup_exc)
|
|
return False
|
|
|
|
|
|
def _db_flush_failed(agent, e: Exception, batch_rows: list[dict[str, Any]], adoption_budget: int,
|
|
messages: list[dict]) -> Optional[str]:
|
|
"""Classify a failed flush and name the one retry the caller should take, or None to fail closed.
|
|
|
|
``"adopted"``: a compression-closed session moved onto its live tip. ``"healed"``: the session row was
|
|
deleted under the live agent and has been recreated (unparented for a delegate child whose parent went
|
|
with it); flush markers are reset so the caller replays the full in-memory transcript. Either retry is
|
|
taken at most once (``adoption_budget``)."""
|
|
agent._db_flush_scan_prefix = None # full re-scan next flush: an exception mid-loop leaves mixed dispositions
|
|
# The only place the SQLite error is visible before it becomes a bare False — classify it so the turn-end
|
|
# explanation names the real cause.
|
|
from agent.agent_init_session import agent_interface
|
|
from persistence import CompressionClosed, StoreQuarantined, StoreReplaced
|
|
from session_rescue import divert_session_transcript_jsonl
|
|
from storage_errors import classify_persistence_error
|
|
agent._last_persistence_error_cause = classify_persistence_error(e)
|
|
if agent._last_persistence_error_cause == "session_row_missing":
|
|
# The session row was removed under this live agent (`hermes sessions delete`, the Desktop/web
|
|
# delete, bulk prune, a profile-repair move, an in-place store rebuild — none visible to the
|
|
# cached agent, so the cached `_session_db_created` flag is stale and every later append hits
|
|
# the FK). The deletion already erased the session's message rows with it, so the durable
|
|
# transcript is empty: drop the stale flag, reset the flush markers, and replay the FULL
|
|
# in-memory transcript onto the recreated row — not just the current tail (#123583).
|
|
# The FK class also covers the sessions table's own parent/system-prompt FKs, and create is an
|
|
# upsert: only heal when the row is really gone, or the replay would duplicate a live transcript.
|
|
if adoption_budget <= 0 or not _db_flush_session_row_gone(agent, agent.session_id):
|
|
return None
|
|
_strip_persistence_markers(messages)
|
|
# The durable history prefix is gone with the row: keep a replay pending until a write succeeds, so
|
|
# a failed recreate or a failed retry write can't let a later flush stamp that prefix durable.
|
|
agent._session_row_replay_pending = agent.session_id
|
|
agent._flushed_db_message_ids = set()
|
|
agent._last_flushed_db_idx = 0
|
|
agent._session_db_created = False
|
|
agent._ensure_db_session()
|
|
parent_id = agent._parent_session_id
|
|
if not agent._session_db_created and parent_id and _db_flush_session_row_gone(agent, parent_id):
|
|
# Delegate child whose parent was deleted (cascade): the row's own parent FK would reject every
|
|
# recreate. Create it unparented for this call only; the relay and hooks still use the parent id.
|
|
agent._parent_session_id = None
|
|
try:
|
|
agent._ensure_db_session()
|
|
finally:
|
|
agent._parent_session_id = parent_id
|
|
if not agent._session_db_created:
|
|
# Row creation failed too (transient store trouble): don't append into a guaranteed
|
|
# rollback — keep the batch unmarked so the next flush retries the whole thing. That flush
|
|
# recreates the row up front (no FK error, no heal); the pending replay covers the history prefix.
|
|
logger.warning("Session DB row for %s is missing and could not be recreated; will retry next flush",
|
|
getattr(agent, "session_id", None))
|
|
return None
|
|
logger.warning("Session DB row for %s was removed under the live agent; recreated it and replaying the transcript",
|
|
getattr(agent, "session_id", None))
|
|
return "healed"
|
|
if isinstance(e, (StoreReplaced, StoreQuarantined)):
|
|
# A replaced/quarantined handle will not take this batch again — keep it on disk, beside its store.
|
|
try:
|
|
# The CAS digest / adopted row are local repair bookkeeping, not transcript payload.
|
|
divert_session_transcript_jsonl(getattr(agent, "session_id", "") or "",
|
|
[{k: v for k, v in r.items() if k not in REPAIR_BOOKKEEPING_FIELDS}
|
|
for r in batch_rows],
|
|
sessions_dir=agent_interface(agent).scope.profile_dir / "sessions")
|
|
except Exception:
|
|
logger.warning("JSONL divert failed after state.db %s for %s",
|
|
agent._last_persistence_error_cause, getattr(agent, "session_id", None), exc_info=True)
|
|
if isinstance(e, CompressionClosed):
|
|
# Compression race: another path rotated this session mid-write. Retry exactly once on the live tip; a
|
|
# second closed-parent write fails closed.
|
|
if adoption_budget > 0 and _db_flush_adopt_compression_tip(agent):
|
|
return "adopted"
|
|
agent._compression_adoption_failed = True # lets the turn explanation name rotation, not full-disk advice
|
|
logger.warning("Session DB append_message failed: %s", e)
|
|
return None
|
|
|
|
|
|
class SessionPersistenceMixin:
|
|
"""Session DB flush and trajectory persistence (see module docstring)."""
|
|
|
|
def _apply_persist_user_message_override(self, messages: list[dict]) -> None:
|
|
"""Rewrite the current-turn user message in place: some paths send an API-only variant that must not
|
|
leak into transcripts or resumed history."""
|
|
idx = getattr(self, "_persist_user_message_idx", None)
|
|
override = getattr(self, "_persist_user_message_override", None)
|
|
timestamp = getattr(self, "_persist_user_message_timestamp", None)
|
|
platform_id = getattr(self, "_persist_user_message_platform_id", None)
|
|
if idx is None or (override is None and timestamp is None and platform_id is None):
|
|
return
|
|
msg = messages[idx] if 0 <= idx < len(messages) else None
|
|
if not (isinstance(msg, dict) and msg.get("role") == "user"):
|
|
return
|
|
if _override_replaces_content(msg, msg.get("content"), override):
|
|
msg["content"] = _content_with_turn_override(msg, msg.get("content"), override)
|
|
if timestamp is not None:
|
|
msg["timestamp"] = timestamp
|
|
if platform_id is not None: # load-bearing for restart drain-window recovery dedup (has_platform_message_id)
|
|
msg["platform_message_id"] = platform_id
|
|
|
|
def _persist_session(self, messages: list[dict], conversation_history: list[dict] | None = None):
|
|
"""Save to SQLite on any exit path. Trailing empty-response scaffolding is dropped from
|
|
the live list; the persist override is applied to the DB row only.
|
|
|
|
The persist user-message *override* is NOT applied here — it is resolved inside
|
|
``_flush_messages_to_session_db`` and written only to the DB row, never mutating the live message
|
|
list used by the API call (#48677 is thus closed for every persist caller, not just this one).
|
|
"""
|
|
from agent.agent_init_session import agent_interface
|
|
from agent.agent_runtime_helpers import note_turn_persisted
|
|
with _persist_lock(self):
|
|
# Only the scaffolding goes here. Closing a tool tail this uncovers is the exit
|
|
# owner's job (``_close_transcript_tail``, ``abort_turn_on_interrupt``): only it knows
|
|
# the reason to record.
|
|
self._drop_trailing_empty_response_scaffolding(messages)
|
|
self._session_messages = messages
|
|
self._flush_messages_to_session_db(messages, conversation_history)
|
|
# Drain async token-accounting deltas at every persist point; cheap no-op when nothing queued.
|
|
if (store := agent_interface(self)) is not None:
|
|
store.usage.flush_usage()
|
|
elif self._session_db is not None:
|
|
self._session_db.flush_token_counts()
|
|
note_turn_persisted(self)
|
|
|
|
def _drop_trailing_empty_response_scaffolding(self, messages: list[dict]) -> None:
|
|
"""Pop empty-response retry scaffolding from the tail. The
|
|
assistant(tool_calls) / tool rows before it stay: they were saved before the tools ran, so
|
|
dropping them from the live history only makes the model repeat a side effect the durable
|
|
transcript already records."""
|
|
def tail(*keys: str) -> bool:
|
|
return bool(messages) and isinstance(messages[-1], dict) and any(messages[-1].get(k) for k in keys)
|
|
|
|
while tail("_empty_recovery_synthetic", "_empty_terminal_sentinel"):
|
|
messages.pop()
|
|
|
|
_repair_message_sequence = _forward("session_rules.turns", "repair_message_sequence")
|
|
|
|
def _flush_messages_to_session_db(self, messages: list[dict], conversation_history: Optional[list[dict]] = None):
|
|
"""Serialize direct and turn-boundary session flushes per agent."""
|
|
with _persist_lock(self):
|
|
return self._flush_messages_to_session_db_unlocked(messages, conversation_history)
|
|
|
|
def _flush_messages_to_session_db_unlocked(
|
|
self, messages: list[dict], conversation_history: Optional[list[dict]] = None, _adoption_budget: int = 1,
|
|
):
|
|
"""Persist un-flushed messages to SQLite. Dedup is the intrinsic ``_DB_PERSISTED_MARKER`` on each written
|
|
dict — not positional slices (drift after sequence repair) nor an ``id(msg)`` set (address reuse). The
|
|
persist override touches the written row only. A compression-closed session adopts its live tip and
|
|
retries exactly once.
|
|
|
|
Deduplicates via an intrinsic ``_DB_PERSISTED_MARKER`` stamped on each written message dict, so
|
|
repeated calls (from multiple exit paths) only write truly new messages — preventing the
|
|
duplicate-write bug (#860) without relying on positional slices that can drift after
|
|
message-sequence repair, and without a retained ``id(msg)`` set that CPython could alias onto a
|
|
freed-then-reused address (#50372). The ``_flushed_db_message_ids`` attribute is now only a one-shot
|
|
seed (translated to markers, then cleared each flush), not a persisted set.
|
|
"""
|
|
# Persistence-isolated agents (background review fork) share the parent's session_id for cache warmth;
|
|
# a write here would land the curator's turn in the user's real history.
|
|
if getattr(self, "_persist_disabled", False) or not self._session_db:
|
|
return None
|
|
batch_rows: list[dict[str, Any]] = []
|
|
try:
|
|
if not self._session_db_created: # retry row creation if the earlier attempt failed transiently
|
|
self._ensure_db_session()
|
|
# getattr: object.__new__ test agents flush without running AIAgent init.
|
|
replay = getattr(self, "_session_row_replay_pending", None) == self.session_id
|
|
batch_rows, batch_msgs = _db_flush_collect(self, messages, conversation_history, replay)
|
|
_db_flush_write(self, batch_rows, batch_msgs, messages)
|
|
self._session_row_replay_pending = None
|
|
# Markers are now the sole truth; reset the one-shot seed so no id() outlives this flush.
|
|
self._flushed_db_message_ids = set()
|
|
self._last_flushed_db_idx = len(messages)
|
|
# Snapshot for the bounded scan — only on full success, so a partial list is never treated as settled.
|
|
self._db_flush_scan_prefix = messages[:]
|
|
return True
|
|
except Exception as e:
|
|
retry = _db_flush_failed(self, e, batch_rows, _adoption_budget, messages)
|
|
if retry is None:
|
|
return False
|
|
# After a heal the pending replay re-sends the history prefix instead of stamping it durable.
|
|
return self._flush_messages_to_session_db_unlocked(messages, conversation_history, _adoption_budget=0)
|
|
|
|
def _get_messages_up_to_last_assistant(self, messages: list[dict]) -> list[dict]:
|
|
"""Messages before the last assistant turn (rollback point for a malformed final answer); all if none."""
|
|
for i in range(len(messages) - 1, -1, -1):
|
|
if messages[i].get("role") == "assistant":
|
|
return messages[:i]
|
|
return messages.copy()
|
|
|
|
_format_tools_for_system_message = _forward("agent.system_prompt", "format_tools_for_system_message")
|
|
_convert_to_trajectory_format = _forward("agent.agent_runtime_helpers", "convert_to_trajectory_format")
|
|
|
|
def _save_trajectory(self, messages: list[dict[str, Any]], user_query: str, completed: bool):
|
|
"""Save conversation trajectory to JSONL file."""
|
|
from tools.hosted_tool_policy import host_feature_enabled
|
|
if not self.save_trajectories or not host_feature_enabled("trajectory_export"):
|
|
return # hosted: no host file for a tenant's transcript
|
|
trajectory = self._convert_to_trajectory_format(messages, user_query, completed)
|
|
_save_trajectory_to_file(trajectory, self.model, completed)
|
|
|
|
_extract_api_error_context = _forward_static("agent.agent_runtime_helpers", "extract_api_error_context")
|
|
_dump_api_request_debug = _forward("agent.agent_runtime_helpers", "dump_api_request_debug")
|