mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:04:08 +03:00
9d7a4fea5a
A token used as a JSON mapping key remains raw in the structured walker; its presence prevents the fixed-point check from passing, so the fallback drops unrelated Kanban handoff, request-dump, and trace argument fields. Redact keys as quoted JSON leaves and pass the sanitized key as value context. Carrier PR: NousResearch/hermes-agent#155252 (c2h5oh) Original structured helper: ZHJay (#87030) Refs NousResearch/hermes-agent#77472 (cherry picked from commit a002f7c30f68727419ed8422e95bde83c8393290)
228 lines
7.3 KiB
Python
228 lines
7.3 KiB
Python
"""Invariant tests for fail-closed structure-aware redaction."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from agent.redact import redact_sensitive_text
|
|
from agent.redact_structured import redact_structured
|
|
|
|
|
|
def _settled(payload) -> bool:
|
|
serialized = json.dumps(payload, ensure_ascii=False, default=str)
|
|
return redact_sensitive_text(serialized, force=True) == serialized
|
|
|
|
|
|
CORRUPTING = [
|
|
{"cmd": "psql; export DB_PASSWORD=pw1longervalue99"},
|
|
{"backslash": "path\\to\\OPENAI_API_KEY=sk-abcdefghijklmnopqrst"},
|
|
]
|
|
|
|
SURVIVING = [
|
|
{"password": "hunter2horse"},
|
|
{"api_key": "abc123def456ghi"},
|
|
{"token": "sk-proj-abc123def456ghi789jkl012"},
|
|
{"note": "ghp_" + "A" * 40},
|
|
{"db": "postgres://user:supersecret@localhost:5432/db"},
|
|
]
|
|
|
|
SENTINEL = "[redaction-unverified]"
|
|
|
|
|
|
@pytest.mark.parametrize("payload", CORRUPTING + SURVIVING)
|
|
def test_result_is_a_fixed_point_of_the_authoritative_redactor(payload):
|
|
assert _settled(redact_structured(payload))
|
|
|
|
|
|
@pytest.mark.parametrize("payload", CORRUPTING)
|
|
def test_secret_is_removed_when_whole_document_redaction_corrupts_json(payload):
|
|
with pytest.raises(json.JSONDecodeError):
|
|
json.loads(redact_sensitive_text(json.dumps(payload), force=True))
|
|
|
|
out = redact_structured(payload)
|
|
|
|
for key, raw in payload.items():
|
|
assert out[key] != raw, f"{key} survived unredacted"
|
|
assert _settled(out)
|
|
|
|
|
|
def test_nested_dict_list_and_tuple_are_walked_without_losing_their_shapes():
|
|
payload = {
|
|
"deep": {"inner": {"password": "hunter2horse"}},
|
|
"rows": [{"api_key": "abc123def456ghi"}, ["x-api-key: abc123"]],
|
|
"pair": ("OPENAI_API_KEY=sk-abcdefghijklmnopqrst", "plain"),
|
|
}
|
|
|
|
out = redact_structured(payload)
|
|
|
|
assert out["deep"]["inner"]["password"] != "hunter2horse"
|
|
assert out["rows"][0]["api_key"] != "abc123def456ghi"
|
|
assert out["rows"][1][0] != "x-api-key: abc123"
|
|
assert out["pair"][0] != "OPENAI_API_KEY=sk-abcdefghijklmnopqrst"
|
|
assert isinstance(out["rows"], list)
|
|
assert isinstance(out["pair"], tuple)
|
|
assert _settled(out)
|
|
|
|
|
|
def test_benign_values_pass_through_verbatim():
|
|
payload = {
|
|
"summary": "fixed the bug in run_agent.py",
|
|
"changed_files": ["run_agent.py", "cli.py"],
|
|
"nested": {"tuple": ("a", "b")},
|
|
"number": 12,
|
|
"ratio": 0.75,
|
|
"ok": True,
|
|
"nothing": None,
|
|
"empty": "",
|
|
}
|
|
|
|
assert redact_structured(payload) == payload
|
|
|
|
|
|
def test_scalar_types_are_preserved():
|
|
payload = {"n": 5, "f": 1.5, "b": False, "nil": None}
|
|
|
|
out = redact_structured(payload)
|
|
|
|
assert out == payload
|
|
assert isinstance(out["n"], int) and not isinstance(out["n"], bool)
|
|
assert isinstance(out["f"], float)
|
|
assert isinstance(out["b"], bool)
|
|
assert out["nil"] is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"key",
|
|
("password", "api_key", "bearer", "key_material", "access_token"),
|
|
)
|
|
def test_dict_key_context_is_preserved(key):
|
|
assert redact_structured({key: "abc123def456ghi"})[key] != "abc123def456ghi"
|
|
|
|
|
|
def test_nonserializable_object_with_secret_carrying_str_fails_closed():
|
|
class Carrier:
|
|
def __str__(self) -> str:
|
|
return "OPENAI_API_KEY=sk-abcdefghijklmnopqrst"
|
|
|
|
out = redact_structured({"obj": Carrier()})
|
|
|
|
assert "sk-abcdefghijklmnopqrst" not in json.dumps(out, default=str)
|
|
assert _settled(out)
|
|
|
|
|
|
def test_nonserializable_benign_leaf_does_not_raise():
|
|
out = redact_structured({"s": {"a"}})
|
|
|
|
assert _settled(out)
|
|
|
|
|
|
def test_redactor_exception_propagates_without_returning_raw_data(monkeypatch):
|
|
def boom(_text, *, force=False, **kwargs):
|
|
raise RuntimeError("redactor unavailable")
|
|
|
|
monkeypatch.setattr("agent.redact.redact_sensitive_text", boom)
|
|
|
|
with pytest.raises(RuntimeError, match="redactor unavailable"):
|
|
redact_structured({"tok": "sk-abcdefghijklmnopqrst"})
|
|
|
|
|
|
def test_truncating_redactor_cannot_return_raw_data(monkeypatch):
|
|
def truncating(text, *, force=False, **kwargs):
|
|
return text[: max(0, len(text) - 6)]
|
|
|
|
monkeypatch.setattr("agent.redact.redact_sensitive_text", truncating)
|
|
|
|
out = redact_structured({"tok": "sk-abcdefghijklmnopqrst"})
|
|
|
|
assert "sk-abcdefghijklmnopqrst" not in json.dumps(out)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("payload", "expected"),
|
|
[
|
|
({"tok": "sk-abcdefghijklmnopqrst"}, {"_redaction": SENTINEL}),
|
|
(["sk-abcdefghijklmnopqrst"], [SENTINEL]),
|
|
(("sk-abcdefghijklmnopqrst",), [SENTINEL]),
|
|
("sk-abcdefghijklmnopqrst", SENTINEL),
|
|
],
|
|
)
|
|
def test_never_settling_redactor_returns_content_free_sentinel(
|
|
monkeypatch, payload, expected
|
|
):
|
|
def always_changes(text, *, force=False, **kwargs):
|
|
return text + "!"
|
|
|
|
monkeypatch.setattr("agent.redact.redact_sensitive_text", always_changes)
|
|
|
|
assert redact_structured(payload) == expected
|
|
|
|
|
|
@pytest.mark.parametrize("glue", ["中", "é", "ü"])
|
|
def test_token_glued_to_non_ascii_text_is_masked_without_erasing_siblings(glue):
|
|
token = "sk-" + "proj" + "A" * 30
|
|
out = redact_structured({"note": glue + token, "safe": "keep", glue: glue + token})
|
|
assert out["safe"] == "keep"
|
|
assert token not in json.dumps(out, ensure_ascii=False)
|
|
assert out["note"].startswith(glue + "sk-")
|
|
assert _settled(out)
|
|
|
|
|
|
_ENV_SECRET = "DB_PASSWORD=pw1longer" + "value99"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"payload, expected",
|
|
[
|
|
({"env": [_ENV_SECRET], "safe": "keep"}, {"env": ["***"], "safe": "keep"}),
|
|
({"safe": "keep", "env": [_ENV_SECRET]}, {"safe": "keep", "env": ["***"]}),
|
|
({"env": [[_ENV_SECRET]], "safe": "keep"}, {"env": [["***"]], "safe": "keep"}),
|
|
({"env": (_ENV_SECRET,), "safe": "keep"}, {"env": ("***",), "safe": "keep"}),
|
|
([_ENV_SECRET, "keep"], ["***", "keep"]),
|
|
],
|
|
)
|
|
def test_env_secret_in_list_item_masks_only_that_item(payload, expected):
|
|
assert redact_structured(payload) == expected
|
|
|
|
|
|
def test_export_assignment_in_list_item_keeps_siblings():
|
|
secret = "export OPENAI_API_KEY=" + "sk-" + "a" * 30
|
|
out = redact_structured({"commands_run": [secret, "ls"], "summary": "done"})
|
|
assert out["summary"] == "done"
|
|
assert out["commands_run"][1] == "ls"
|
|
assert "a" * 30 not in json.dumps(out)
|
|
assert _settled(out)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"secret_text",
|
|
[
|
|
"curl -H 'Authorization: Bearer " + "abcdef0123456789abcdef" + "' https://api.example.com",
|
|
"mysql --password=" + "Zq8xR2vT9pL4mN7kQwE" + " -h db",
|
|
"TOKEN=" + "6f1d2a9c8b3e4f5a6d7c8b9a0e1f2d3c",
|
|
"db.password=" + "hunter2verylongpassword",
|
|
],
|
|
)
|
|
def test_secret_whose_first_mask_is_masked_again_keeps_siblings(secret_text):
|
|
# One redaction pass leaves a head/tail mask another rule masks on the next pass;
|
|
# the leaf pass must reach that fixed point itself, or _settled fails and the hard
|
|
# pass erases every sibling.
|
|
out = redact_structured({"cmd": secret_text, "argv": [secret_text], "summary": "keep"})
|
|
|
|
assert out["summary"] == "keep"
|
|
assert out["cmd"] != secret_text
|
|
assert out["argv"][0] != secret_text
|
|
assert _settled(out)
|
|
|
|
|
|
def test_credential_in_mapping_key_does_not_erase_safe_siblings():
|
|
token = "ghp_" + "A" * 40
|
|
payload = {"counts": {token: "one call"}, "summary": "keep"}
|
|
|
|
out = redact_structured(payload)
|
|
|
|
assert out["summary"] == "keep"
|
|
assert list(out["counts"].values()) == ["one call"]
|
|
assert token not in json.dumps(out)
|
|
assert _settled(out)
|