diff --git a/.cursor/rules/general.mdc b/.cursor/rules/general.mdc index 577c96e..6328741 100644 --- a/.cursor/rules/general.mdc +++ b/.cursor/rules/general.mdc @@ -22,6 +22,7 @@ When working with this project, follow these rules: - If possible, try to update files in a single edit when making multiple changes. - Do NOT "cd" to the project directory. - NEVER edit/delete/regenerate .env files without explicit permission. +- Don't use rg, it doesn't work anyway. ## Testing & Validation - Do NOT "test the implementation" when you are done. The only exception is when you @@ -32,8 +33,7 @@ When working with this project, follow these rules: - If the typecheck passed, there's no need for checking the linter errors. ## Async Operations -- When you need a delay, use `await delay(millis);` from `@/utils/utils` in an async function. If the current function is not async, - make it async. +- When you need a delay, use `await delay(millis);` from `@/utils/utils` in an async function. If the current function is not async, make it async. ## Database - NEVER create database migrations, they are auto-generated. @@ -59,7 +59,7 @@ When working with this project, follow these rules: - Put SCSS into one folder per page ## Animations with Framer Motion -- Don't use variants if they are used only once +- Don't use variants if they are used only once ## Debug Mode - When in debug mode and the issue is not yet fixed, ALWAYS end responses with `` containing the steps to reproduce the issue and trigger logging diff --git a/.cursor/skills/decrypt/SKILL.md b/.cursor/skills/decrypt/SKILL.md new file mode 100644 index 0000000..e0d16ed --- /dev/null +++ b/.cursor/skills/decrypt/SKILL.md @@ -0,0 +1,170 @@ +--- +name: decrypt +description: Decrypts a DM message on localhost by granting temporary compliance extract access, running extract and decrypt CLI commands, then reverting all temporary changes. Use when the user asks to decrypt a message or run the compliance decryption workflow on localhost. +--- + +# Decrypt (local debug) + +End-to-end workflow for extracting and decrypting one message on **localhost**. All temporary backend access must be removed when finished. + +**After decrypt:** what to do with the plaintext and files (compare, inspect, report, etc.) comes from the **current conversation** — not from this skill. + +## Prerequisites + +- Web dev server running at `http://localhost:8301` (browser rules default). +- `compliance_keypair.txt` at the Web repo root (`Web/compliance_keypair.txt`). +- Python deps for `scripts/compliance/decryption/` (`cryptography`, etc.). + +## Step 0 — Read the compliance keypair + +Read `Web/compliance_keypair.txt` before any decrypt step. Confirm the **private** key line exists (line after `PRIVATE KEY` header, base64). + +The decrypt CLI loads `compliance_keypair.txt` from the **current working directory**. Always run extract/decrypt with `cd` to the Web repo root: + +```bash +cd /path/to/Web +``` + +If `load_compliance_private_key` fails, the file format may need a `PRIVATE_KEY=` line (the loader expects that or a 43-char base64 line; 32-byte keys are often 44 chars with padding). + +## Step 1 — Create a temporary user and capture token + +Generate random credentials (username 3–20 chars: letters, digits, `-`, `_`; password 5–50 chars, no spaces): + +```bash +DEBUG_USER="decryptdbg$(openssl rand -hex 3)" +DEBUG_PASS="$(openssl rand -base64 12 | tr -d '/+=' | head -c 16)" +echo "user=$DEBUG_USER pass=$DEBUG_PASS" +``` + +Register and save **`token`** from the JSON response (not the derived login secret): + +```bash +REGISTER_JSON=$(curl -sS -X POST "http://localhost:8301/api/register" \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"${DEBUG_USER}\",\"display_name\":\"DecryptDebug\",\"password\":\"${DEBUG_PASS}\",\"confirm_password\":\"${DEBUG_PASS}\"}") +echo "$REGISTER_JSON" +DEBUG_TOKEN=$(python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])' <<< "$REGISTER_JSON") +echo "token saved (length ${#DEBUG_TOKEN})" +``` + +Alternatively export for the CLI: `export FROMCHAT_API_TOKEN="${DEBUG_TOKEN}"` + +Save `user.id` from the same JSON if needed for debugging. + +## Step 2 — Temporary compliance extract permission (marker blocks) + +`GET /api/dm/compliance/extract/{message_id}` is restricted to **user id 1** in: + +`backend/services/main/routes/envelope_messaging.py` (function `extract_message_for_compliance`). + +Add a **temporary** bypass between searchable markers (replace `DEBUG_USERNAME` with the user from step 1): + +```python + # TEMP_COMPLIANCE_DEBUG_START — remove after decrypt debug run + _TEMP_COMPLIANCE_DEBUG_USERNAMES = {"DEBUG_USERNAME"} + # TEMP_COMPLIANCE_DEBUG_END + + # Security check: only user ID 1 can access this + if current_user.id != 1 and current_user.username not in _TEMP_COMPLIANCE_DEBUG_USERNAMES: +``` + +Backend will auto-reload after this change, you don't need to do anything. + +To remove later: search the repo for `TEMP_COMPLIANCE_DEBUG` and delete the marker block + revert the `if` to only `current_user.id != 1`. + +## Step 3 — Extract bundle (online) + +From Web repo root, set variables and run (user supplies `MESSAGE_ID`): + +```bash +MESSAGE_ID= +RANDOM_FOLDER="run_$(openssl rand -hex 4)" +BUNDLE_DIR="/tmp/debug_decrypt/${RANDOM_FOLDER}" +DECRYPT_DIR="/tmp/debug_decrypt/${RANDOM_FOLDER}_dec" +mkdir -p /tmp/debug_decrypt + +cd /path/to/Web +python scripts/compliance/decryption/main.py extract \ + --server localhost:8301 \ + --http \ + --token "${DEBUG_TOKEN}" \ + --message-ids "${MESSAGE_ID}" \ + --out-dir "${BUNDLE_DIR}" +``` + +Equivalent using env (no `--token` flag): + +```bash +export FROMCHAT_API_TOKEN="${DEBUG_TOKEN}" +python scripts/compliance/decryption/main.py extract \ + --server localhost:8301 \ + --http \ + --message-ids "${MESSAGE_ID}" \ + --out-dir "${BUNDLE_DIR}" +``` + +Confirm `${BUNDLE_DIR}/bundle.json` exists. + +**Auth flags:** use `--token` (Bearer from register/login), or `--username` + `--password` (CLI derives the login secret). Do not pass both. + +## Step 4 — Decrypt bundle (offline) + +Still from Web repo root (`compliance_keypair.txt` must resolve): + +```bash +python scripts/compliance/decryption/main.py decrypt \ + --bundle-dir "${BUNDLE_DIR}" \ + --output-dir "${DECRYPT_DIR}" +``` + +Outputs: + +- `${DECRYPT_DIR}/messages//message.decrypted.txt` — message plaintext +- `${DECRYPT_DIR}/messages//files/` — decrypted attachments +- `${DECRYPT_DIR}/index.html` — HTML report + +## Step 5 — Use decrypted output (conversation-driven) + +Follow the **user’s request in the current chat** for what to do next (e.g. compare hashes, inspect text, verify a specific attachment). This skill stops at producing `${DECRYPT_DIR}`; do not assume a fixed post-decrypt task. + +## Step 6 — Cleanup (required) + +1. **Remove decrypt dirs:** + ```bash + rm -rf "${BUNDLE_DIR}" "${DECRYPT_DIR}" + ``` + +2. **Delete temp user** (local SQLite default DB): + ```bash + sqlite3 backend/data/database.db "DELETE FROM users WHERE username='${DEBUG_USER}';" + ``` + If your deployment uses another DB, delete the same username there. + +3. **Remove temporary permission:** search `TEMP_COMPLIANCE_DEBUG` in the repo, delete the marker block, restore the original `if current_user.id != 1:` check, restart backend. + +4. Do **not** commit `compliance_keypair.txt` or any decrypt output. + +## Checklist + +``` +- [ ] Read compliance_keypair.txt +- [ ] Register DEBUG_USER; save DEBUG_TOKEN from response +- [ ] Add TEMP_COMPLIANCE_DEBUG_* bypass; restart backend +- [ ] extract --token → bundle.json present +- [ ] decrypt → output under DECRYPT_DIR +- [ ] Post-decrypt work per conversation context +- [ ] rm -rf BUNDLE_DIR and DECRYPT_DIR +- [ ] DELETE temp user from DB +- [ ] Remove TEMP_COMPLIANCE_DEBUG markers; restart backend +``` + +## Reference + +- Extract API: `GET /api/dm/compliance/extract/{message_id}` (see `bundle_extract.py`). +- Decrypt implementation: `scripts/compliance/decryption/bundle_decrypt.py`, `crypto.py`. +- Extract auth: `--token` (Bearer), env `FROMCHAT_API_TOKEN` / `FROMCHAT_TOKEN`, or `--username` + plain `--password` (CLI calls `derive_auth_secret` for login only). + +## Rules +- NEVER use sleep in any command. +- Set a timeout on EVERY command that may request user input. diff --git a/backend/services/messaging/encryption.py b/backend/services/messaging/encryption.py index 01d8294..0a3ede3 100644 --- a/backend/services/messaging/encryption.py +++ b/backend/services/messaging/encryption.py @@ -26,6 +26,8 @@ logger = logging.getLogger(__name__) TRANSPORT_NONCE_SIZE = 24 # For X25519 transport encryption (PyNaCl Box/XSalsa20Poly1305) MEK_NONCE_SIZE = 12 # For AES-GCM content encryption MEK_SIZE = 32 # Message Encryption Key size +GCM_TAG_SIZE = 16 # AES-GCM authentication tag appended to file ciphertext +FILE_ENCRYPT_CHUNK_SIZE = 1024 * 1024 # Client streaming transport format (chunked AES-256-GCM): FCAE | version | frames… FCAE_MAGIC = b"FCAE" @@ -216,7 +218,12 @@ def decrypt_fcae_transport_blob_to_file( def encrypt_message_to_file(plaintext_path: Path, mek: bytes, output_path: Path) -> str: - """AES-GCM encrypt a file on disk; returns nonce_b64. Ciphertext written to output_path.""" + """ + AES-GCM encrypt a file on disk; returns nonce_b64. + + On-disk layout: ``ciphertext || tag`` (16-byte GCM tag at EOF). + Hazmat ``encryptor.finalize()`` does not emit the tag; it is taken from ``encryptor.tag``. + """ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes nonce = generate_nonce(MEK_NONCE_SIZE) @@ -224,14 +231,68 @@ def encrypt_message_to_file(plaintext_path: Path, mek: bytes, output_path: Path) encryptor = Cipher(algorithms.AES(mek), modes.GCM(nonce)).encryptor() with open(plaintext_path, "rb") as src, open(output_path, "wb") as dst: while True: - chunk = src.read(1024 * 1024) + chunk = src.read(FILE_ENCRYPT_CHUNK_SIZE) if not chunk: break dst.write(encryptor.update(chunk)) - dst.write(encryptor.finalize()) + encryptor.finalize() + dst.write(encryptor.tag) return base64.b64encode(nonce).decode("utf-8") +def decrypt_message_to_file( + nonce_b64: str, + mek: bytes, + encrypted_path: Path, + output_path: Path, +) -> int: + """ + Decrypt a file produced by [encrypt_message_to_file] (ciphertext || tag). + + Returns plaintext byte count. + """ + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + nonce = base64.b64decode(nonce_b64) + enc_size = encrypted_path.stat().st_size + if enc_size < GCM_TAG_SIZE: + raise ValueError("Encrypted file is too short") + + output_path.parent.mkdir(parents=True, exist_ok=True) + ciphertext_length = enc_size - GCM_TAG_SIZE + total_out = 0 + + with open(encrypted_path, "rb") as src: + src.seek(ciphertext_length) + tag = src.read(GCM_TAG_SIZE) + if len(tag) != GCM_TAG_SIZE: + raise ValueError("Encrypted file truncated (missing GCM tag)") + + decryptor = Cipher(algorithms.AES(mek), modes.GCM(nonce, tag)).decryptor() + src.seek(0) + + with open(output_path, "wb") as dst: + processed = 0 + while processed < ciphertext_length: + to_read = min(FILE_ENCRYPT_CHUNK_SIZE, ciphertext_length - processed) + chunk = src.read(to_read) + if len(chunk) != to_read: + raise ValueError("Encrypted file truncated") + processed += len(chunk) + plain = decryptor.update(chunk) + if plain: + dst.write(plain) + total_out += len(plain) + final = decryptor.finalize() + if final: + dst.write(final) + total_out += len(final) + + if total_out <= 0: + raise ValueError("Decrypted file is empty") + return total_out + + def decrypt_transport_blob( client_public_key_b64: str, encrypted_blob: bytes, diff --git a/scripts/compliance/decryption/bundle_extract.py b/scripts/compliance/decryption/bundle_extract.py index b753b92..da85e36 100644 --- a/scripts/compliance/decryption/bundle_extract.py +++ b/scripts/compliance/decryption/bundle_extract.py @@ -82,18 +82,27 @@ def extract_single_message_to_bundle(api_base_url: str, token: str, message_id: with open(enc_abs, "wb") as outf: outf.write(file_bytes) - meta_out = { + envelope_compliance_mek = data.get("compliance_wrapped_mek_b64") + use_compliance_mek = ( + isinstance(envelope_compliance_mek, str) + and envelope_compliance_mek + and wrapped_mek_b64 == envelope_compliance_mek + ) + meta_out: Dict[str, Any] = { "kind": "dm_file", "message_id": data.get("message_id"), "dm_file_id": file_id, "filename": name, "path": path, "nonce_b64": nonce_b64, - "wrapped_mek_b64": wrapped_mek_b64, - "wrap_context": "sender_wrap_key", - "wrap_public_key_b64": sender_public_key_b64, "encrypted_file_local": enc_rel, } + if use_compliance_mek: + meta_out["compliance_wrapped_mek_b64"] = wrapped_mek_b64 + else: + meta_out["wrapped_mek_b64"] = wrapped_mek_b64 + meta_out["wrap_context"] = "sender_wrap_key" + meta_out["wrap_public_key_b64"] = sender_public_key_b64 meta_filename = f"{message_id}_{file_id or 'x'}_{safe_name}.meta.json" meta_abs = os.path.join(files_dir, meta_filename) meta_rel = os.path.relpath(meta_abs, bundle_root) diff --git a/scripts/compliance/decryption/cli.py b/scripts/compliance/decryption/cli.py index 9c92fa2..a60fbeb 100644 --- a/scripts/compliance/decryption/cli.py +++ b/scripts/compliance/decryption/cli.py @@ -192,11 +192,29 @@ def _logout(api_base_url: str, token: str) -> None: pass +def _resolve_bearer_token( + *, + token: Optional[str] = None, + jwt: Optional[str] = None, +) -> Optional[str]: + """CLI flag, deprecated --jwt alias, or FROMCHAT_API_TOKEN / FROMCHAT_TOKEN env.""" + if token and jwt: + raise SystemExit("Provide only one of --token or --jwt.") + explicit = (token or jwt or "").strip() + if explicit: + return explicit + for env_name in ("FROMCHAT_API_TOKEN", "FROMCHAT_TOKEN"): + env_val = os.environ.get(env_name, "").strip() + if env_val: + return env_val + return None + + def _ensure_online_auth( *, server: Optional[str], https: Optional[bool], - jwt: Optional[str], + bearer_token: Optional[str], username: Optional[str], password: Optional[str], ) -> _AuthResult: @@ -205,19 +223,22 @@ def _ensure_online_auth( use_https = bool(https) if https is not None else _prompt_bool("Use HTTPS", default=True) api_base_url = _build_api_base(server, https=use_https) - if jwt and (username or password): - raise SystemExit("Provide either --jwt OR --username/--password, not both.") + if bearer_token and (username or password): + raise SystemExit("Provide either --token OR --username/--password, not both.") - if jwt: - return _AuthResult(api_base_url=api_base_url, token=jwt.strip(), did_login=False) + if bearer_token: + return _AuthResult(api_base_url=api_base_url, token=bearer_token, did_login=False) step("Authentication") try: if not username and password is None: - method = _choose_option(["1) Login + password", "2) JWT token"], default="1") + method = _choose_option( + ["1) Login + password", "2) API token (Bearer)"], + default="1", + ) if method.strip() == "2": - jwt_in = _prompt("JWT token") - return _AuthResult(api_base_url=api_base_url, token=jwt_in.strip(), did_login=False) + token_in = _prompt("API token") + return _AuthResult(api_base_url=api_base_url, token=token_in.strip(), did_login=False) if not username: username = _prompt("Username") @@ -243,9 +264,13 @@ def cmd_extract(args: argparse.Namespace) -> None: else: https_choice = _prompt_bool_required("Use HTTPS") - jwt: Optional[str] = args.jwt + bearer_token: Optional[str] = _resolve_bearer_token( + token=getattr(args, "token", None), + jwt=getattr(args, "jwt", None), + ) username: Optional[str] = args.username password: Optional[str] = args.password + used_password_login = bool(username or password is not None) message_ids: List[int] = [] if getattr(args, "message_ids", None): @@ -261,7 +286,7 @@ def cmd_extract(args: argparse.Namespace) -> None: auth = _ensure_online_auth( server=server, https=https_choice, - jwt=jwt, + bearer_token=bearer_token, username=username, password=password, ) @@ -270,15 +295,21 @@ def cmd_extract(args: argparse.Namespace) -> None: msg = str(e) warning(msg) if "HTTP 401" in msg or "HTTP 403" in msg: - warning("Auth failed. Please enter username and password again.") - jwt = None - username = _prompt("Username") - password = _prompt("Password", secret=True) + if bearer_token and not used_password_login: + warning("Auth failed. Please enter a valid API token again.") + bearer_token = _prompt("API token") + else: + warning("Auth failed. Please enter username and password again.") + bearer_token = None + username = _prompt("Username") + password = _prompt("Password", secret=True) + used_password_login = True continue - jwt = None + bearer_token = None username = None password = None + used_password_login = False if not _prompt_bool("Try again", default=True): raise SystemExit(1) continue @@ -301,10 +332,15 @@ def cmd_extract(args: argparse.Namespace) -> None: msg = str(e) if "HTTP 401" in msg or "HTTP 403" in msg: warning(msg) - warning("Auth failed. Please enter username and password again.") - jwt = None - username = _prompt("Username") - password = _prompt("Password", secret=True) + if bearer_token and not used_password_login: + warning("Auth failed. Please enter a valid API token again.") + bearer_token = _prompt("API token") + else: + warning("Auth failed. Please enter username and password again.") + bearer_token = None + username = _prompt("Username") + password = _prompt("Password", secret=True) + used_password_login = True continue else: raise @@ -380,8 +416,13 @@ def build_parser() -> argparse.ArgumentParser: extract_parser.add_argument("--server", required=False, help="Server host:port (e.g. localhost:8301)") extract_parser.add_argument("--https", action="store_true", help="Use HTTPS (default in interactive mode)") extract_parser.add_argument("--http", action="store_true", help="Use HTTP") - extract_parser.add_argument("--jwt", required=False, help="JWT token (Bearer)") - extract_parser.add_argument("--username", required=False, help="Login username (alternative to --jwt)") + extract_parser.add_argument( + "--token", + required=False, + help="API Bearer token (from login/register). Also FROMCHAT_API_TOKEN or FROMCHAT_TOKEN env.", + ) + extract_parser.add_argument("--jwt", required=False, help=argparse.SUPPRESS) + extract_parser.add_argument("--username", required=False, help="Login username (alternative to --token)") extract_parser.add_argument("--password", required=False, help="Login password (will be prompted if omitted)") extract_parser.add_argument("--message-ids", required=False, type=int, nargs="+", help="Message IDs to extract") extract_parser.add_argument("--out-dir", required=False, help="Directory to write the extracted bundle") @@ -422,6 +463,7 @@ def _run_full_interactive() -> None: server=None, https=False, http=False, + token=None, jwt=None, username=None, password=None, diff --git a/scripts/compliance/decryption/crypto.py b/scripts/compliance/decryption/crypto.py index ef57267..5678580 100644 --- a/scripts/compliance/decryption/crypto.py +++ b/scripts/compliance/decryption/crypto.py @@ -114,27 +114,40 @@ def decrypt_message(envelope_data: Dict[str, Any], compliance_private_key: X2551 return plaintext.decode("utf-8") -def decrypt_file_bytes_from_meta(meta: Dict[str, Any], encrypted_bytes: bytes, *, key_file: str = "compliance_keypair.txt") -> bytes: +GCM_TAG_SIZE = 16 + + +def _unwrap_mek_from_meta(meta: Dict[str, Any], *, key_file: str) -> bytes: nonce_b64 = get_str(meta, keys=["nonce_b64", "iv_b64", "nonce", "iv"], label="nonce/iv (base64)") - nonce = base64.b64decode(nonce_b64) + _ = base64.b64decode(nonce_b64) # validate early mek_key = first_present_key(meta, ["compliance_wrapped_mek_b64", "compliance_wrapped_mek"]) if mek_key: compliance_private_key = load_compliance_private_key(key_file=key_file) compliance_public_key = compliance_private_key.public_key() - mek = decrypt_compliance_mek(str(meta[mek_key]), compliance_private_key, compliance_public_key) - else: - wrap_public_key_b64 = get_str( - meta, - keys=["wrap_public_key_b64", "wrap_public_key", "public_key_b64"], - label="wrap public key (base64)", - ) - wrap_context = get_str(meta, keys=["wrap_context"], label="wrap context") - wrapped_mek_b64 = get_str(meta, keys=["wrapped_mek_b64", "wrapped_mek"], label="wrapped MEK (base64)") - mek = decrypt_wrapped_mek_with_public_key(wrapped_mek_b64, wrap_public_key_b64, wrap_context) + return decrypt_compliance_mek(str(meta[mek_key]), compliance_private_key, compliance_public_key) - aesgcm = AESGCM(mek) - return aesgcm.decrypt(nonce, encrypted_bytes, None) + wrap_public_key_b64 = get_str( + meta, + keys=["wrap_public_key_b64", "wrap_public_key", "public_key_b64"], + label="wrap public key (base64)", + ) + wrap_context = get_str(meta, keys=["wrap_context"], label="wrap context") + wrapped_mek_b64 = get_str(meta, keys=["wrapped_mek_b64", "wrapped_mek"], label="wrapped MEK (base64)") + return decrypt_wrapped_mek_with_public_key(wrapped_mek_b64, wrap_public_key_b64, wrap_context) + + +def decrypt_file_bytes_from_meta(meta: Dict[str, Any], encrypted_bytes: bytes, *, key_file: str = "compliance_keypair.txt") -> bytes: + """ + Decrypt file ciphertext from [encrypt_message_to_file]: ``ciphertext || tag`` (tag last 16 bytes). + """ + if len(encrypted_bytes) < GCM_TAG_SIZE: + raise ValueError("Encrypted file is too short (missing GCM tag)") + + nonce_b64 = get_str(meta, keys=["nonce_b64", "iv_b64", "nonce", "iv"], label="nonce/iv (base64)") + nonce = base64.b64decode(nonce_b64) + mek = _unwrap_mek_from_meta(meta, key_file=key_file) + return AESGCM(mek).decrypt(nonce, encrypted_bytes, None) def derive_auth_secret(username: str, password: str) -> str: