diff --git a/.cursor/commands/security-audit.md b/.cursor/commands/security-audit.md index 4e811a0..f33471c 100644 --- a/.cursor/commands/security-audit.md +++ b/.cursor/commands/security-audit.md @@ -5,6 +5,7 @@ Perform a comprehensive security audit of the FromChat application codebase. ## Project Context **FromChat** is a 100% open source secure messaging application with: + - React/TypeScript frontend - Python FastAPI backend - End-to-end encryption for DMs and calls @@ -17,43 +18,36 @@ Perform a comprehensive security audit of the FromChat application codebase. When auditing, remember these are **intentional design choices**: 1. **Public messages endpoint** - Open forum accessible without authentication (by design) - - The public chat is meant to be an open forum - - Private DMs are properly E2E encrypted and require authentication - + - The public chat is meant to be an open forum + - Private DMs are properly E2E encrypted and require authentication 2. **Public user list** - All users visible in DMs tab (by design) - - Users can see all registered accounts - - This is intentional for a community-based chat app - + - Users can see all registered accounts + - This is intentional for a community-based chat app 3. **XSS protection** - Multi-layer defense already implemented: - - React auto-escaping - - DOMPurify for sanitization - - Caddy CSP headers - - Do NOT flag localStorage key storage as critical (already well-protected) - + - React auto-escaping + - DOMPurify for sanitization + - Caddy CSP headers + - Do NOT flag localStorage key storage as critical (already well-protected) 4. **File upload security** - Docker isolation in place: - - Server runs in Docker without executable flags - - Files cannot execute on server - - PIL re-encodes images - - Do NOT flag Content-Type validation as critical - + - Server runs in Docker without executable flags + - Files cannot execute on server + - PIL re-encodes images + - Do NOT flag Content-Type validation as critical 5. **CSRF protection** - Not needed: - - No cookies used - - JWT tokens in Authorization headers only - - CSRF attacks don't apply to this auth model - + - No cookies used + - JWT tokens in Authorization headers only + - CSRF attacks don't apply to this auth model 6. **Beta domain CSP** - 'unsafe-inline' is required: - - Beta domain (beta.fromchat.ru) points to development machine - - Vite dev server requires 'unsafe-inline' to function - - Production domain has strict CSP - + - Beta domain (beta.fromchat.ru) points to development machine + - Vite dev server requires 'unsafe-inline' to function + - Production domain has strict CSP 7. **Security logging** - Already implemented: - - All events are logged including security-related activity - - Do NOT flag as missing - + - All events are logged including security-related activity + - Do NOT flag as missing 8. **100% Open Source** - This is a security strength: - - Full transparency - - Community review capability - - No hidden backdoors + - Full transparency + - Community review capability + - No hidden backdoors ## Android App @@ -63,88 +57,6 @@ When auditing, remember these are **intentional design choices**: The application runs behind Caddy reverse proxy with comprehensive security controls: -### Caddyfile Configuration - -```caddyfile -fromchat.ru { - reverse_proxy 172.18.0.1:8301 host.docker.internal:8301 172.17.0.1:8301 { - lb_policy first - } - - # Security headers - header { - X-XSS-Protection "1; mode=block" # Prevent XSS attacks - X-Content-Type-Options "nosniff" # Prevent MIME type sniffing - X-Frame-Options "DENY" # Prevent clickjacking - Referrer-Policy "strict-origin-when-cross-origin" - Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https: blob:; font-src 'self' data:; connect-src 'self'; frame-ancestors 'none';" - Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" - Permissions-Policy "geolocation=(), microphone=(self), camera=(self)" - } - - rate_limit { - zone global { - key {remote_ip} - window 1m - burst 20 - events 500 - } - } - - handle_errors { - @errors { - expression {err.status_code} >= 400 - } - - handle @errors { - rewrite * /{err.status_code} - reverse_proxy https://http.cat { - header_up Host {upstream_hostport} - replace_status {err.status_code} - } - } - } -} - -beta.fromchat.ru { - reverse_proxy 95.165.0.162:8301 - - # Security headers - header { - X-XSS-Protection "1; mode=block" # Prevent XSS attacks - X-Content-Type-Options "nosniff" # Prevent MIME type sniffing - X-Frame-Options "DENY" # Prevent clickjacking - Referrer-Policy "strict-origin-when-cross-origin" - Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https: blob:; font-src 'self' data:; connect-src 'self'; frame-ancestors 'none';" - Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" - Permissions-Policy "geolocation=(), microphone=(self), camera=(self)" - } - - rate_limit { - zone global { - key {remote_ip} - window 1m - burst 20 - events 1000 - } - } - - handle_errors { - @errors { - expression {err.status_code} >= 400 - } - - handle @errors { - rewrite * /{err.status_code} - reverse_proxy https://http.cat { - header_up Host {upstream_hostport} - replace_status {err.status_code} - } - } - } -} -``` - ### Key Infrastructure Protections - ✅ **HTTPS enforcement** - Automatic SSL/TLS with Caddy @@ -193,6 +105,7 @@ Provide a **clean, concise report** with: ## Common False Positives to Avoid ❌ **DO NOT FLAG THESE AS ISSUES:** + - Public messages endpoint (intentional) - Username enumeration (users list is public by design) - Keys in localStorage (XSS is well-protected) @@ -205,6 +118,7 @@ Provide a **clean, concise report** with: ## Key Security Features to Verify ✅ **MUST CHECK:** + - CORS configuration in backend/app.py - Password validation in backend/validation.py - JWT token generation and validation diff --git a/.cursor/commands/fix-typecheck-errors.md b/.cursor/commands/typecheck.md similarity index 100% rename from .cursor/commands/fix-typecheck-errors.md rename to .cursor/commands/typecheck.md diff --git a/.gitignore b/.gitignore index fac61db..98b7a25 100644 --- a/.gitignore +++ b/.gitignore @@ -580,4 +580,4 @@ backend/alembic/** .cursor/plans tmp compliance_keypair.txt -backend/files \ No newline at end of file +backend/files diff --git a/backend/services/main/routes/envelope_messaging.py b/backend/services/main/routes/envelope_messaging.py index c01405e..e7e48b7 100644 --- a/backend/services/main/routes/envelope_messaging.py +++ b/backend/services/main/routes/envelope_messaging.py @@ -10,9 +10,11 @@ Handles: import logging import json +import time from datetime import datetime from pathlib import Path from typing import Optional +import httpx from fastapi import APIRouter, Depends, HTTPException, status, Request from sqlalchemy.orm import Session from pydantic import BaseModel, Field @@ -26,10 +28,26 @@ from ..service_calls import ( get_compliance_public_key, process_message_with_files_in_messaging_service, store_encrypted_file, + init_resumable_upload_in_storage, + get_resumable_upload_status_in_storage, + upload_resumable_chunk_in_storage, + complete_resumable_upload_in_storage, + get_resumable_upload_data_in_storage, + delete_resumable_upload_in_storage, ) from .messaging import messagingManager, convert_dm_envelope logger = logging.getLogger("uvicorn.error") + + +def _compliance_public_key_required() -> bool: + try: + from services.shared.message_retention import get_message_retention + except ImportError: + from backend.services.shared.message_retention import get_message_retention # type: ignore + return not get_message_retention().never_store_compliance_mek() + + router = APIRouter(prefix="/dm", tags=["Direct Messages"]) @@ -51,8 +69,10 @@ class SendEncryptedMessageRequest(BaseModel): transport_ciphertext_b64: str sender_public_key_b64: str recipient_public_key_b64: str + client_message_id: Optional[str] = None reply_to_id: Optional[int] = None files: list[FileModel] = Field(default_factory=list, alias="transport_files") + uploaded_file_ids: list[str] = Field(default_factory=list, alias="uploaded_file_ids") class Config: allow_population_by_field_name = True @@ -67,17 +87,29 @@ class EditEncryptedMessageRequest(BaseModel): recipient_public_key_b64: str +class InitResumableUploadRequest(BaseModel): + filename: str + total_size: int + recipient_id: int + chunk_size: Optional[int] = None + + +class UploadChunkRequest(BaseModel): + offset: int + data_b64: str + + # ============================================================================ # Key Management Endpoint # ============================================================================ @router.get("/key/transport/public") -async def get_transport_public_key_endpoint(): +async def get_transport_public_key_endpoint(request: Request): """ Get the current messaging service ephemeral transport public key. - + Clients use this key to encrypt their messages with X25519 + ChaCha20-Poly1305. - + Returns: { "key_id": "key-identifier", @@ -85,8 +117,11 @@ async def get_transport_public_key_endpoint(): "created_at": "unix-timestamp" } """ + client_ip = getattr(request.client, 'host', 'unknown') if request.client else 'unknown' + try: - return await get_messaging_transport_public_key() + result = await get_messaging_transport_public_key() + return result except Exception as e: logger.error("Failed to fetch transport public key: %s", e) raise HTTPException( @@ -95,27 +130,64 @@ async def get_transport_public_key_endpoint(): ) -@router.get("/key/compliance/public") -async def get_compliance_public_key_endpoint(): - """ - Get the compliance system public key (for MEK wrapping). +@router.post("/upload/init") +async def init_resumable_upload( + request: InitResumableUploadRequest, + current_user: User = Depends(get_current_user), +): + if request.total_size <= 0: + raise HTTPException(status_code=400, detail="total_size must be > 0") + + if current_user.id == request.recipient_id: + raise HTTPException(status_code=400, detail="Cannot send files to yourself") + + payload = await init_resumable_upload_in_storage( + filename=request.filename, + total_size=request.total_size, + allowed_user_ids=[current_user.id, request.recipient_id], + chunk_size=request.chunk_size, + ) + return payload + + +@router.get("/upload/{upload_id}") +async def get_resumable_upload_status( + upload_id: str, + current_user: User = Depends(get_current_user), +): + return await get_resumable_upload_status_in_storage(upload_id, current_user.id) + + +@router.patch("/upload/{upload_id}") +async def upload_resumable_chunk( + upload_id: str, + request: UploadChunkRequest, + current_user: User = Depends(get_current_user), +): + return await upload_resumable_chunk_in_storage( + upload_id=upload_id, + user_id=current_user.id, + offset=request.offset, + data_b64=request.data_b64, + ) + + +@router.post("/upload/{upload_id}/complete") +async def complete_resumable_upload( + upload_id: str, + current_user: User = Depends(get_current_user), +): + return await complete_resumable_upload_in_storage(upload_id, current_user.id) + + +@router.delete("/upload/{upload_id}") +async def delete_resumable_upload( + upload_id: str, + current_user: User = Depends(get_current_user), +): + return await delete_resumable_upload_in_storage(upload_id, current_user.id) - This key is generated offline on an air-gapped machine and used to wrap MEKs - so the compliance system can decrypt archived messages for audit. - Returns: - { - "public_key_b64": "base64-encoded-key" - } - """ - try: - return await get_compliance_public_key() - except Exception as e: - logger.error("Failed to fetch compliance public key: %s", e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Failed to fetch compliance key" - ) # ============================================================================ @@ -170,13 +242,33 @@ async def send_encrypted_message( # Fetch compliance public key and process through messaging service compliance_key_response = await get_compliance_public_key() - compliance_public_key_b64 = compliance_key_response.get("public_key_b64") - if not compliance_public_key_b64: + compliance_public_key_b64 = compliance_key_response.get("public_key_b64") or "" + if _compliance_public_key_required() and not compliance_public_key_b64: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to retrieve compliance key" ) + all_transport_files: list[dict[str, object]] = [ + { + "encrypted_file_data_b64": f.encrypted_file_data_b64, + "filename": f.filename, + "file_size": f.file_size, + } + for f in request.files + ] + + for upload_id in request.uploaded_file_ids: + uploaded_payload = await get_resumable_upload_data_in_storage(upload_id, current_user.id) + all_transport_files.append( + { + "encrypted_file_data_b64": uploaded_payload["encrypted_file_data_b64"], + "filename": uploaded_payload["filename"], + "file_size": uploaded_payload["file_size"], + "upload_id": upload_id, + } + ) + processed = await process_message_with_files_in_messaging_service( client_public_key_b64=request.client_public_key_b64, transport_nonce_b64=request.transport_nonce_b64, @@ -184,7 +276,13 @@ async def send_encrypted_message( compliance_public_key_b64=compliance_public_key_b64, sender_public_key_b64=request.sender_public_key_b64, recipient_public_key_b64=request.recipient_public_key_b64, - transport_files=[{"encrypted_file_data_b64": f.encrypted_file_data_b64} for f in request.files], + transport_files=[ + { + "encrypted_file_data_b64": str(f["encrypted_file_data_b64"]), + "filename": str(f.get("filename", "file")), + } + for f in all_transport_files + ], ) logger.info( @@ -213,14 +311,14 @@ async def send_encrypted_message( # We persist per-file nonce (for AES-GCM) but do not persist per-file wrapped MEKs. try: file_results: list[dict] = processed.get("files", []) or [] - if len(file_results) != len(request.files): + if len(file_results) != len(all_transport_files): raise HTTPException(status_code=500, detail="File processing count mismatch") - for i, tf in enumerate(request.files): + for i, tf in enumerate(all_transport_files): fr = file_results[i] file_storage_result = await store_encrypted_file( encrypted_file_data_b64=fr["ciphertext"], - filename=tf.filename, + filename=str(tf["filename"]), content_type="application/octet-stream", sender_id=current_user.id, recipient_id=request.recipient_id, @@ -231,12 +329,18 @@ async def send_encrypted_message( sender_id=current_user.id, recipient_id=dm_envelope.recipient_id, path=file_storage_result.get("path") or f"/uploads/files/encrypted/{file_storage_result['file_id']}", - name=Path(tf.filename).name, + name=Path(str(tf["filename"])).name, nonce_b64=fr["nonce"], ) db.add(df) db.commit() + for upload_id in request.uploaded_file_ids: + try: + await delete_resumable_upload_in_storage(upload_id, current_user.id) + except Exception as cleanup_error: + logger.warning("Failed to cleanup resumable upload %s: %s", upload_id, cleanup_error) + except HTTPException: raise except Exception: @@ -257,6 +361,8 @@ async def send_encrypted_message( await messagingManager.send_update_to_user(dm_envelope.recipient_id, "dmNew", recipient_payload, db) sender_payload = convert_dm_envelope(db, dm_envelope, dm_envelope.sender_id) + if request.client_message_id: + sender_payload["client_message_id"] = request.client_message_id await messagingManager.send_update_to_user(dm_envelope.sender_id, "dmNew", sender_payload, db) return { @@ -264,6 +370,7 @@ async def send_encrypted_message( "sender_id": dm_envelope.sender_id, "recipient_id": dm_envelope.recipient_id, "timestamp": dm_envelope.timestamp.isoformat(), + "client_message_id": request.client_message_id, "reply_to_id": dm_envelope.reply_to_id, } @@ -296,47 +403,32 @@ async def extract_message_for_compliance( to an air-gapped machine for decryption using the compliance private key. """ # Log compliance access attempt - client_ip = getattr(request.client, "host", "unknown") if request.client else "unknown" - log_security( - "compliance_access_attempt", - "warning", - username=current_user.username, - user_id=current_user.id, - message_id=message_id, - ip=client_ip, - ) + client_ip = getattr(request.client, 'host', 'unknown') if request.client else 'unknown' + log_security("compliance_access_attempt", "warning", + username=current_user.username, user_id=current_user.id, + message_id=message_id, ip=client_ip) # Security check: only user ID 1 can access this if current_user.id != 1: - log_security( - "compliance_access_denied", - "error", - username=current_user.username, - user_id=current_user.id, - message_id=message_id, - ip=client_ip, - reason="Unauthorized user (compliance officer access required)", - ) + log_security("compliance_access_denied", "error", + username=current_user.username, user_id=current_user.id, + message_id=message_id, ip=client_ip, + reason="Unauthorized user (compliance officer access required)") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail="Access denied. This endpoint is restricted to compliance officers.", + detail="Access denied. This endpoint is restricted to compliance officers." ) # Find the message envelope = db.query(DMEnvelope).filter(DMEnvelope.id == message_id).first() if not envelope: - log_security( - "compliance_access_failed", - "warning", - username=current_user.username, - user_id=current_user.id, - message_id=message_id, - ip=client_ip, - reason="Message not found", - ) + log_security("compliance_access_failed", "warning", + username=current_user.username, user_id=current_user.id, + message_id=message_id, ip=client_ip, + reason="Message not found") raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail="Message not found", + detail="Message not found" ) # Get sender and recipient usernames for logging @@ -378,8 +470,6 @@ async def extract_message_for_compliance( "edited_by_username": edited_by_user.username if edited_by_user else "unknown", "previous_ciphertext_b64": edit_entry.previous_ciphertext_b64, "previous_iv_b64": edit_entry.previous_iv_b64, - "previous_sender_wrapped_mek_b64": edit_entry.previous_sender_wrapped_mek_b64, - "previous_recipient_wrapped_mek_b64": edit_entry.previous_recipient_wrapped_mek_b64, "previous_compliance_wrapped_mek_b64": edit_entry.previous_compliance_wrapped_mek_b64, }) @@ -396,21 +486,14 @@ async def extract_message_for_compliance( "total_edits": len(edit_history_data), "extraction_timestamp": datetime.now().isoformat(), "extracted_by_user_id": current_user.id, - "compliance_system_ready": envelope.compliance_wrapped_mek_b64 is not None, + "compliance_system_ready": envelope.compliance_wrapped_mek_b64 is not None } - log_security( - "compliance_extraction_success", - "info", - username=current_user.username, - user_id=current_user.id, - message_id=message_id, - sender_id=envelope.sender_id, - recipient_id=envelope.recipient_id, - sender_username=sender_username, - recipient_username=recipient_username, - ip=client_ip, - ) + log_security("compliance_extraction_success", "info", + username=current_user.username, user_id=current_user.id, + message_id=message_id, sender_id=envelope.sender_id, + recipient_id=envelope.recipient_id, sender_username=sender_username, + recipient_username=recipient_username, ip=client_ip) return { "status": "success", @@ -419,8 +502,8 @@ async def extract_message_for_compliance( "instructions": [ "Transfer this data to an air-gapped machine", "Use compliance_decryption.py decrypt --input-file ", - "Keep the compliance private key offline at all times", - ], + "Keep the compliance private key offline at all times" + ] } @@ -475,7 +558,7 @@ async def get_encrypted_conversation( detail="User not found" ) - # Fetch messages in both directions, sorted by timestamp + # Fetch messages in both directions, sorted by timestamp (exclude deleted) messages = ( db.query(DMEnvelope) .filter( @@ -486,7 +569,8 @@ async def get_encrypted_conversation( | ( (DMEnvelope.sender_id == other_user_id) & (DMEnvelope.recipient_id == current_user.id) - ) + ), + DMEnvelope.deleted_at.is_(None) # Exclude soft-deleted messages ) .order_by(DMEnvelope.timestamp.desc()) .limit(limit) @@ -572,9 +656,10 @@ async def get_owner_compliance_view( ) try: - # Fetch all messages + # Fetch all non-deleted messages messages = ( db.query(DMEnvelope) + .filter(DMEnvelope.deleted_at.is_(None)) # Exclude soft-deleted messages .order_by(DMEnvelope.timestamp.desc()) .all() ) @@ -679,8 +764,6 @@ async def get_dm_edit_history_for_compliance( "dm_envelope_id": entry.message_id, "previous_ciphertext_b64": entry.previous_ciphertext_b64, "previous_iv_b64": entry.previous_iv_b64, - "previous_sender_wrapped_mek_b64": entry.previous_sender_wrapped_mek_b64, - "previous_recipient_wrapped_mek_b64": entry.previous_recipient_wrapped_mek_b64, "previous_compliance_wrapped_mek_b64": entry.previous_compliance_wrapped_mek_b64, "edited_at": entry.edited_at.isoformat(), "edited_by_username": edited_by_user.username if edited_by_user else "unknown", @@ -757,7 +840,10 @@ async def edit_encrypted_message( """ try: # Find the message - msg = db.query(DMEnvelope).filter(DMEnvelope.id == message_id).first() + msg = db.query(DMEnvelope).filter( + DMEnvelope.id == message_id, + DMEnvelope.deleted_at.is_(None) # Can't edit deleted messages + ).first() if not msg: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -773,8 +859,8 @@ async def edit_encrypted_message( # Fetch compliance public key and process through messaging service compliance_key_response = await get_compliance_public_key() - compliance_public_key_b64 = compliance_key_response.get("public_key_b64") - if not compliance_public_key_b64: + compliance_public_key_b64 = compliance_key_response.get("public_key_b64") or "" + if _compliance_public_key_required() and not compliance_public_key_b64: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to retrieve compliance key" @@ -791,22 +877,11 @@ async def edit_encrypted_message( transport_files=[], # No file support for edits currently ) - # Store edit history in compliance storage before updating - edit_history = DMEditHistory( - message_id=msg.id, - dm_envelope_id=msg.id, # Match existing DB schema - previous_ciphertext_b64=msg.ciphertext_b64, - previous_iv_b64=msg.iv_b64, - previous_sender_wrapped_mek_b64=msg.sender_wrapped_mek_b64, - previous_recipient_wrapped_mek_b64=msg.recipient_wrapped_mek_b64, - previous_compliance_wrapped_mek_b64=msg.compliance_wrapped_mek_b64 or "", - edited_by=current_user.id, - edited_by_user_id=current_user.id # Match existing DB schema - ) - db.add(edit_history) - - # Update the message with new processed content + # Update the message with new processed content (commit first so edit always succeeds) processed_msg = processed["message"] + prev_ciphertext = msg.ciphertext_b64 + prev_iv = msg.iv_b64 + prev_wrapped_mek = msg.compliance_wrapped_mek_b64 or "" msg.ciphertext_b64 = processed_msg["ciphertext"] msg.iv_b64 = processed_msg["nonce"] msg.sender_wrapped_mek_b64 = processed["sender_wrapped_mek"] @@ -817,6 +892,26 @@ async def edit_encrypted_message( db.commit() db.refresh(msg) + # Best-effort: store edit history for compliance (table may not exist yet) + try: + edit_history = DMEditHistory( + message_id=msg.id, + dm_envelope_id=msg.id, + previous_ciphertext_b64=prev_ciphertext, + previous_iv_b64=prev_iv, + previous_compliance_wrapped_mek_b64=prev_wrapped_mek, + edited_by=current_user.id, + edited_by_user_id=current_user.id, + ) + db.add(edit_history) + db.commit() + except Exception as history_err: + db.rollback() + logger.warning( + "Could not store DM edit history (table dm_edit_history may not exist): %s", + history_err, + ) + logger.info( "Edited encrypted message msg_id=%s by user_id=%s", message_id, @@ -884,7 +979,9 @@ async def delete_encrypted_message( detail="Cannot delete others' messages" ) - db.delete(msg) + # Soft delete: set deleted_at timestamp instead of hard delete + from datetime import datetime + msg.deleted_at = datetime.now() db.commit() logger.info( diff --git a/backend/services/main/routes/messaging.py b/backend/services/main/routes/messaging.py index 9d40641..a57a0d3 100644 --- a/backend/services/main/routes/messaging.py +++ b/backend/services/main/routes/messaging.py @@ -10,12 +10,9 @@ import time import unicodedata from collections import defaultdict, deque from difflib import SequenceMatcher -from types import SimpleNamespace from typing import Any import httpx from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect, UploadFile, File, Form, Request, status -from fastapi.responses import FileResponse -from fastapi.security import HTTPAuthorizationCredentials from sqlalchemy.orm import Session from ..dependencies import get_current_user, get_db from .account import convert_user @@ -55,7 +52,6 @@ def _get_file_storage_url() -> str: or "http://127.0.0.1:8302" ) - _SPAM_WINDOW_SECONDS = 45 _SPAM_SIMILARITY_THRESHOLD = 0.88 _SPAM_MESSAGE_LIMIT = 5 @@ -628,135 +624,6 @@ async def mark_messages_read(request: Request, read_request: MarkReadRequest, cu return {"status": "success", "updated": int(updated_count)} -@router.post("/dm/send-legacy") -@rate_limit_per_ip("20/minute") -async def dm_send( - request: Request, - payload: dict | None = None, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), - # Multipart support - dm_payload: str | None = Form(default=None), - files: list[UploadFile] = File(default=[]), - fileNames: str | None = Form(default=None), # JSON array of filenames corresponding to files -): - if dm_payload and payload is None: - try: - payload = json.loads(dm_payload) - except Exception: - raise HTTPException(status_code=400, detail="Invalid dm_payload JSON") - - if payload is None: - raise HTTPException(status_code=400, detail="Missing payload") - - required = ["recipientId", "iv", "ciphertext", "salt", "iv2", "wrappedMk"] - for key in required: - if key not in payload: - raise HTTPException(status_code=400, detail=f"Missing {key}") - - try: - recipient_id = int(payload["recipientId"]) - except (ValueError, TypeError): - raise HTTPException(status_code=400, detail="Invalid recipientId") - - if recipient_id <= 0: - raise HTTPException(status_code=400, detail="Invalid recipientId") - - if recipient_id == current_user.id: - raise HTTPException(status_code=400, detail="Cannot send DM to yourself") - - # Verify recipient exists - recipient = db.query(User).filter(User.id == recipient_id).first() - if not recipient or recipient.deleted or recipient.suspended: - raise HTTPException(status_code=404, detail="Recipient not found") - - env = DMEnvelope( - sender_id=current_user.id, - recipient_id=recipient_id, - iv_b64=payload["iv"], - ciphertext_b64=payload["ciphertext"], - sender_wrapped_mek_b64=payload.get("wrappedMk", ""), - recipient_wrapped_mek_b64=payload.get("wrappedMk", ""), - reply_to_id=payload.get("replyToId") if isinstance(payload.get("replyToId"), int) else None, - ) - db.add(env) - db.commit() - db.refresh(env) - - # Save encrypted files if any (no processing) - if files: - # Validate total size - total_size = 0 - for file in files: - if hasattr(file, "size") and file.size is not None: - total_size += int(file.size) - else: - data = await file.read() - file.file.seek(0) - total_size += len(data) - if total_size > MAX_TOTAL_SIZE: - raise HTTPException(status_code=400, detail="Total attachments size exceeds 4GB") - - names: list[str] = [] - if fileNames: - try: - decoded = json.loads(fileNames) - if isinstance(decoded, list): - names = [str(x) for x in decoded] - except Exception: - names = [] - - for i, file in enumerate(files): - provided = names[i] if i < len(names) else None - # Sanitize provided name to avoid path traversal - if provided and not re.match(r"^[A-Za-z0-9._-]{1,200}$", provided): - provided = None - original_name = provided or Path(file.filename or "file").name - # Save using provided/original name to allow client to reference path directly - safe_name = uid = uuid.uuid4().hex - out_name = f"{current_user.id}_{env.recipient_id}_{env.id}_{safe_name}" - out_path = FILES_ENCRYPTED_DIR / out_name - - content = await file.read() - with open(out_path, "wb") as f: - f.write(content) - - # Save DM file record - df = DMFile( - message_id=env.id, - sender_id=current_user.id, - recipient_id=env.recipient_id, - path=f"/api/uploads/files/encrypted/{out_name}", - name=original_name - ) - db.add(df) - db.commit() - db.refresh(env) - - # Send push notification for DM - try: - await push_service.send_dm_notification(db, env, current_user) - except Exception as e: - logger.error(f"Failed to send push notification for DM {env.id}: {e}") - - # Send user-specific WebSocket updates (each user gets only their MEK) - recipient_payload = convert_dm_envelope(db, env, env.recipient_id) - await messagingManager.send_update_to_user(env.recipient_id, "dmNew", recipient_payload, db) - - sender_payload = convert_dm_envelope(db, env, env.sender_id) - await messagingManager.send_update_to_user(env.sender_id, "dmNew", sender_payload, db) - - log_dm( - "message_sent", - dm_envelope_id=env.id, - sender_id=current_user.id, - sender_username=current_user.username, - recipient_id=env.recipient_id, - attachment_count=len(env.files or []), - reply_to=env.reply_to_id, - ) - - return {"status": "ok", "id": env.id} @router.get("/dm/fetch") @@ -1116,14 +983,23 @@ class MessaggingSocketManager: if websocket.client_state.name == "CONNECTED": await websocket.send_json({"type": type, "error": {"code": e.status_code, "detail": e.detail}}) - async def _get_next_sequence(self, user_id: int) -> int: + async def _get_next_sequence(self, user_id: int, db: Session | None = None) -> int: """Get the next sequence number for a user (shared across all their connections) - thread-safe""" if user_id not in self._sequence_lock: self._sequence_lock[user_id] = asyncio.Lock() - + async with self._sequence_lock[user_id]: if user_id not in self.sequence_numbers: - self.sequence_numbers[user_id] = 0 + # Initialize from database to avoid conflicts on restart + if db: + try: + from ..models import UpdateLog + latest = db.query(UpdateLog).filter(UpdateLog.user_id == user_id).order_by(UpdateLog.sequence.desc()).first() + self.sequence_numbers[user_id] = latest.sequence if latest else 0 + except Exception: + self.sequence_numbers[user_id] = 0 + else: + self.sequence_numbers[user_id] = 0 self.sequence_numbers[user_id] += 1 return self.sequence_numbers[user_id] @@ -1226,7 +1102,7 @@ class MessaggingSocketManager: logger.warning(f"Attempted to flush updates for unauthenticated websocket, skipping") return - seq = await self._get_next_sequence(user_id) + seq = await self._get_next_sequence(user_id, db) # Store updates in database for gap detection (only once per user per sequence) if db: @@ -1549,12 +1425,14 @@ async def chat_websocket( # File serving proxy endpoints # Proxy file requests to file_storage service +import httpx + @router.api_route("/uploads/files/normal/{filename:path}", methods=["GET"]) async def proxy_normal_file( request: Request, filename: str, - current_user: User = Depends(get_current_user), + current_user: User = Depends(get_current_user) ): """Proxy file requests to file_storage service.""" mod = service_calls._get_file_storage_module() @@ -1574,12 +1452,11 @@ async def proxy_normal_file( try: response = await client.get(target_url, headers=headers) from fastapi.responses import Response - return Response( content=response.content, status_code=response.status_code, headers=dict(response.headers), - media_type=response.headers.get("content-type"), + media_type=response.headers.get("content-type") ) except httpx.RequestError as e: logger.error("Failed to proxy file request: %s", e) @@ -1592,15 +1469,15 @@ async def test_proxy(): file_storage_url = _get_file_storage_url() target_url = f"{file_storage_url}/health" - logger.info("Testing proxy to: %s", target_url) + logger.info(f"Testing proxy to: {target_url}") async with httpx.AsyncClient(timeout=10.0) as client: try: response = await client.get(target_url, follow_redirects=False) - logger.info("Test proxy response: %s", response.status_code) + logger.info(f"Test proxy response: {response.status_code}") return {"status": "ok", "response_code": response.status_code} except Exception as e: - logger.error("Test proxy failed: %s", e) + logger.error(f"Test proxy failed: {e}") return {"status": "error", "error": str(e)} @@ -1608,7 +1485,7 @@ async def test_proxy(): async def proxy_encrypted_file( request: Request, filename: str, - current_user: User = Depends(get_current_user), + current_user: User = Depends(get_current_user) ): """Proxy file requests to file_storage service.""" mod = service_calls._get_file_storage_module() @@ -1629,12 +1506,11 @@ async def proxy_encrypted_file( try: response = await client.get(target_url, headers=headers, follow_redirects=False) from fastapi.responses import Response - return Response( content=response.content, status_code=response.status_code, headers=dict(response.headers), - media_type=response.headers.get("content-type"), + media_type=response.headers.get("content-type") ) except Exception as e: logger.error("Failed to proxy file request: %s", e) @@ -1646,7 +1522,7 @@ async def get_message_edit_history_for_compliance( request: Request, message_id: int, current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), + db: Session = Depends(get_db) ): """ Get complete edit history for a public message (compliance access only). @@ -1663,72 +1539,58 @@ async def get_message_edit_history_for_compliance( Returns: Complete edit history for the message """ - client_ip = getattr(request.client, "host", "unknown") if request.client else "unknown" + client_ip = getattr(request.client, 'host', 'unknown') if request.client else 'unknown' # Log compliance access attempt - log_security( - "message_edit_history_access_attempt", - "warning", - user_id=current_user.id, - username=current_user.username, - ip=client_ip, - message_id=message_id, - ) + log_security("message_edit_history_access_attempt", "warning", + user_id=current_user.id, + username=current_user.username, + ip=client_ip, + message_id=message_id) # Only user_id 1 (compliance officer) can access if current_user.id != 1: - log_security( - "message_edit_history_access_denied", - "error", - user_id=current_user.id, - username=current_user.username, - ip=client_ip, - reason="Unauthorized user (compliance officer access required)", - ) + log_security("message_edit_history_access_denied", "error", + user_id=current_user.id, + username=current_user.username, + ip=client_ip, + reason="Unauthorized user (compliance officer access required)") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail="Access denied. This endpoint is restricted to compliance officers.", + detail="Access denied. This endpoint is restricted to compliance officers." ) try: # Get the original message message = db.query(Message).filter(Message.id == message_id).first() if not message: - log_security( - "message_edit_history_access_failed", - "warning", - user_id=current_user.id, - ip=client_ip, - message_id=message_id, - reason="Message not found", - ) + log_security("message_edit_history_access_failed", "warning", + user_id=current_user.id, + ip=client_ip, + message_id=message_id, + reason="Message not found") raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail="Message not found", + detail="Message not found" ) # Get edit history - edit_history = ( - db.query(MessageEditHistory) - .filter(MessageEditHistory.message_id == message_id) - .order_by(MessageEditHistory.edited_at) - .all() - ) + edit_history = db.query(MessageEditHistory).filter( + MessageEditHistory.message_id == message_id + ).order_by(MessageEditHistory.edited_at).all() # Convert to response format history_entries = [] for entry in edit_history: edited_by_user = db.query(User).filter(User.id == entry.edited_by_user_id).first() - history_entries.append( - { - "id": entry.id, - "message_id": entry.message_id, - "previous_content": entry.previous_content, - "edited_at": entry.edited_at.isoformat(), - "edited_by_username": edited_by_user.username if edited_by_user else "unknown", - "edited_by_user_id": entry.edited_by_user_id, - } - ) + history_entries.append({ + "id": entry.id, + "message_id": entry.message_id, + "previous_content": entry.previous_content, + "edited_at": entry.edited_at.isoformat(), + "edited_by_username": edited_by_user.username if edited_by_user else "unknown", + "edited_by_user_id": entry.edited_by_user_id + }) # Current message data current_data = { @@ -1736,25 +1598,22 @@ async def get_message_edit_history_for_compliance( "content": message.content, "user_id": message.user_id, "timestamp": message.timestamp.isoformat(), - "is_edited": message.is_edited, + "is_edited": message.is_edited } result = { "message_id": message_id, "current_version": current_data, "edit_history": history_entries, - "total_edits": len(history_entries), + "total_edits": len(history_entries) } - log_security( - "message_edit_history_access_success", - "info", - user_id=current_user.id, - username=current_user.username, - ip=client_ip, - message_id=message_id, - edit_count=len(history_entries), - ) + log_security("message_edit_history_access_success", "info", + user_id=current_user.id, + username=current_user.username, + ip=client_ip, + message_id=message_id, + edit_count=len(history_entries)) return result @@ -1762,15 +1621,12 @@ async def get_message_edit_history_for_compliance( raise except Exception as e: logger.exception("Error retrieving message edit history: %s", e) - log_security( - "message_edit_history_access_error", - "error", - user_id=current_user.id, - ip=client_ip, - message_id=message_id, - error=str(e), - ) + log_security("message_edit_history_access_error", "error", + user_id=current_user.id, + ip=client_ip, + message_id=message_id, + error=str(e)) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Failed to retrieve edit history", - ) + detail="Failed to retrieve edit history" + ) \ No newline at end of file diff --git a/backend/services/main/security/audit.py b/backend/services/main/security/audit.py index 836e931..f035d2d 100644 --- a/backend/services/main/security/audit.py +++ b/backend/services/main/security/audit.py @@ -46,6 +46,39 @@ def _yes_no(flag: Any) -> str: def _render_security(action: str, fields: Dict[str, Any]) -> List[str]: + # Handle compliance-related actions with beautiful formatting + if action == "compliance_access_attempt": + lines = [f"Compliance access attempt for message {fields.get('message_id', 'unknown')}"] + lines.append(f"User: {_format_user(fields)}") + if fields.get("ip"): + lines.append(f"IP: {fields['ip']}") + return lines + if action == "compliance_access_denied": + lines = [f"Compliance access denied for message {fields.get('message_id', 'unknown')}"] + lines.append(f"User: {_format_user(fields)}") + if fields.get("ip"): + lines.append(f"IP: {fields['ip']}") + if fields.get("reason"): + lines.append(f"Reason: {fields['reason']}") + return lines + if action == "compliance_access_failed": + lines = [f"Compliance access failed for message {fields.get('message_id', 'unknown')}"] + lines.append(f"User: {_format_user(fields)}") + if fields.get("ip"): + lines.append(f"IP: {fields['ip']}") + if fields.get("reason"): + lines.append(f"Reason: {fields['reason']}") + return lines + if action == "compliance_extraction_success": + lines = [f"Compliance extraction successful for message {fields.get('message_id', 'unknown')}"] + lines.append(f"Officer: {_format_user(fields)}") + sender_id = fields.get("sender_id") + recipient_id = fields.get("recipient_id") + if sender_id is not None and recipient_id is not None: + lines.append(f"Message: {_format_user({'username': fields.get('sender_username'), 'user_id': sender_id})} → {_format_user({'username': fields.get('recipient_username'), 'user_id': recipient_id})}") + if fields.get("ip"): + lines.append(f"IP: {fields['ip']}") + return lines if action == "login_success": lines = [f"Login approved for {_format_user(fields)}"] session = fields.get("session_id") @@ -61,14 +94,14 @@ def _render_security(action: str, fields: Dict[str, Any]) -> List[str]: if client_bits: lines.append(f"Client: {', '.join(client_bits)}") if fields.get("ip"): - lines.append(f"IP address: {fields['ip']}") + lines.append(f"IP: {fields['ip']}") return lines if action == "login_failed": lines = [f"Login denied for {_format_user(fields)}"] if fields.get("reason"): lines.append(f"Reason: {fields['reason']}") if fields.get("ip"): - lines.append(f"IP address: {fields['ip']}") + lines.append(f"IP: {fields['ip']}") return lines if action == "auth_bruteforce_detected": lines = ["Brute-force login pattern detected"] @@ -78,7 +111,7 @@ def _render_security(action: str, fields: Dict[str, Any]) -> List[str]: for key, value in failures.items(): lines.append(f"{key}: {value}") if fields.get("ip"): - lines.append(f"IP address: {fields['ip']}") + lines.append(f"IP: {fields['ip']}") if fields.get("window_seconds"): lines.append(f"Observation window: {fields['window_seconds']} seconds") return lines @@ -103,14 +136,14 @@ def _render_security(action: str, fields: Dict[str, Any]) -> List[str]: lines = [f"Password changed for {_format_user(fields)}"] lines.append(f"Other sessions revoked: {_yes_no(fields.get('logout_others'))}") if fields.get("ip"): - lines.append(f"IP address: {fields['ip']}") + lines.append(f"IP: {fields['ip']}") return lines if action == "logout": lines = [f"Logout recorded for {_format_user(fields)}"] if fields.get("session_id"): lines.append(f"Session: {fields['session_id']}") if fields.get("ip"): - lines.append(f"IP address: {fields['ip']}") + lines.append(f"IP: {fields['ip']}") return lines if action == "admin_delete_user": return [ @@ -367,6 +400,43 @@ def _render_access(action: str, fields: Dict[str, Any]) -> List[str]: continue lines.append(f"{key.replace('_', ' ').capitalize()}: {value}") return lines + if action == "compliance_access_attempt": + lines = [f"Compliance access attempt for message {fields.get('message_id', 'unknown')}"] + lines.append(f"User: {_format_user(fields)}") + if fields.get("ip"): + lines.append(f"IP: {fields['ip']}") + return lines + if action == "compliance_access_denied": + lines = [f"Compliance access denied for message {fields.get('message_id', 'unknown')}"] + lines.append(f"User: {_format_user(fields)}") + if fields.get("ip"): + lines.append(f"IP: {fields['ip']}") + if fields.get("reason"): + lines.append(f"Reason: {fields['reason']}") + return lines + if action == "compliance_access_failed": + lines = [f"Compliance access failed for message {fields.get('message_id', 'unknown')}"] + lines.append(f"User: {_format_user(fields)}") + if fields.get("ip"): + lines.append(f"IP: {fields['ip']}") + if fields.get("reason"): + lines.append(f"Reason: {fields['reason']}") + return lines + if action == "compliance_extraction_success": + lines = [f"Compliance extraction successful for message {fields.get('message_id', 'unknown')}"] + lines.append(f"Officer: {_format_user(fields)}") + sender_id = fields.get("sender_id") + recipient_id = fields.get("recipient_id") + if sender_id is not None and recipient_id is not None: + lines.append(f"Message: {_format_user({'username': fields.get('sender_username'), 'user_id': sender_id})} → {_format_user({'username': fields.get('recipient_username'), 'user_id': recipient_id})}") + if fields.get("ip"): + lines.append(f"IP: {fields['ip']}") + return lines + if action == "compliance_public_key_access": + lines = [f"Compliance public key accessed"] + if fields.get("ip"): + lines.append(f"IP: {fields['ip']}") + return lines return [f"{action.replace('_', ' ').capitalize()}"] + [ f"{key.replace('_', ' ').capitalize()}: {value}" for key, value in fields.items() diff --git a/backend/services/messaging/processor.py b/backend/services/messaging/processor.py index 1629afa..d745cae 100644 --- a/backend/services/messaging/processor.py +++ b/backend/services/messaging/processor.py @@ -116,6 +116,11 @@ def process_encrypted_message( sender_wrapped_mek = wrap_mek(mek, sender_wrap_key) recipient_wrapped_mek = wrap_mek(mek, recipient_wrap_key) + logger.info(f"🔐 MEK wrapping complete:") + logger.info(f" Compliance MEK: {compliance_wrapped_mek[:30]}... ({len(compliance_wrapped_mek)} chars)") + logger.info(f" Sender MEK: {sender_wrapped_mek[:30]}... ({len(sender_wrapped_mek)} chars)") + logger.info(f" Recipient MEK: {recipient_wrapped_mek[:30]}... ({len(recipient_wrapped_mek)} chars)") + duration = time.time() - start_time logger.info( "CRYPTO: Successfully processed message with 3 MEK wraps (compliance/sender/recipient) in %.2fms", diff --git a/docs/COMPLIANCE_DECRYPTION.md b/docs/COMPLIANCE_DECRYPTION.md new file mode 100644 index 0000000..9ca77fb --- /dev/null +++ b/docs/COMPLIANCE_DECRYPTION.md @@ -0,0 +1,197 @@ +# Compliance Message Decryption Guide + +This guide explains how to decrypt encrypted messages for compliance and legal purposes using the secure offline compliance system. + +## Overview + +The application uses client-server encryption for direct messages (DMs). While regular users can only decrypt their own messages, compliance officers can decrypt any message for legal compliance purposes using a secure offline process. + +## Security Model + +- **Regular Users**: Can only decrypt messages encrypted with their own public keys +- **Compliance Officers**: Can decrypt any message using the compliance private key (stored offline) +- **No Server Access**: Compliance private keys are never stored on production servers +- **Audit Trail**: All compliance access is logged with timestamps and user IDs + +## Prerequisites + +### 1. Compliance Officer Access + +- Must be logged in as user ID 1 (system administrator) +- Requires valid JWT authentication token + +### 2. Air-Gapped Machine + +- A secure, offline computer for decryption +- Compliance private key stored securely +- Python environment with required dependencies + +### 3. Files Required + +- `compliance_keypair.txt` - Contains compliance X25519 keypair +- `compliance_decryption.py` - Decryption script +- Message data extracted from the server + +## Step-by-Step Instructions + +### Step 1: Extract Message Data from Server + +**On the production server (as compliance officer):** + +1. Log in to the application as user ID 1 +2. Get your JWT token from browser developer tools: + - Open DevTools (F12) + - Go to Application → Local Storage + - Copy the `token` value +3. Extract message data using the API: + ```bash + curl -H "Authorization: Bearer YOUR_JWT_TOKEN" \ + http://localhost:8300/api/dm/compliance/extract/MESSAGE_ID \ + > compliance_MESSAGE_ID.json + ``` + Replace `MESSAGE_ID` with the actual message ID you want to decrypt. +4. Verify the extraction was successful: + ```bash + cat compliance_MESSAGE_ID.json | jq . + ``` + Expected response: + +### Step 2: Transfer Data to Air-Gapped Machine + +**Securely transfer the JSON file to your air-gapped machine:** + +- Use encrypted USB drive +- Use secure file transfer protocol +- Never transfer over network if air-gapping is required + +### Step 3: Decrypt Message on Air-Gapped Machine + +**On the air-gapped machine:** + +1. Ensure you have the required files: + - `compliance_keypair.txt` (compliance private key) + - `compliance_decryption.py` (decryption script) + - `compliance_MESSAGE_ID.json` (extracted message data) +2. Run the decryption: + ```bash + python compliance_decryption.py decrypt --input-file compliance_MESSAGE_ID.json + ``` +3. The script will output the decrypted message: + ``` + 🔓 Loading compliance data from: compliance_MESSAGE_ID.json + 📄 Loaded message ID: 123 + 📅 Timestamp: 2026-01-10T19:23:37.938054 + 👤 Sender: 456, Recipient: 789 + 🔐 Has compliance MEK: ✅ + 🔑 Loading compliance private key... + 🔓 Decrypting message content... + + ✅ DECRYPTION SUCCESSFUL + ================================================== + Message ID: 123 + From: User 456 + To: User 789 + Timestamp: 2026-01-10T19:23:37.938054 + Decrypted at: 2026-01-10T19:33:20.782656 + -------------------------------------------------- + MESSAGE CONTENT: + {"type":"text","data":{"content":"Your encrypted message here"}} + -------------------------------------------------- + ⚠️ This content has been accessed for compliance purposes + ``` + +## Message Format + +Decrypted messages contain the original message payload in JSON format: + +```json +{ + "type": "text", + "data": { + "content": "The actual message text", + "files": [...] // Optional file attachments + } +} +``` + +## Security Considerations + +### Key Management + +- **Compliance private key**: Never stored on production servers +- **Access control**: Only user ID 1 can extract messages +- **Audit logging**: All extractions are logged with timestamps + +### Data Handling + +- **Secure transfer**: Use encrypted channels for data transfer +- **Immediate destruction**: Delete decrypted content after review +- **No caching**: Don't store decrypted messages + +### Operational Security + +- **Air-gapped environment**: Use dedicated offline machine for decryption +- **Access controls**: Limit physical access to compliance officers +- **Regular audits**: Review access logs regularly + +## Troubleshooting + +### "Access denied" Error + +- Ensure you're logged in as user ID 1 +- Check that your JWT token is valid and not expired + +### "Message not found" Error + +- Verify the message ID exists +- Check that the message hasn't been deleted + +### Decryption Failures + +- Ensure `compliance_keypair.txt` is present and contains valid keys +- Check that the JSON file wasn't corrupted during transfer +- Verify Python environment has required cryptography dependencies + +### Network Errors + +- Ensure the server is running and accessible +- Check firewall and network connectivity +- Verify API endpoints are correctly configured + +## API Reference + +### Compliance Extraction Endpoint + +``` +GET /api/dm/compliance/extract/{message_id} +Authorization: Bearer +Response: JSON with encrypted message data +``` + +**Restrictions:** + +- Requires user ID 1 authentication +- Returns encrypted data only (no plaintext) +- Logs all access for audit purposes + +### Decryption Script + +```bash +python compliance_decryption.py decrypt --input-file +``` + +**Requirements:** + +- `compliance_keypair.txt` in current directory +- Valid JSON file from extraction API +- Python with cryptography library + +## Compliance Workflow Summary + +``` +1. Legal Request → 2. Compliance Officer → 3. Server Extraction → 4. Secure Transfer → 5. Offline Decryption → 6. Content Review → 7. Audit Logging + ↓ ↓ ↓ ↓ ↓ ↓ ↓ + Legal basis User ID 1 login API call with token Encrypted transfer Air-gapped machine Content analysis Access recorded +``` + +This ensures complete separation between production systems and compliance decryption, maintaining security while enabling legal access to encrypted communications \ No newline at end of file diff --git a/scripts/compliance-decryption/assets/report.css b/scripts/compliance-decryption/assets/report.css new file mode 100644 index 0000000..7dd6743 --- /dev/null +++ b/scripts/compliance-decryption/assets/report.css @@ -0,0 +1,347 @@ +/* FromChat compliance bundle report styles (conversation-like, minimal JS) */ + +:root { + --bg: #0b0f14; + --panel: #0f1520; + --panel-2: #121a27; + --text: #e6edf3; + --muted: #9aa7b2; + --border: #223045; + --accent: #4f7cff; + --bubble-in: #131b28; + --bubble-out: #1a2540; + --shadow: rgba(0, 0, 0, 0.35); +} + +html, body { + height: 100%; +} + +body { + margin: 0; + background: var(--bg); + color: var(--text); + font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial; +} + +a { + color: var(--accent); + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +.wrap { + max-width: 980px; + margin: 0 auto; + padding: 22px 14px 64px; +} + +.topbar { + position: sticky; + top: 0; + z-index: 10; + backdrop-filter: blur(10px); + background: rgba(11, 15, 20, 0.75); + border-bottom: 1px solid rgba(34, 48, 69, 0.7); +} + +.topbar-inner { + max-width: 980px; + margin: 0 auto; + padding: 14px 14px; + display: flex; + flex-wrap: wrap; + gap: 12px; + align-items: center; + justify-content: space-between; +} + +.brand { + display: flex; + flex-direction: column; + gap: 2px; + min-width: 0; +} + +.brand-title { + font-size: 15px; + font-weight: 700; + letter-spacing: 0.2px; +} + +.brand-subtitle { + font-size: 12px; + color: var(--muted); + white-space: normal; + word-break: break-word; +} + +.tools { + display: flex; + gap: 10px; + align-items: center; +} + +.search { + width: min(420px, 55vw); + border: 1px solid var(--border); + border-radius: 12px; + background: rgba(255, 255, 255, 0.03); + padding: 9px 10px; + color: var(--text); + outline: none; +} + +.search:focus { + border-color: rgba(79, 124, 255, 0.65); + box-shadow: 0 0 0 3px rgba(79, 124, 255, 0.18); +} + +.hint { + font-size: 12px; + color: var(--muted); +} + +.conversation { + margin-top: 16px; + border: 1px solid var(--border); + background: var(--panel); + border-radius: 16px; + overflow: hidden; + box-shadow: 0 12px 28px var(--shadow); +} + +.conv-header { + cursor: default; + padding: 14px 14px; + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + border-bottom: 1px solid rgba(34, 48, 69, 0.65); +} + +.conv-title { + display: flex; + flex-direction: column; + gap: 2px; + min-width: 0; +} + +.conv-title .line1 { + font-weight: 700; + font-size: 14px; +} + +.conv-title .line2 { + font-size: 12px; + color: var(--muted); +} + +.conv-meta { + display: flex; + gap: 8px; + flex-wrap: wrap; + justify-content: flex-end; + color: var(--muted); + font-size: 12px; +} + +.pill { + border: 1px solid rgba(34, 48, 69, 0.9); + border-radius: 999px; + padding: 3px 8px; + background: rgba(255, 255, 255, 0.02); +} + +.messages { + padding: 12px 10px 14px; +} + +.day { + display: flex; + justify-content: center; + margin: 12px 0 10px; +} + +.day span { + font-size: 12px; + color: var(--muted); + border: 1px solid rgba(34, 48, 69, 0.8); + background: rgba(255, 255, 255, 0.02); + padding: 3px 10px; + border-radius: 999px; +} + +.message-container { + display: flex; + margin: 8px 0; + gap: 12px; + align-items: flex-start; +} + +.edit-tabs-vertical { + display: flex; + flex-direction: column; + min-width: 80px; + gap: 4px; +} + +.tab-vertical { + padding: 6px 4px; + cursor: pointer; + border: 1px solid rgba(34, 48, 69, 0.6); + border-radius: 6px; + background: rgba(255, 255, 255, 0.02); + text-align: center; + transition: background-color 0.15s ease; + display: flex; + flex-direction: column; + align-items: center; + gap: 1px; + min-height: 40px; +} + +.tab-vertical:hover { + background: rgba(79, 124, 255, 0.08); +} + +.tab-vertical.active { + background: var(--accent); + color: white; + border-color: var(--accent); +} + +.tab-label-vertical { + font-size: 10px; + font-weight: 700; + line-height: 1.1; +} + +.tab-time-vertical { + font-size: 8px; + opacity: 0.9; + line-height: 1.1; + white-space: nowrap; +} + +.bubble-area { + flex: 1; + min-width: 0; +} + +.bubble { + display: none; + max-width: min(720px, 92%); + border: 1px solid rgba(34, 48, 69, 0.9); + border-radius: 16px; + padding: 10px 10px 9px; + box-shadow: 0 10px 24px rgba(0, 0, 0, 0.18); + background: var(--bubble-in); + border-top-left-radius: 6px; +} + +.bubble.active { + display: block; +} + +.bubble-header { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 10px; + margin-bottom: 6px; +} + +.who { + font-size: 12px; + color: var(--muted); +} + +.who strong { + color: var(--text); + font-weight: 700; +} + +.text { + white-space: pre-wrap; + word-break: break-word; + font-size: 14px; + line-height: 1.45; +} + +.msg-meta { + margin-top: 8px; + font-size: 11px; + color: var(--muted); + display: flex; + justify-content: space-between; + gap: 10px; +} + +.msg-meta-left { + white-space: nowrap; +} + +.msg-meta-right { + white-space: nowrap; + text-align: right; +} + +.attachments { + margin-top: 10px; + display: grid; + grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); + gap: 10px; +} + +.att { + border: 1px solid rgba(34, 48, 69, 0.9); + border-radius: 12px; + padding: 10px; + background: rgba(255, 255, 255, 0.02); +} + +.att-name { + font-size: 13px; + font-weight: 700; + margin-bottom: 7px; +} + +.thumb { + width: 100%; + max-height: 260px; + object-fit: contain; + border-radius: 10px; + border: 1px solid rgba(34, 48, 69, 0.9); + background: rgba(0, 0, 0, 0.18); +} + +.att-actions { + margin-top: 8px; + display: flex; + gap: 10px; + align-items: center; + flex-wrap: wrap; +} + +.att-size { + font-size: 12px; + color: var(--muted); +} + + +.footer { + margin-top: 18px; + font-size: 12px; + color: var(--muted); + padding: 10px 2px; +} + +.hidden { + display: none !important; +} + diff --git a/scripts/compliance-decryption/assets/report.js b/scripts/compliance-decryption/assets/report.js new file mode 100644 index 0000000..b8229f2 --- /dev/null +++ b/scripts/compliance-decryption/assets/report.js @@ -0,0 +1,105 @@ +/* Minimal JS for filtering messages in the static report. */ + +function normalizeText(s) { + return (s || "").toString().toLowerCase(); +} + +function filterReport(query) { + const q = normalizeText(query).trim(); + const conversations = document.querySelectorAll(".conversation"); + let anyVisible = false; + + conversations.forEach((conv) => { + const rows = conv.querySelectorAll("[data-search]"); + let visibleInConv = 0; + + rows.forEach((row) => { + const hay = normalizeText(row.getAttribute("data-search")); + const match = !q || hay.includes(q); + row.classList.toggle("hidden", !match); + if (match) visibleInConv += 1; + }); + + const convMatch = visibleInConv > 0; + conv.classList.toggle("hidden", !convMatch); + if (convMatch) anyVisible = true; + }); + + const hint = document.getElementById("filterHint"); + if (hint) { + hint.textContent = q + ? (anyVisible ? "Filtered" : "No matches") + : "Type to filter by text, user id, filename"; + } +} + +function setupEditHistoryTabs() { + document.querySelectorAll(".edit-tabs-vertical").forEach((tabsContainer) => { + const tabs = tabsContainer.querySelectorAll(".tab-vertical"); + + tabs.forEach((tab) => { + tab.addEventListener("click", () => { + const version = tab.getAttribute("data-version"); + const messageId = tab.getAttribute("data-message-id"); + + // Find the corresponding message container + const messageContainer = document.querySelector(`.message-container:has([data-message-id="${messageId}"])`); + if (!messageContainer) return; + + // Update tab states within this message + const allTabs = messageContainer.querySelectorAll(".tab-vertical"); + allTabs.forEach(t => t.classList.remove("active")); + tab.classList.add("active"); + + // Update bubble states within this message + const allBubbles = messageContainer.querySelectorAll(".bubble"); + allBubbles.forEach(bubble => { + bubble.classList.toggle("active", bubble.getAttribute("data-version") === version); + }); + }); + }); + }); +} + +function convertTimestampsToLocal() { + // Convert all timestamps to local timezone + document.querySelectorAll("[data-timestamp]").forEach((element) => { + const timestamp = element.getAttribute("data-timestamp"); + if (!timestamp) return; + + try { + // Parse the ISO timestamp + const date = new Date(timestamp.replace(" ", "T").replace("Z", "+00:00")); + + // Format in local timezone + const localTime = date.toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hour12: false + }); + + // Update the displayed text + element.textContent = localTime; + } catch (e) { + // If parsing fails, leave the original text + console.warn("Failed to parse timestamp:", timestamp); + } + }); +} + +document.addEventListener("DOMContentLoaded", () => { + const input = document.getElementById("searchInput"); + if (input) { + input.addEventListener("input", (e) => { + filterReport(e.target.value); + }); + } + + // Initialize edit history tabs + setupEditHistoryTabs(); + + // Convert timestamps to local timezone + convertTimestampsToLocal(); +}); + diff --git a/scripts/compliance-decryption/http_client.py b/scripts/compliance-decryption/http_client.py new file mode 100644 index 0000000..cb55c17 --- /dev/null +++ b/scripts/compliance-decryption/http_client.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, Optional +from urllib import error, request +from urllib.parse import quote + + +def http_get_bytes(url: str, token: str, timeout_seconds: float = 30.0) -> bytes: + req = request.Request(url, method="GET") + req.add_header("Authorization", f"Bearer {token}") + try: + with request.urlopen(req, timeout=timeout_seconds) as r: + return r.read() + except error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") if hasattr(e, "read") else "" + raise RuntimeError(f"HTTP {e.code} for {url}: {body[:500]}") + + +def http_get_json(url: str, token: str, timeout_seconds: float = 30.0) -> Dict[str, Any]: + raw = http_get_bytes(url, token, timeout_seconds=timeout_seconds) + try: + return json.loads(raw.decode("utf-8")) + except Exception as e: + raise RuntimeError(f"Failed to parse JSON from {url}: {e}") + + +def http_post_json( + url: str, + body: Dict[str, Any], + *, + token: Optional[str] = None, + timeout_seconds: float = 30.0, +) -> Dict[str, Any]: + payload = json.dumps(body, ensure_ascii=False).encode("utf-8") + req = request.Request(url, method="POST", data=payload) + req.add_header("Content-Type", "application/json") + if token: + req.add_header("Authorization", f"Bearer {token}") + try: + with request.urlopen(req, timeout=timeout_seconds) as r: + raw = r.read() + except error.HTTPError as e: + body_txt = e.read().decode("utf-8", errors="replace") if hasattr(e, "read") else "" + raise RuntimeError(f"HTTP {e.code} for {url}: {body_txt[:500]}") + + try: + return json.loads(raw.decode("utf-8")) + except Exception as e: + raise RuntimeError(f"Failed to parse JSON from {url}: {e}") + + +def join_api_url(api_base_url: str, path: str) -> str: + """ + Join an API base URL (usually ends with '/api') with a path that may start with: + - '/api/...' + - '/uploads/...' + - 'uploads/...' + """ + base = api_base_url.rstrip("/") + p = (path or "").strip() + if p.startswith("http://") or p.startswith("https://"): + return p + + p_quoted = quote(p, safe="/:?&=%") + + if p.startswith("/api/"): + origin = base[:-4] if base.endswith("/api") else base + return origin.rstrip("/") + p_quoted + + if not p.startswith("/"): + p_quoted = "/" + p_quoted + return base + p_quoted + diff --git a/scripts/compliance-decryption/main.py b/scripts/compliance-decryption/main.py new file mode 100644 index 0000000..9820c68 --- /dev/null +++ b/scripts/compliance-decryption/main.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +""" +FromChat compliance decryption tool entrypoint. + +Run: + python scripts/compliance-decryption/main.py ... +""" + +from __future__ import annotations + +import os +import sys + + +def main() -> None: + root_dir = os.path.dirname(os.path.abspath(__file__)) + if root_dir not in sys.path: + sys.path.insert(0, root_dir) + + from cli import main as cli_main + + cli_main() + + +if __name__ == "__main__": + main() + diff --git a/scripts/compliance-decryption/report_assets.py b/scripts/compliance-decryption/report_assets.py new file mode 100644 index 0000000..fa0ae7c --- /dev/null +++ b/scripts/compliance-decryption/report_assets.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from pathlib import Path + + +def assets_source_dir() -> Path: + """ + Directory that stores static templates (css/js) for report generation. + + Layout: + scripts/compliance-decryption/ + main.py + assets/ + report.css + report.js + *.py + """ + root_dir = Path(__file__).resolve().parent + return root_dir / "assets" + + +def read_asset_text(name: str) -> str: + path = assets_source_dir() / name + return path.read_text(encoding="utf-8") + + +def write_assets(output_dir: Path) -> tuple[str, str]: + assets_dir = output_dir / "assets" + assets_dir.mkdir(parents=True, exist_ok=True) + + css_src = read_asset_text("report.css") + js_src = read_asset_text("report.js") + + css_rel = "assets/report.css" + js_rel = "assets/report.js" + + (assets_dir / "report.css").write_text(css_src, encoding="utf-8") + (assets_dir / "report.js").write_text(js_src, encoding="utf-8") + + return css_rel, js_rel + diff --git a/scripts/compliance-decryption/utils.py b/scripts/compliance-decryption/utils.py new file mode 100644 index 0000000..641e544 --- /dev/null +++ b/scripts/compliance-decryption/utils.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import json +import os +from typing import Any, Dict +from urllib.parse import quote + + +def safe_filename(name: str, max_len: int = 140) -> str: + base = "".join(c for c in (name or "") if c.isalnum() or c in " ._-()[]{}").strip() + base = base.replace(" ", " ") + base = base.replace("/", "_").replace("\\", "_") + if not base: + base = "file" + if len(base) > max_len: + base = base[:max_len].rstrip() + return base + + +def html_escape(text: str) -> str: + return ( + (text or "") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + +def href_escape(rel_path: str) -> str: + """ + Percent-encode a relative path for use in HTML href/src. + Keep slashes so nested paths work. + """ + return quote(rel_path, safe="/") + + +def guess_is_image(filename: str) -> bool: + ext = (os.path.splitext(filename or "")[1] or "").lower() + return ext in {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg"} + + +def parse_message_plaintext(plaintext: str) -> Dict[str, Any]: + """ + Best-effort parse of decrypted message JSON. + + Returns: + - kind: "json" | "text" + - text: best-effort human-readable text + - raw: original plaintext + - json: parsed object (if kind=="json") + """ + raw = plaintext or "" + try: + obj = json.loads(raw) + content = "" + if isinstance(obj, dict): + data = obj.get("data") + if isinstance(data, dict): + content_val = data.get("content") + if isinstance(content_val, str): + content = content_val + return {"kind": "json", "text": content or raw, "raw": raw, "json": obj} + except Exception: + return {"kind": "text", "text": raw, "raw": raw} +