diff --git a/backend/services/main/key_lifecycle.py b/backend/services/main/key_lifecycle.py new file mode 100644 index 0000000..8d5d773 --- /dev/null +++ b/backend/services/main/key_lifecycle.py @@ -0,0 +1,200 @@ +""" +Key lifecycle: time-based removal of compliance MEK, soft-deleted DM keys, and edit history. + +Uses MESSAGE_RETENTION_DAYS from the environment (see services.shared.message_retention). +""" + +import logging +from datetime import datetime +from sqlalchemy.orm import Session + +from .models import DMEnvelope, MessageEditHistory, DMEditHistory + +logger = logging.getLogger("uvicorn.error") + + +def _retention_timedelta_or_skip(): + try: + from services.shared.message_retention import get_message_retention + except ImportError: + from backend.services.shared.message_retention import get_message_retention # type: ignore + r = get_message_retention() + if not r.cleanup_enabled(): + return None + return r.retention_timedelta() + + +def destroy_compliance_keys_for_message(db: Session, message_id: int) -> int: + try: + envelopes = db.query(DMEnvelope).filter(DMEnvelope.id == message_id).all() + + destroyed_count = 0 + for envelope in envelopes: + if envelope.compliance_wrapped_mek_b64: + envelope.compliance_wrapped_mek_b64 = None + destroyed_count += 1 + + if destroyed_count > 0: + db.commit() + logger.info( + "Destroyed compliance keys for %s DM envelopes (message_id=%s)", + destroyed_count, + message_id, + ) + + return destroyed_count + + except Exception as e: + logger.error("Failed to destroy compliance keys for message %s: %s", message_id, e) + db.rollback() + return 0 + + +def destroy_compliance_keys_for_dm_envelope(db: Session, dm_envelope_id: int) -> bool: + try: + envelope = db.query(DMEnvelope).filter(DMEnvelope.id == dm_envelope_id).first() + if envelope and envelope.compliance_wrapped_mek_b64: + envelope.compliance_wrapped_mek_b64 = None + db.commit() + logger.info("Destroyed compliance key for DM envelope %s", dm_envelope_id) + return True + return False + + except Exception as e: + logger.error("Failed to destroy compliance key for DM envelope %s: %s", dm_envelope_id, e) + db.rollback() + return False + + +def cleanup_expired_compliance_keys(db: Session) -> int: + delta = _retention_timedelta_or_skip() + if delta is None: + return 0 + + try: + cutoff_date = datetime.now() - delta + + expired_envelopes = db.query(DMEnvelope).filter( + DMEnvelope.timestamp < cutoff_date, + DMEnvelope.compliance_wrapped_mek_b64.isnot(None), + ).all() + + destroyed_count = 0 + for envelope in expired_envelopes: + envelope.compliance_wrapped_mek_b64 = None + destroyed_count += 1 + + if destroyed_count > 0: + db.commit() + logger.info("Cleaned up %s expired compliance MEK fields", destroyed_count) + + return destroyed_count + + except Exception as e: + logger.error("Failed to cleanup expired compliance keys: %s", e) + db.rollback() + return 0 + + +def cleanup_expired_message_keys(db: Session) -> int: + delta = _retention_timedelta_or_skip() + if delta is None: + return 0 + + try: + cutoff_date = datetime.now() - delta + + expired_messages = db.query(DMEnvelope).filter( + DMEnvelope.deleted_at.is_not(None), + DMEnvelope.deleted_at < cutoff_date, + ).all() + + if not expired_messages: + return 0 + + keys_destroyed = 0 + + for message in expired_messages: + message.sender_wrapped_mek_b64 = "" + message.recipient_wrapped_mek_b64 = "" + keys_destroyed += 2 + + logger.debug( + "Destroyed keys for soft-deleted message id=%s (deleted %s)", + message.id, + message.deleted_at.isoformat(), + ) + + db.commit() + logger.info( + "Message key cleanup: destroyed %s keys across %s messages", + keys_destroyed, + len(expired_messages), + ) + + return keys_destroyed + + except Exception as e: + logger.error("Failed to cleanup expired message keys: %s", e) + db.rollback() + return 0 + + +def cleanup_expired_edit_history(db: Session) -> int: + delta = _retention_timedelta_or_skip() + if delta is None: + return 0 + + try: + cutoff_date = datetime.now() - delta + + public_deleted = db.query(MessageEditHistory).filter( + MessageEditHistory.edited_at < cutoff_date + ).delete(synchronize_session=False) + + dm_deleted = db.query(DMEditHistory).filter( + DMEditHistory.edited_at < cutoff_date + ).delete(synchronize_session=False) + + total_deleted = public_deleted + dm_deleted + + if total_deleted > 0: + db.commit() + logger.info("Cleaned up %s expired edit history entries", total_deleted) + + return total_deleted + + except Exception as e: + logger.error("Failed to cleanup expired edit history: %s", e) + db.rollback() + return 0 + + +def run_key_lifecycle_cleanup(db: Session) -> dict: + stats = { + "compliance_keys_destroyed": cleanup_expired_compliance_keys(db), + "message_keys_destroyed": cleanup_expired_message_keys(db), + "edit_history_entries_removed": cleanup_expired_edit_history(db), + "timestamp": datetime.now().isoformat(), + } + + if ( + stats["compliance_keys_destroyed"] + or stats["message_keys_destroyed"] + or stats["edit_history_entries_removed"] + ): + logger.info("Key lifecycle cleanup completed: %s", stats) + return stats + + +def get_key_lifecycle_config() -> dict: + try: + from services.shared.message_retention import get_message_retention + except ImportError: + from backend.services.shared.message_retention import get_message_retention # type: ignore + r = get_message_retention() + return { + "message_retention_days": r.days, + "cleanup_enabled": r.cleanup_enabled(), + "never_store_compliance_mek": r.never_store_compliance_mek(), + } diff --git a/backend/services/main/key_lifecycle_task.py b/backend/services/main/key_lifecycle_task.py new file mode 100644 index 0000000..2f219b8 --- /dev/null +++ b/backend/services/main/key_lifecycle_task.py @@ -0,0 +1,45 @@ +""" +Periodic key lifecycle cleanup (compliance MEK, deleted-message keys, edit history). +Poll interval is derived from MESSAGE_RETENTION_DAYS (no separate env var). +""" + +import asyncio +import logging + +from .db import SessionLocal +from .key_lifecycle import run_key_lifecycle_cleanup + +logger = logging.getLogger("uvicorn.error") + + +def key_lifecycle_poll_seconds() -> int | None: + try: + from services.shared.message_retention import get_message_retention + except ImportError: + from backend.services.shared.message_retention import get_message_retention # type: ignore + r = get_message_retention() + if not r.cleanup_enabled(): + return None + sec = r.retention_timedelta().total_seconds() + # Bound poll: responsive after cutoff without hammering the DB + return max(15, min(3600, max(1, int(sec / 1000)))) + + +async def start_key_lifecycle_cleanup_task(interval_seconds: int) -> None: + while True: + try: + with SessionLocal() as db: + run_key_lifecycle_cleanup(db) + except asyncio.CancelledError: + break + except Exception as e: + logger.error("Error in key lifecycle cleanup task: %s", e) + try: + await asyncio.sleep(60) + except asyncio.CancelledError: + break + continue + try: + await asyncio.sleep(interval_seconds) + except asyncio.CancelledError: + break diff --git a/backend/services/main/main.py b/backend/services/main/main.py index 513b3ee..eb330cf 100644 --- a/backend/services/main/main.py +++ b/backend/services/main/main.py @@ -44,6 +44,9 @@ def _running_in_docker() -> bool: @asynccontextmanager async def lifespan(app: FastAPI): + cleanup_task = None + key_lifecycle_task = None + # Startup - run migration in subprocess to avoid logging interference try: logger.info("Starting database migration check...") @@ -122,6 +125,19 @@ async def lifespan(app: FastAPI): logger.error(f"Failed to start rate limit cleanup task: {e}") cleanup_task = None + try: + from .key_lifecycle_task import key_lifecycle_poll_seconds, start_key_lifecycle_cleanup_task + _poll = key_lifecycle_poll_seconds() + if _poll is not None: + key_lifecycle_task = asyncio.create_task(start_key_lifecycle_cleanup_task(_poll)) + logger.info("Key lifecycle cleanup task started (interval=%ss)", _poll) + else: + key_lifecycle_task = None + logger.info("Key lifecycle cleanup disabled (MESSAGE_RETENTION_DAYS is 0 or -1)") + except Exception as e: + logger.error("Failed to start key lifecycle cleanup task: %s", e) + key_lifecycle_task = None + yield # Shutdown - cancel cleanup task if it exists @@ -132,6 +148,13 @@ async def lifespan(app: FastAPI): except asyncio.CancelledError: pass + if key_lifecycle_task: + key_lifecycle_task.cancel() + try: + await key_lifecycle_task + except asyncio.CancelledError: + pass + # Initialize FastAPI app = FastAPI(title="FromChat", lifespan=lifespan) diff --git a/backend/services/main/push_service.py b/backend/services/main/push_service.py index d8524d7..cbcd463 100644 --- a/backend/services/main/push_service.py +++ b/backend/services/main/push_service.py @@ -12,22 +12,19 @@ from firebase_admin import messaging as firebase_messaging logger = logging.getLogger("uvicorn.error") +# backend/firebase-cert.json — fixed path; Docker bind-mounts this file to /app/firebase-cert.json +_FIREBASE_CERT_PATH = Path(__file__).resolve().parents[2] / "firebase-cert.json" -def _load_firebase_service_account_dict(firebase_cert: str) -> dict: - """Load Firebase service account JSON from FIREBASE_CERT path (relative to process cwd, e.g. backend/).""" - s = (firebase_cert or "").strip() - if not s: - raise RuntimeError("FIREBASE_CERT env variable is required (path to service account JSON file)") - p = Path(s).expanduser() - if not p.is_absolute(): - p = Path.cwd() / p - if not p.is_file(): +def _load_firebase_service_account_dict(cert_path: Path) -> dict: + """Load Firebase service account JSON from ``cert_path`` (must exist).""" + cert_path = cert_path.resolve() + if not cert_path.is_file(): raise FileNotFoundError( - f"FIREBASE_CERT is not a readable file: {p} (set FIREBASE_CERT to the JSON key path)" + f"Firebase credentials file missing or not a file: {cert_path} (expected backend/firebase-cert.json)" ) - with p.open(encoding="utf-8") as f: + with cert_path.open(encoding="utf-8") as f: data = json.load(f) if not isinstance(data, dict) or data.get("type") != "service_account": raise ValueError("Firebase credentials file must be a service account JSON object") @@ -38,23 +35,17 @@ class PushNotificationService: def __init__(self): self.vapid_private_key = os.getenv("VAPID_PRIVATE_KEY") self.vapid_public_key = os.getenv("VAPID_PUBLIC_KEY") - # Firebase Admin is required for main (FCM). FIREBASE_CERT = path to service account JSON. + # Firebase Admin is required for main (FCM); cert path is backend/firebase-cert.json. self.firebase_initialized = False - firebase_cert = os.getenv("FIREBASE_CERT") - if not (firebase_cert or "").strip(): - raise RuntimeError( - "FIREBASE_CERT is required (path to Firebase service account JSON); " - "docker-compose sets this and bind-mounts backend/firebase-cert.json" - ) try: - sa_dict = _load_firebase_service_account_dict(firebase_cert) + sa_dict = _load_firebase_service_account_dict(_FIREBASE_CERT_PATH) cred = firebase_credentials.Certificate(sa_dict) firebase_admin.initialize_app(cred) self.firebase_initialized = True - logger.info("Firebase Admin SDK initialized for push sending (FIREBASE_CERT)") + logger.info("Firebase Admin SDK initialized (%s)", _FIREBASE_CERT_PATH) except Exception as e: - logger.error(f"Failed to initialize Firebase Admin SDK from FIREBASE_CERT: {e}") + logger.error("Failed to initialize Firebase Admin SDK from %s: %s", _FIREBASE_CERT_PATH, e) raise if (not self.vapid_public_key) or (not self.vapid_private_key): @@ -203,7 +194,7 @@ class PushNotificationService: """Send an FCM data-only push to a single device token using Firebase Admin SDK. Notification display is handled by the app, not FCM.""" if not self.firebase_initialized: - raise RuntimeError("Firebase Admin SDK not initialized (FIREBASE_CERT required)") + raise RuntimeError("Firebase Admin SDK not initialized") try: # Send only data payload - let the app handle notification display diff --git a/backend/services/main/service_calls.py b/backend/services/main/service_calls.py index c6434f8..70dedb1 100644 --- a/backend/services/main/service_calls.py +++ b/backend/services/main/service_calls.py @@ -74,6 +74,13 @@ async def get_compliance_public_key(timeout: float = 5.0) -> Dict[str, Any]: """ Return compliance system public key (for MEK wrapping). """ + try: + from services.shared.message_retention import get_message_retention + except ImportError: + from backend.services.shared.message_retention import get_message_retention # type: ignore + if get_message_retention().never_store_compliance_mek(): + return {"public_key_b64": ""} + mod = _get_messaging_module() if mod: # in-process async call diff --git a/backend/services/messaging/key_lifecycle.py b/backend/services/messaging/key_lifecycle.py deleted file mode 100644 index 04e75d4..0000000 --- a/backend/services/messaging/key_lifecycle.py +++ /dev/null @@ -1,279 +0,0 @@ -""" -Key Lifecycle Management for Compliance and Security. - -This module handles automatic destruction of compliance keys, selective key destruction -on message deletion, and configurable retention policies for cryptographic keys. - -Key Features: -- Automatic compliance key destruction (default: 6 months) -- Selective key destruction on message deletion (default: 6 months) -- Configurable retention policies -- Background cleanup jobs for expired keys -""" - -import logging -import os -from datetime import datetime, timedelta -from typing import List, Optional -from sqlalchemy.orm import Session - -from ..main.models import DMEnvelope, MessageEditHistory, DMEditHistory -from .encryption import generate_nonce, TRANSPORT_NONCE_SIZE - -logger = logging.getLogger("uvicorn.error") - -# Default retention periods (in days) -DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS = 180 # 6 months -DEFAULT_MESSAGE_KEY_RETENTION_DAYS = 180 # 6 months for deleted messages - -# Environment variable overrides -COMPLIANCE_KEY_RETENTION_DAYS = int(os.getenv("COMPLIANCE_KEY_RETENTION_DAYS", DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS)) -MESSAGE_KEY_RETENTION_DAYS = int(os.getenv("MESSAGE_KEY_RETENTION_DAYS", DEFAULT_MESSAGE_KEY_RETENTION_DAYS)) - - -def get_compliance_key_retention_period() -> timedelta: - """Get the retention period for compliance keys.""" - return timedelta(days=COMPLIANCE_KEY_RETENTION_DAYS) - - -def get_message_key_retention_period() -> timedelta: - """Get the retention period for message keys after deletion.""" - return timedelta(days=MESSAGE_KEY_RETENTION_DAYS) - - -def destroy_compliance_keys_for_message(db: Session, message_id: int) -> int: - """ - Destroy compliance keys for a specific message. - - This removes the compliance_wrapped_mek_b64 from DM envelopes, - making the message permanently inaccessible for compliance purposes. - - Args: - db: Database session - message_id: ID of the message to destroy compliance keys for - - Returns: - Number of envelopes affected - """ - try: - # Find all DM envelopes for this message - envelopes = db.query(DMEnvelope).filter(DMEnvelope.id == message_id).all() - - destroyed_count = 0 - for envelope in envelopes: - if envelope.compliance_wrapped_mek_b64: - envelope.compliance_wrapped_mek_b64 = None - destroyed_count += 1 - - if destroyed_count > 0: - db.commit() - logger.info(f"Destroyed compliance keys for {destroyed_count} DM envelopes (message_id={message_id})") - - return destroyed_count - - except Exception as e: - logger.error(f"Failed to destroy compliance keys for message {message_id}: {e}") - db.rollback() - return 0 - - -def destroy_compliance_keys_for_dm_envelope(db: Session, dm_envelope_id: int) -> bool: - """ - Destroy compliance key for a specific DM envelope. - - Args: - db: Database session - dm_envelope_id: ID of the DM envelope - - Returns: - True if key was destroyed, False otherwise - """ - try: - envelope = db.query(DMEnvelope).filter(DMEnvelope.id == dm_envelope_id).first() - if envelope and envelope.compliance_wrapped_mek_b64: - envelope.compliance_wrapped_mek_b64 = None - db.commit() - logger.info(f"Destroyed compliance key for DM envelope {dm_envelope_id}") - return True - return False - - except Exception as e: - logger.error(f"Failed to destroy compliance key for DM envelope {dm_envelope_id}: {e}") - db.rollback() - return False - - -def cleanup_expired_compliance_keys(db: Session) -> int: - """ - Clean up expired compliance keys based on retention policy. - - This removes compliance_wrapped_mek_b64 from DM envelopes that are older - than the retention period, making them permanently inaccessible for compliance. - - Args: - db: Database session - - Returns: - Number of keys destroyed - """ - try: - cutoff_date = datetime.now() - get_compliance_key_retention_period() - - # Find DM envelopes older than retention period that still have compliance keys - expired_envelopes = db.query(DMEnvelope).filter( - DMEnvelope.timestamp < cutoff_date, - DMEnvelope.compliance_wrapped_mek_b64.isnot(None) - ).all() - - destroyed_count = 0 - for envelope in expired_envelopes: - envelope.compliance_wrapped_mek_b64 = None - destroyed_count += 1 - - if destroyed_count > 0: - db.commit() - logger.info(f"Cleaned up {destroyed_count} expired compliance keys (retention: {COMPLIANCE_KEY_RETENTION_DAYS} days)") - - return destroyed_count - - except Exception as e: - logger.error(f"Failed to cleanup expired compliance keys: {e}") - db.rollback() - return 0 - - -def cleanup_expired_message_keys(db: Session) -> int: - """ - Clean up message keys for deleted messages after retention period. - - This removes sender and recipient wrapped keys from DM envelopes that have been - soft-deleted and are past the retention period, making them completely inaccessible - except through compliance access (which preserves the compliance key). - - Args: - db: Database session - - Returns: - Number of keys destroyed - """ - try: - from datetime import datetime, timedelta - from ..main.models import DMEnvelope - - # Calculate cutoff date for expired messages - cutoff_date = datetime.now() - get_message_key_retention_period() - - # Find soft-deleted messages past retention period - expired_messages = db.query(DMEnvelope).filter( - DMEnvelope.deleted_at.is_not(None), - DMEnvelope.deleted_at < cutoff_date - ).all() - - if not expired_messages: - logger.info("Message key cleanup: No expired deleted messages to process") - return 0 - - keys_destroyed = 0 - - for message in expired_messages: - # Destroy sender and recipient keys (compliance key remains for legal access) - message.sender_wrapped_mek_b64 = "" - message.recipient_wrapped_mek_b64 = "" - keys_destroyed += 2 - - logger.info( - "Destroyed keys for soft-deleted message id=%s (deleted %s)", - message.id, - message.deleted_at.isoformat() - ) - - db.commit() - logger.info("Message key cleanup: Destroyed %d keys across %d messages", - keys_destroyed, len(expired_messages)) - - return keys_destroyed - - except Exception as e: - logger.error(f"Failed to cleanup expired message keys: {e}") - db.rollback() - return 0 - - -def cleanup_expired_edit_history(db: Session) -> int: - """ - Clean up old edit history entries based on retention policy. - - This removes edit history entries that are older than the compliance - retention period. - - Args: - db: Database session - - Returns: - Number of edit history entries removed - """ - try: - cutoff_date = datetime.now() - get_compliance_key_retention_period() - - # Clean up public message edit history - public_deleted = db.query(MessageEditHistory).filter( - MessageEditHistory.edited_at < cutoff_date - ).delete(synchronize_session=False) - - # Clean up DM edit history - dm_deleted = db.query(DMEditHistory).filter( - DMEditHistory.edited_at < cutoff_date - ).delete(synchronize_session=False) - - total_deleted = public_deleted + dm_deleted - - if total_deleted > 0: - db.commit() - logger.info(f"Cleaned up {total_deleted} expired edit history entries (retention: {COMPLIANCE_KEY_RETENTION_DAYS} days)") - - return total_deleted - - except Exception as e: - logger.error(f"Failed to cleanup expired edit history: {e}") - db.rollback() - return 0 - - -def run_key_lifecycle_cleanup(db: Session) -> dict: - """ - Run all key lifecycle cleanup operations. - - This should be called periodically (e.g., daily) to maintain key lifecycle policies. - - Args: - db: Database session - - Returns: - Dict with cleanup statistics - """ - logger.info("Starting key lifecycle cleanup") - - stats = { - "compliance_keys_destroyed": cleanup_expired_compliance_keys(db), - "message_keys_destroyed": cleanup_expired_message_keys(db), - "edit_history_entries_removed": cleanup_expired_edit_history(db), - "timestamp": datetime.now().isoformat() - } - - logger.info(f"Key lifecycle cleanup completed: {stats}") - return stats - - -def get_key_lifecycle_config() -> dict: - """ - Get current key lifecycle configuration. - - Returns: - Dict with current configuration values - """ - return { - "compliance_key_retention_days": COMPLIANCE_KEY_RETENTION_DAYS, - "message_key_retention_days": MESSAGE_KEY_RETENTION_DAYS, - "default_compliance_retention": DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS, - "default_message_retention": DEFAULT_MESSAGE_KEY_RETENTION_DAYS - } \ No newline at end of file diff --git a/backend/services/messaging/main.py b/backend/services/messaging/main.py index 4149f25..cc331c3 100644 --- a/backend/services/messaging/main.py +++ b/backend/services/messaging/main.py @@ -50,7 +50,19 @@ def _initialize_compliance_key(): The private key never exists on the server - all decryption is done offline. """ global _COMPLIANCE_PUBLIC_KEY_B64 - + + try: + from services.shared.message_retention import get_message_retention + except ImportError: + from backend.services.shared.message_retention import get_message_retention # type: ignore + + if get_message_retention().never_store_compliance_mek(): + _COMPLIANCE_PUBLIC_KEY_B64 = "" + logger.info( + "Compliance MEK not stored (MESSAGE_RETENTION_DAYS=-1); COMPLIANCE_PUBLIC_KEY optional" + ) + return + env_key = os.getenv("COMPLIANCE_PUBLIC_KEY", "").strip() if not env_key: raise RuntimeError( @@ -58,7 +70,7 @@ def _initialize_compliance_key(): "Generate offline on an air-gapped machine: " "X25519 private key → export public key (base64) → set as env var" ) - + _COMPLIANCE_PUBLIC_KEY_B64 = env_key logger.info("Loaded compliance public key from COMPLIANCE_PUBLIC_KEY environment variable") @@ -151,6 +163,13 @@ except ImportError: if add_security_middleware: add_security_middleware(app) +try: + from services.shared.inter_service_rate_limit import attach_internal_service_rate_limit +except ImportError: + from backend.services.shared.inter_service_rate_limit import attach_internal_service_rate_limit # type: ignore + +_internal_limiter = attach_internal_service_rate_limit(app, default_limit="5000/minute") + # CORS configuration for inter-service communication app.add_middleware( CORSMiddleware, @@ -202,6 +221,7 @@ class ProcessMessageWithFilesRequest(ProcessMessageRequest): # ============================================================================ @app.get("/health", response_model=None) +@_internal_limiter.exempt async def health_check(): """Health check endpoint for messaging service.""" return {"status": "healthy", "service": "messaging"} diff --git a/backend/services/messaging/processor.py b/backend/services/messaging/processor.py index e282f14..abcc116 100644 --- a/backend/services/messaging/processor.py +++ b/backend/services/messaging/processor.py @@ -30,6 +30,14 @@ from .encryption import ( logger = logging.getLogger("uvicorn.error") +def _store_compliance_wrapped_mek() -> bool: + try: + from services.shared.message_retention import get_message_retention + except ImportError: + from backend.services.shared.message_retention import get_message_retention # type: ignore + return not get_message_retention().never_store_compliance_mek() + + def process_encrypted_message( client_public_key_b64: str, transport_nonce_b64: str, @@ -101,32 +109,50 @@ def process_encrypted_message( # Use HKDF with recipient public key bytes as input to derive wrap keys # This is deterministic and doesn't require storing ephemeral keys import base64 - compliance_key_bytes = base64.b64decode(compliance_public_key_b64) sender_key_bytes = base64.b64decode(sender_public_key_b64) recipient_key_bytes = base64.b64decode(recipient_public_key_b64) - logger.info(f"🔑 Deriving wrap keys for sender={sender_public_key_b64[:20]}... recipient={recipient_public_key_b64[:20]}...") + logger.info( + "🔑 Deriving wrap keys for sender=%s... recipient=%s...", + sender_public_key_b64[:20], + recipient_public_key_b64[:20], + ) - compliance_wrap_key = derive_key_from_shared_secret(compliance_key_bytes, "compliance_wrap_key") sender_wrap_key = derive_key_from_shared_secret(sender_key_bytes, "sender_wrap_key") recipient_wrap_key = derive_key_from_shared_secret(recipient_key_bytes, "recipient_wrap_key") - logger.info("✅ Wrap keys derived successfully") + logger.info("✅ Sender/recipient wrap keys derived successfully") + + if _store_compliance_wrapped_mek(): + if not (compliance_public_key_b64 or "").strip(): + raise ValueError( + "compliance public key required when MESSAGE_RETENTION_DAYS is not -1" + ) + compliance_key_bytes = base64.b64decode(compliance_public_key_b64) + compliance_wrap_key = derive_key_from_shared_secret( + compliance_key_bytes, "compliance_wrap_key" + ) + compliance_wrapped_mek = wrap_mek(mek, compliance_wrap_key) + logger.info( + "🔐 Compliance MEK: %s... (%s chars)", + compliance_wrapped_mek[:30], + len(compliance_wrapped_mek), + ) + else: + compliance_wrapped_mek = None + logger.info("CRYPTO: Compliance MEK not stored (MESSAGE_RETENTION_DAYS=-1)") - # Step 4b: Wrap MEK for each recipient - compliance_wrapped_mek = wrap_mek(mek, compliance_wrap_key) sender_wrapped_mek = wrap_mek(mek, sender_wrap_key) recipient_wrapped_mek = wrap_mek(mek, recipient_wrap_key) logger.info(f"🔐 MEK wrapping complete:") - logger.info(f" Compliance MEK: {compliance_wrapped_mek[:30]}... ({len(compliance_wrapped_mek)} chars)") logger.info(f" Sender MEK: {sender_wrapped_mek[:30]}... ({len(sender_wrapped_mek)} chars)") logger.info(f" Recipient MEK: {recipient_wrapped_mek[:30]}... ({len(recipient_wrapped_mek)} chars)") duration = time.time() - start_time logger.info( - "CRYPTO: Successfully processed message with 3 MEK wraps (compliance/sender/recipient) in %.2fms", - duration * 1000 + "CRYPTO: Successfully processed message with MEK wraps in %.2fms", + duration * 1000, ) # Get the transport public key for storage with the message @@ -251,15 +277,25 @@ def process_encrypted_message_and_files( files_out.append(entry) # Derive wrap keys deterministically (same as existing flow) - compliance_key_bytes = base64.b64decode(compliance_public_key_b64) sender_key_bytes = base64.b64decode(sender_public_key_b64) recipient_key_bytes = base64.b64decode(recipient_public_key_b64) - compliance_wrap_key = derive_key_from_shared_secret(compliance_key_bytes, "compliance_wrap_key") sender_wrap_key = derive_key_from_shared_secret(sender_key_bytes, "sender_wrap_key") recipient_wrap_key = derive_key_from_shared_secret(recipient_key_bytes, "recipient_wrap_key") - compliance_wrapped_mek = wrap_mek(mek, compliance_wrap_key) + if _store_compliance_wrapped_mek(): + if not (compliance_public_key_b64 or "").strip(): + raise ValueError( + "compliance public key required when MESSAGE_RETENTION_DAYS is not -1" + ) + compliance_key_bytes = base64.b64decode(compliance_public_key_b64) + compliance_wrap_key = derive_key_from_shared_secret( + compliance_key_bytes, "compliance_wrap_key" + ) + compliance_wrapped_mek = wrap_mek(mek, compliance_wrap_key) + else: + compliance_wrapped_mek = None + sender_wrapped_mek = wrap_mek(mek, sender_wrap_key) recipient_wrapped_mek = wrap_mek(mek, recipient_wrap_key) diff --git a/backend/services/shared/inter_service_rate_limit.py b/backend/services/shared/inter_service_rate_limit.py new file mode 100644 index 0000000..9fbb5b9 --- /dev/null +++ b/backend/services/shared/inter_service_rate_limit.py @@ -0,0 +1,51 @@ +""" +Per-IP rate limits for internal FastAPI apps (messaging, file_storage). + +Complements the main service's endpoint-specific limits. Uses a generous default +because traffic is mostly from the main backend (single Docker bridge IP). +""" + +from __future__ import annotations + +from fastapi import FastAPI, Request +from slowapi import Limiter, _rate_limit_exceeded_handler +from slowapi.errors import RateLimitExceeded +from slowapi.middleware import SlowAPIMiddleware +from slowapi.util import get_remote_address + + +def _client_ip_key(request: Request) -> str: + if request is None: + return "unknown" + headers = request.headers + real = (headers.get("x-real-ip") or headers.get("X-Real-IP") or "").strip() + if real: + return real + forwarded = headers.get("x-forwarded-for") or headers.get("X-Forwarded-For") + if forwarded: + first = forwarded.split(",")[0].strip() + if first: + return first + if request.client and request.client.host: + return request.client.host + return get_remote_address(request) + + +def attach_internal_service_rate_limit( + app: FastAPI, + *, + default_limit: str = "6000/minute", +) -> Limiter: + """ + Register SlowAPI on ``app`` with a default limit for all routes. + Use ``@limiter.exempt`` on ``/health`` (and similar) so probes are not throttled. + """ + limiter = Limiter( + key_func=_client_ip_key, + default_limits=[default_limit], + storage_uri="memory://", + ) + app.state.limiter = limiter + app.add_middleware(SlowAPIMiddleware) + app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) + return limiter diff --git a/backend/services/shared/message_retention.py b/backend/services/shared/message_retention.py new file mode 100644 index 0000000..1cca054 --- /dev/null +++ b/backend/services/shared/message_retention.py @@ -0,0 +1,106 @@ +""" +Single MESSAGE_RETENTION_DAYS policy (required env, no in-code default). + +- Positive float: age-based cleanup after that many days (same cutoff for compliance MEK, + soft-deleted DM keys, and edit-history rows). Value may be an arithmetic expression. +- 0: retain forever (no time-based cleanup; compliance MEK is still stored when a public key is configured). +- -1: do not store compliance-wrapped MEK; no time-based cleanup (same as 0 for expiry). +""" + +from __future__ import annotations + +import ast +import math +import os +from dataclasses import dataclass +from datetime import timedelta +MESSAGE_RETENTION_DAYS = "MESSAGE_RETENTION_DAYS" + +_state: MessageRetentionState | None = None + + +@dataclass(frozen=True) +class MessageRetentionState: + """Parsed MESSAGE_RETENTION_DAYS (days, after evaluating optional expression).""" + + days: float + + def never_store_compliance_mek(self) -> bool: + return self.days == -1.0 + + def cleanup_enabled(self) -> bool: + return self.days > 0.0 + + def retention_timedelta(self) -> timedelta: + return timedelta(days=self.days) + + +def _eval_numeric(node: ast.AST) -> float: + if isinstance(node, ast.Constant): + if isinstance(node.value, bool): + raise ValueError("MESSAGE_RETENTION_DAYS expression must be numeric") + if isinstance(node.value, (int, float)): + return float(node.value) + raise ValueError("MESSAGE_RETENTION_DAYS expression must be numeric") + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + return -_eval_numeric(node.operand) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.UAdd): + return _eval_numeric(node.operand) + if isinstance(node, ast.BinOp): + left = _eval_numeric(node.left) + right = _eval_numeric(node.right) + if isinstance(node.op, ast.Add): + return left + right + if isinstance(node.op, ast.Sub): + return left - right + if isinstance(node.op, ast.Mult): + return left * right + if isinstance(node.op, ast.Div): + return left / right + if isinstance(node.op, ast.FloorDiv): + return left // right + if isinstance(node.op, ast.Mod): + return left % right + if isinstance(node.op, ast.Pow): + return left ** right + raise ValueError("Unsupported operator in MESSAGE_RETENTION_DAYS") + if isinstance(node, ast.Num): # py<3.8 compatibility + return float(node.n) + raise ValueError("Unsupported syntax in MESSAGE_RETENTION_DAYS (only numbers and + - * / // % **)") + + +def eval_message_retention_expression(raw: str) -> float: + s = raw.strip() + if not s: + raise ValueError("MESSAGE_RETENTION_DAYS must not be empty") + tree = ast.parse(s, mode="eval") + if not isinstance(tree, ast.Expression): + raise ValueError("Invalid MESSAGE_RETENTION_DAYS expression") + value = _eval_numeric(tree.body) + if math.isnan(value) or math.isinf(value): + raise ValueError("MESSAGE_RETENTION_DAYS must be finite") + if value < 0 and value != -1.0: + raise ValueError("MESSAGE_RETENTION_DAYS must be >= 0, or exactly -1") + return value + + +def load_message_retention_from_env() -> MessageRetentionState: + raw = os.getenv(MESSAGE_RETENTION_DAYS) + if raw is None or not str(raw).strip(): + raise ValueError( + "MESSAGE_RETENTION_DAYS environment variable must be set " + "(float days; expressions like 1/24/60*5 allowed; 0 = retain forever; -1 = do not store compliance MEK)" + ) + return MessageRetentionState(days=eval_message_retention_expression(str(raw))) + + +def get_message_retention() -> MessageRetentionState: + global _state + if _state is None: + _state = load_message_retention_from_env() + return _state + + +def reset_message_retention_cache_for_tests() -> None: + global _state + _state = None diff --git a/deployment/README.md b/deployment/README.md index b89482a..5b63578 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -36,9 +36,11 @@ This directory contains the Docker configuration for the 3-service compliance ar ## Security Features -- **Network Isolation**: Messaging and file storage services have NO external network access +- **Network Isolation**: Messaging and file storage services attach only to the internal `services` network (`internal: true`) — no path to the public internet. PostgreSQL is on `services` only (not on `public`), so other `public`-only containers cannot reach the DB over Docker DNS; the host still uses the published `127.0.0.1:5432` port map. +- **Inter-service rate limits**: The messaging and file_storage apps use SlowAPI with a high per-IP default (`5000/minute`) plus an exempt `/health` route; traffic is mostly from the main service. The main API keeps finer per-route limits. +- **Firewall note**: Isolation is enforced with Docker networks (not iptables inside containers). Optional **gVisor / runsc** remains a manual host-level step (see plan); it is not automated here. - **Database Separation**: Each service has its own schema with minimal required permissions -- **Secure File Storage**: File storage uses restricted permissions and user isolation +- **Secure File Storage**: File storage uses restricted permissions and user isolation (stored files `chmod 600`, dirs `700`) - **Ephemeral Keys**: Messaging service generates temporary keys (never persisted) ## Environment Variables Required @@ -61,7 +63,7 @@ VAPID_PRIVATE_KEY=generated_vapid_private_key COMPLIANCE_PUBLIC_KEY=base64_encoded_public_key ``` -The main backend **requires** Firebase for Android push (FCM). It is not generated into `.env`: `docker-compose.yml` sets `FIREBASE_CERT` and read-only-mounts `backend/firebase-cert.json` from the repo. Place your Firebase service account JSON at `backend/firebase-cert.json` before `docker compose up` (gitignored; excluded from the image build via the repo-root `.dockerignore`). +The main backend **requires** Firebase for Android push (FCM). It is not generated into `.env`. The code loads `backend/firebase-cert.json` (path fixed relative to the backend tree); `docker-compose.yml` read-only-mounts that file into the container. Place your Firebase service account JSON at `backend/firebase-cert.json` before `docker compose up` (gitignored; excluded from the image build via the repo-root `.dockerignore`). ## Deployment Commands @@ -85,10 +87,10 @@ For local development, set `SERVICE_MODE=development` to run all services in a s ## Network Architecture -- **public**: External client access (main service, frontend, reverse proxy) -- **services**: Internal service communication only (database, messaging, file storage) -- Messaging and file storage services have NO external network access -- All inter-service communication is HTTP-based with proper authentication +- **public**: External client access (main service, frontend, reverse proxy). Main is also on `services` so it can reach Postgres, messaging, and file_storage. +- **services**: Internal bridge (`internal: true`). Postgres, messaging, file_storage, and main. The **frontend** is on both `public` and `services` so the Node server can reach `main` and `file_storage` (`FILE_STORAGE_HOST`) for SSR/proxy paths without exposing those backends on `public` directly. +- Messaging and file_storage are **not** on `public` and cannot reach the internet. +- Inter-service traffic is HTTP with shared middleware (request size cap, rate limits on internal apps). ## Database Schema Separation diff --git a/scripts/generate:env.sh b/scripts/generate:env.sh index 3d48597..2fcd2de 100755 --- a/scripts/generate:env.sh +++ b/scripts/generate:env.sh @@ -2,15 +2,14 @@ # ============================================================================= # _ENV_TEMPLATE: one KEY=value per line. Use for stdin prompts. Use # only where a dedicated step is needed. Any $(command) here runs when -# this script executes (after cd "$ROOT"). Piped stdin order: five lines -# (TURN_USERNAME, TURN_SECRET, DEPLOYMENT_SERVER, FIREBASE_CERT, RELEASES_TOKEN), +# this script executes (after cd "$ROOT"). Piped stdin order: four lines +# (TURN_USERNAME, TURN_SECRET, DEPLOYMENT_SERVER, RELEASES_TOKEN), # then commit (y/n), then deployment output directory (blank = deployment), then # writes /.env and /compliance_keypair.txt (default dir: deployment); then # if each target exists, backup prompt [Y/n] (Enter = yes; only n/no skips). # Nothing is written until commit=y (including compliance_keypair.txt). Backups after commit=y, default yes. -# Backups use deployment/.env.backup.<6-char sha256 prefix>.bak (git-style); same -# contents reuse one file. If that name exists with different content, full hash is used. -# Each written file uses .backup..bak beside the target (same hash rules). +# Backups use .<6-char sha256>.bak (same contents reuse one file). If that +# name exists with different content, full 64-char hash is used before .bak. # Template is read from fd 3 so stdin stays free. # ============================================================================= set -euo pipefail @@ -32,7 +31,6 @@ COMPLIANCE_PUBLIC_KEY= TURN_USERNAME= TURN_SECRET= DEPLOYMENT_SERVER= -FIREBASE_CERT= POSTGRES_PASSWORD=$(openssl rand -hex 8 .bak (use "${src}.backup") +# Backup path: {src}.{short-hash}.bak, or {src}.{full-hash}.bak on short-hash collision _do_backup_copy() { local src="$1" - local dest_prefix="$2" local full short dest full="$(openssl dgst -sha256 -r <"$src" | awk '{print $1}')" short="${full:0:6}" - dest="${dest_prefix}.${short}.bak" + dest="${src}.${short}.bak" if [[ -f "$dest" ]]; then if cmp -s "$src" "$dest"; then print_kv_row "backup" "$GRAY" "backup_unchanged" "$dest" return 0 fi - dest="${dest_prefix}.${full}.bak" + dest="${src}.${full}.bak" if [[ -f "$dest" ]] && cmp -s "$src" "$dest"; then print_kv_row "backup" "$GRAY" "backup_unchanged" "$dest" return 0 @@ -295,11 +292,11 @@ ENV_PATH="${DEPLOY_OUTPUT_DIR}/.env" COMPLIANCE_TXT="${DEPLOY_OUTPUT_DIR}/compliance_keypair.txt" if [[ -f "$ENV_PATH" ]] && read_yes_default_yes "File exists: ${ENV_PATH}. Create backup before overwrite? [Y/n]: "; then - _do_backup_copy "$ENV_PATH" "${ENV_PATH}.backup" + _do_backup_copy "$ENV_PATH" fi if [[ -f "$COMPLIANCE_TXT" ]] && read_yes_default_yes "File exists: ${COMPLIANCE_TXT}. Create backup before overwrite? [Y/n]: "; then - _do_backup_copy "$COMPLIANCE_TXT" "${COMPLIANCE_TXT}.backup" + _do_backup_copy "$COMPLIANCE_TXT" fi mkdir -p "$(dirname "$ENV_PATH")" diff --git a/scripts/generate_compliance_keypair.py b/scripts/generate_compliance_keypair.py index fc7e02e..73c6550 100644 --- a/scripts/generate_compliance_keypair.py +++ b/scripts/generate_compliance_keypair.py @@ -73,11 +73,21 @@ def main(): action="store_true", help="Output only the public key (for scripts)" ) - + parser.add_argument( + "--emit-key-lines", + action="store_true", + help="Print private key line then public key line to stdout only (no file; for generate:env.sh)", + ) + args = parser.parse_args() - + private_b64, public_b64 = generate_compliance_keypair() + if args.emit_key_lines: + print(private_b64) + print(public_b64) + return + if args.public_only: # Output only public key for script integration print(public_b64)