Implement compliance key lifecycle on main service, add message retention and rate limits

backend/services/main/key_lifecycle.py

@@ -0,0 +1,200 @@
+"""
+Key lifecycle: time-based removal of compliance MEK, soft-deleted DM keys, and edit history.
+
+Uses MESSAGE_RETENTION_DAYS from the environment (see services.shared.message_retention).
+"""
+
+import logging
+from datetime import datetime
+from sqlalchemy.orm import Session
+
+from .models import DMEnvelope, MessageEditHistory, DMEditHistory
+
+logger = logging.getLogger("uvicorn.error")
+
+
+def _retention_timedelta_or_skip():
+    try:
+        from services.shared.message_retention import get_message_retention
+    except ImportError:
+        from backend.services.shared.message_retention import get_message_retention  # type: ignore
+    r = get_message_retention()
+    if not r.cleanup_enabled():
+        return None
+    return r.retention_timedelta()
+
+
+def destroy_compliance_keys_for_message(db: Session, message_id: int) -> int:
+    try:
+        envelopes = db.query(DMEnvelope).filter(DMEnvelope.id == message_id).all()
+
+        destroyed_count = 0
+        for envelope in envelopes:
+            if envelope.compliance_wrapped_mek_b64:
+                envelope.compliance_wrapped_mek_b64 = None
+                destroyed_count += 1
+
+        if destroyed_count > 0:
+            db.commit()
+            logger.info(
+                "Destroyed compliance keys for %s DM envelopes (message_id=%s)",
+                destroyed_count,
+                message_id,
+            )
+
+        return destroyed_count
+
+    except Exception as e:
+        logger.error("Failed to destroy compliance keys for message %s: %s", message_id, e)
+        db.rollback()
+        return 0
+
+
+def destroy_compliance_keys_for_dm_envelope(db: Session, dm_envelope_id: int) -> bool:
+    try:
+        envelope = db.query(DMEnvelope).filter(DMEnvelope.id == dm_envelope_id).first()
+        if envelope and envelope.compliance_wrapped_mek_b64:
+            envelope.compliance_wrapped_mek_b64 = None
+            db.commit()
+            logger.info("Destroyed compliance key for DM envelope %s", dm_envelope_id)
+            return True
+        return False
+
+    except Exception as e:
+        logger.error("Failed to destroy compliance key for DM envelope %s: %s", dm_envelope_id, e)
+        db.rollback()
+        return False
+
+
+def cleanup_expired_compliance_keys(db: Session) -> int:
+    delta = _retention_timedelta_or_skip()
+    if delta is None:
+        return 0
+
+    try:
+        cutoff_date = datetime.now() - delta
+
+        expired_envelopes = db.query(DMEnvelope).filter(
+            DMEnvelope.timestamp < cutoff_date,
+            DMEnvelope.compliance_wrapped_mek_b64.isnot(None),
+        ).all()
+
+        destroyed_count = 0
+        for envelope in expired_envelopes:
+            envelope.compliance_wrapped_mek_b64 = None
+            destroyed_count += 1
+
+        if destroyed_count > 0:
+            db.commit()
+            logger.info("Cleaned up %s expired compliance MEK fields", destroyed_count)
+
+        return destroyed_count
+
+    except Exception as e:
+        logger.error("Failed to cleanup expired compliance keys: %s", e)
+        db.rollback()
+        return 0
+
+
+def cleanup_expired_message_keys(db: Session) -> int:
+    delta = _retention_timedelta_or_skip()
+    if delta is None:
+        return 0
+
+    try:
+        cutoff_date = datetime.now() - delta
+
+        expired_messages = db.query(DMEnvelope).filter(
+            DMEnvelope.deleted_at.is_not(None),
+            DMEnvelope.deleted_at < cutoff_date,
+        ).all()
+
+        if not expired_messages:
+            return 0
+
+        keys_destroyed = 0
+
+        for message in expired_messages:
+            message.sender_wrapped_mek_b64 = ""
+            message.recipient_wrapped_mek_b64 = ""
+            keys_destroyed += 2
+
+            logger.debug(
+                "Destroyed keys for soft-deleted message id=%s (deleted %s)",
+                message.id,
+                message.deleted_at.isoformat(),
+            )
+
+        db.commit()
+        logger.info(
+            "Message key cleanup: destroyed %s keys across %s messages",
+            keys_destroyed,
+            len(expired_messages),
+        )
+
+        return keys_destroyed
+
+    except Exception as e:
+        logger.error("Failed to cleanup expired message keys: %s", e)
+        db.rollback()
+        return 0
+
+
+def cleanup_expired_edit_history(db: Session) -> int:
+    delta = _retention_timedelta_or_skip()
+    if delta is None:
+        return 0
+
+    try:
+        cutoff_date = datetime.now() - delta
+
+        public_deleted = db.query(MessageEditHistory).filter(
+            MessageEditHistory.edited_at < cutoff_date
+        ).delete(synchronize_session=False)
+
+        dm_deleted = db.query(DMEditHistory).filter(
+            DMEditHistory.edited_at < cutoff_date
+        ).delete(synchronize_session=False)
+
+        total_deleted = public_deleted + dm_deleted
+
+        if total_deleted > 0:
+            db.commit()
+            logger.info("Cleaned up %s expired edit history entries", total_deleted)
+
+        return total_deleted
+
+    except Exception as e:
+        logger.error("Failed to cleanup expired edit history: %s", e)
+        db.rollback()
+        return 0
+
+
+def run_key_lifecycle_cleanup(db: Session) -> dict:
+    stats = {
+        "compliance_keys_destroyed": cleanup_expired_compliance_keys(db),
+        "message_keys_destroyed": cleanup_expired_message_keys(db),
+        "edit_history_entries_removed": cleanup_expired_edit_history(db),
+        "timestamp": datetime.now().isoformat(),
+    }
+
+    if (
+        stats["compliance_keys_destroyed"]
+        or stats["message_keys_destroyed"]
+        or stats["edit_history_entries_removed"]
+    ):
+        logger.info("Key lifecycle cleanup completed: %s", stats)
+    return stats
+
+
+def get_key_lifecycle_config() -> dict:
+    try:
+        from services.shared.message_retention import get_message_retention
+    except ImportError:
+        from backend.services.shared.message_retention import get_message_retention  # type: ignore
+    r = get_message_retention()
+    return {
+        "message_retention_days": r.days,
+        "cleanup_enabled": r.cleanup_enabled(),
+        "never_store_compliance_mek": r.never_store_compliance_mek(),
+    }

backend/services/main/key_lifecycle_task.py

@@ -0,0 +1,45 @@
+"""
+Periodic key lifecycle cleanup (compliance MEK, deleted-message keys, edit history).
+Poll interval is derived from MESSAGE_RETENTION_DAYS (no separate env var).
+"""
+
+import asyncio
+import logging
+
+from .db import SessionLocal
+from .key_lifecycle import run_key_lifecycle_cleanup
+
+logger = logging.getLogger("uvicorn.error")
+
+
+def key_lifecycle_poll_seconds() -> int | None:
+    try:
+        from services.shared.message_retention import get_message_retention
+    except ImportError:
+        from backend.services.shared.message_retention import get_message_retention  # type: ignore
+    r = get_message_retention()
+    if not r.cleanup_enabled():
+        return None
+    sec = r.retention_timedelta().total_seconds()
+    # Bound poll: responsive after cutoff without hammering the DB
+    return max(15, min(3600, max(1, int(sec / 1000))))
+
+
+async def start_key_lifecycle_cleanup_task(interval_seconds: int) -> None:
+    while True:
+        try:
+            with SessionLocal() as db:
+                run_key_lifecycle_cleanup(db)
+        except asyncio.CancelledError:
+            break
+        except Exception as e:
+            logger.error("Error in key lifecycle cleanup task: %s", e)
+            try:
+                await asyncio.sleep(60)
+            except asyncio.CancelledError:
+                break
+            continue
+        try:
+            await asyncio.sleep(interval_seconds)
+        except asyncio.CancelledError:
+            break

backend/services/main/main.py

@@ -44,6 +44,9 @@ def _running_in_docker() -> bool:
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
+    cleanup_task = None
+    key_lifecycle_task = None
+
     # Startup - run migration in subprocess to avoid logging interference
     try:
         logger.info("Starting database migration check...")
@@ -122,6 +125,19 @@ async def lifespan(app: FastAPI):
         logger.error(f"Failed to start rate limit cleanup task: {e}")
         cleanup_task = None
 
+    try:
+        from .key_lifecycle_task import key_lifecycle_poll_seconds, start_key_lifecycle_cleanup_task
+        _poll = key_lifecycle_poll_seconds()
+        if _poll is not None:
+            key_lifecycle_task = asyncio.create_task(start_key_lifecycle_cleanup_task(_poll))
+            logger.info("Key lifecycle cleanup task started (interval=%ss)", _poll)
+        else:
+            key_lifecycle_task = None
+            logger.info("Key lifecycle cleanup disabled (MESSAGE_RETENTION_DAYS is 0 or -1)")
+    except Exception as e:
+        logger.error("Failed to start key lifecycle cleanup task: %s", e)
+        key_lifecycle_task = None
+
     yield
 
     # Shutdown - cancel cleanup task if it exists
@@ -132,6 +148,13 @@ async def lifespan(app: FastAPI):
         except asyncio.CancelledError:
             pass
 
+    if key_lifecycle_task:
+        key_lifecycle_task.cancel()
+        try:
+            await key_lifecycle_task
+        except asyncio.CancelledError:
+            pass
+
 # Initialize FastAPI
 app = FastAPI(title="FromChat", lifespan=lifespan)
 

backend/services/main/push_service.py

@@ -12,22 +12,19 @@ from firebase_admin import messaging as firebase_messaging
 
 logger = logging.getLogger("uvicorn.error")
 
+# backend/firebase-cert.json — fixed path; Docker bind-mounts this file to /app/firebase-cert.json
+_FIREBASE_CERT_PATH = Path(__file__).resolve().parents[2] / "firebase-cert.json"
 
-def _load_firebase_service_account_dict(firebase_cert: str) -> dict:
-    """Load Firebase service account JSON from FIREBASE_CERT path (relative to process cwd, e.g. backend/)."""
-    s = (firebase_cert or "").strip()
-    if not s:
-        raise RuntimeError("FIREBASE_CERT env variable is required (path to service account JSON file)")
 
-    p = Path(s).expanduser()
+def _load_firebase_service_account_dict(cert_path: Path) -> dict:
-    if not p.is_absolute():
+    """Load Firebase service account JSON from ``cert_path`` (must exist)."""
-        p = Path.cwd() / p
+    cert_path = cert_path.resolve()
-    if not p.is_file():
+    if not cert_path.is_file():
         raise FileNotFoundError(
-            f"FIREBASE_CERT is not a readable file: {p} (set FIREBASE_CERT to the JSON key path)"
+            f"Firebase credentials file missing or not a file: {cert_path} (expected backend/firebase-cert.json)"
         )
 
-    with p.open(encoding="utf-8") as f:
+    with cert_path.open(encoding="utf-8") as f:
         data = json.load(f)
     if not isinstance(data, dict) or data.get("type") != "service_account":
         raise ValueError("Firebase credentials file must be a service account JSON object")
@@ -38,23 +35,17 @@ class PushNotificationService:
     def __init__(self):
         self.vapid_private_key = os.getenv("VAPID_PRIVATE_KEY")
         self.vapid_public_key = os.getenv("VAPID_PUBLIC_KEY")
-        # Firebase Admin is required for main (FCM). FIREBASE_CERT = path to service account JSON.
+        # Firebase Admin is required for main (FCM); cert path is backend/firebase-cert.json.
         self.firebase_initialized = False
-        firebase_cert = os.getenv("FIREBASE_CERT")
-        if not (firebase_cert or "").strip():
-            raise RuntimeError(
-                "FIREBASE_CERT is required (path to Firebase service account JSON); "
-                "docker-compose sets this and bind-mounts backend/firebase-cert.json"
-            )
         try:
-            sa_dict = _load_firebase_service_account_dict(firebase_cert)
+            sa_dict = _load_firebase_service_account_dict(_FIREBASE_CERT_PATH)
 
             cred = firebase_credentials.Certificate(sa_dict)
             firebase_admin.initialize_app(cred)
             self.firebase_initialized = True
-            logger.info("Firebase Admin SDK initialized for push sending (FIREBASE_CERT)")
+            logger.info("Firebase Admin SDK initialized (%s)", _FIREBASE_CERT_PATH)
         except Exception as e:
-            logger.error(f"Failed to initialize Firebase Admin SDK from FIREBASE_CERT: {e}")
+            logger.error("Failed to initialize Firebase Admin SDK from %s: %s", _FIREBASE_CERT_PATH, e)
             raise
 
         if (not self.vapid_public_key) or (not self.vapid_private_key):
@@ -203,7 +194,7 @@ class PushNotificationService:
         """Send an FCM data-only push to a single device token using Firebase Admin SDK.
         Notification display is handled by the app, not FCM."""
         if not self.firebase_initialized:
-            raise RuntimeError("Firebase Admin SDK not initialized (FIREBASE_CERT required)")
+            raise RuntimeError("Firebase Admin SDK not initialized")
 
         try:
             # Send only data payload - let the app handle notification display

backend/services/main/service_calls.py

@@ -74,6 +74,13 @@ async def get_compliance_public_key(timeout: float = 5.0) -> Dict[str, Any]:
     """
     Return compliance system public key (for MEK wrapping).
     """
+    try:
+        from services.shared.message_retention import get_message_retention
+    except ImportError:
+        from backend.services.shared.message_retention import get_message_retention  # type: ignore
+    if get_message_retention().never_store_compliance_mek():
+        return {"public_key_b64": ""}
+
     mod = _get_messaging_module()
     if mod:
         # in-process async call

backend/services/messaging/key_lifecycle.py

@@ -1,279 +0,0 @@
-"""
-Key Lifecycle Management for Compliance and Security.
-
-This module handles automatic destruction of compliance keys, selective key destruction
-on message deletion, and configurable retention policies for cryptographic keys.
-
-Key Features:
-- Automatic compliance key destruction (default: 6 months)
-- Selective key destruction on message deletion (default: 6 months)
-- Configurable retention policies
-- Background cleanup jobs for expired keys
-"""
-
-import logging
-import os
-from datetime import datetime, timedelta
-from typing import List, Optional
-from sqlalchemy.orm import Session
-
-from ..main.models import DMEnvelope, MessageEditHistory, DMEditHistory
-from .encryption import generate_nonce, TRANSPORT_NONCE_SIZE
-
-logger = logging.getLogger("uvicorn.error")
-
-# Default retention periods (in days)
-DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS = 180  # 6 months
-DEFAULT_MESSAGE_KEY_RETENTION_DAYS = 180     # 6 months for deleted messages
-
-# Environment variable overrides
-COMPLIANCE_KEY_RETENTION_DAYS = int(os.getenv("COMPLIANCE_KEY_RETENTION_DAYS", DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS))
-MESSAGE_KEY_RETENTION_DAYS = int(os.getenv("MESSAGE_KEY_RETENTION_DAYS", DEFAULT_MESSAGE_KEY_RETENTION_DAYS))
-
-
-def get_compliance_key_retention_period() -> timedelta:
-    """Get the retention period for compliance keys."""
-    return timedelta(days=COMPLIANCE_KEY_RETENTION_DAYS)
-
-
-def get_message_key_retention_period() -> timedelta:
-    """Get the retention period for message keys after deletion."""
-    return timedelta(days=MESSAGE_KEY_RETENTION_DAYS)
-
-
-def destroy_compliance_keys_for_message(db: Session, message_id: int) -> int:
-    """
-    Destroy compliance keys for a specific message.
-
-    This removes the compliance_wrapped_mek_b64 from DM envelopes,
-    making the message permanently inaccessible for compliance purposes.
-
-    Args:
-        db: Database session
-        message_id: ID of the message to destroy compliance keys for
-
-    Returns:
-        Number of envelopes affected
-    """
-    try:
-        # Find all DM envelopes for this message
-        envelopes = db.query(DMEnvelope).filter(DMEnvelope.id == message_id).all()
-
-        destroyed_count = 0
-        for envelope in envelopes:
-            if envelope.compliance_wrapped_mek_b64:
-                envelope.compliance_wrapped_mek_b64 = None
-                destroyed_count += 1
-
-        if destroyed_count > 0:
-            db.commit()
-            logger.info(f"Destroyed compliance keys for {destroyed_count} DM envelopes (message_id={message_id})")
-
-        return destroyed_count
-
-    except Exception as e:
-        logger.error(f"Failed to destroy compliance keys for message {message_id}: {e}")
-        db.rollback()
-        return 0
-
-
-def destroy_compliance_keys_for_dm_envelope(db: Session, dm_envelope_id: int) -> bool:
-    """
-    Destroy compliance key for a specific DM envelope.
-
-    Args:
-        db: Database session
-        dm_envelope_id: ID of the DM envelope
-
-    Returns:
-        True if key was destroyed, False otherwise
-    """
-    try:
-        envelope = db.query(DMEnvelope).filter(DMEnvelope.id == dm_envelope_id).first()
-        if envelope and envelope.compliance_wrapped_mek_b64:
-            envelope.compliance_wrapped_mek_b64 = None
-            db.commit()
-            logger.info(f"Destroyed compliance key for DM envelope {dm_envelope_id}")
-            return True
-        return False
-
-    except Exception as e:
-        logger.error(f"Failed to destroy compliance key for DM envelope {dm_envelope_id}: {e}")
-        db.rollback()
-        return False
-
-
-def cleanup_expired_compliance_keys(db: Session) -> int:
-    """
-    Clean up expired compliance keys based on retention policy.
-
-    This removes compliance_wrapped_mek_b64 from DM envelopes that are older
-    than the retention period, making them permanently inaccessible for compliance.
-
-    Args:
-        db: Database session
-
-    Returns:
-        Number of keys destroyed
-    """
-    try:
-        cutoff_date = datetime.now() - get_compliance_key_retention_period()
-
-        # Find DM envelopes older than retention period that still have compliance keys
-        expired_envelopes = db.query(DMEnvelope).filter(
-            DMEnvelope.timestamp < cutoff_date,
-            DMEnvelope.compliance_wrapped_mek_b64.isnot(None)
-        ).all()
-
-        destroyed_count = 0
-        for envelope in expired_envelopes:
-            envelope.compliance_wrapped_mek_b64 = None
-            destroyed_count += 1
-
-        if destroyed_count > 0:
-            db.commit()
-            logger.info(f"Cleaned up {destroyed_count} expired compliance keys (retention: {COMPLIANCE_KEY_RETENTION_DAYS} days)")
-
-        return destroyed_count
-
-    except Exception as e:
-        logger.error(f"Failed to cleanup expired compliance keys: {e}")
-        db.rollback()
-        return 0
-
-
-def cleanup_expired_message_keys(db: Session) -> int:
-    """
-    Clean up message keys for deleted messages after retention period.
-
-    This removes sender and recipient wrapped keys from DM envelopes that have been
-    soft-deleted and are past the retention period, making them completely inaccessible
-    except through compliance access (which preserves the compliance key).
-
-    Args:
-        db: Database session
-
-    Returns:
-        Number of keys destroyed
-    """
-    try:
-        from datetime import datetime, timedelta
-        from ..main.models import DMEnvelope
-
-        # Calculate cutoff date for expired messages
-        cutoff_date = datetime.now() - get_message_key_retention_period()
-
-        # Find soft-deleted messages past retention period
-        expired_messages = db.query(DMEnvelope).filter(
-            DMEnvelope.deleted_at.is_not(None),
-            DMEnvelope.deleted_at < cutoff_date
-        ).all()
-
-        if not expired_messages:
-            logger.info("Message key cleanup: No expired deleted messages to process")
-            return 0
-
-        keys_destroyed = 0
-
-        for message in expired_messages:
-            # Destroy sender and recipient keys (compliance key remains for legal access)
-            message.sender_wrapped_mek_b64 = ""
-            message.recipient_wrapped_mek_b64 = ""
-            keys_destroyed += 2
-
-            logger.info(
-                "Destroyed keys for soft-deleted message id=%s (deleted %s)",
-                message.id,
-                message.deleted_at.isoformat()
-            )
-
-        db.commit()
-        logger.info("Message key cleanup: Destroyed %d keys across %d messages",
-                   keys_destroyed, len(expired_messages))
-
-        return keys_destroyed
-
-    except Exception as e:
-        logger.error(f"Failed to cleanup expired message keys: {e}")
-        db.rollback()
-        return 0
-
-
-def cleanup_expired_edit_history(db: Session) -> int:
-    """
-    Clean up old edit history entries based on retention policy.
-
-    This removes edit history entries that are older than the compliance
-    retention period.
-
-    Args:
-        db: Database session
-
-    Returns:
-        Number of edit history entries removed
-    """
-    try:
-        cutoff_date = datetime.now() - get_compliance_key_retention_period()
-
-        # Clean up public message edit history
-        public_deleted = db.query(MessageEditHistory).filter(
-            MessageEditHistory.edited_at < cutoff_date
-        ).delete(synchronize_session=False)
-
-        # Clean up DM edit history
-        dm_deleted = db.query(DMEditHistory).filter(
-            DMEditHistory.edited_at < cutoff_date
-        ).delete(synchronize_session=False)
-
-        total_deleted = public_deleted + dm_deleted
-
-        if total_deleted > 0:
-            db.commit()
-            logger.info(f"Cleaned up {total_deleted} expired edit history entries (retention: {COMPLIANCE_KEY_RETENTION_DAYS} days)")
-
-        return total_deleted
-
-    except Exception as e:
-        logger.error(f"Failed to cleanup expired edit history: {e}")
-        db.rollback()
-        return 0
-
-
-def run_key_lifecycle_cleanup(db: Session) -> dict:
-    """
-    Run all key lifecycle cleanup operations.
-
-    This should be called periodically (e.g., daily) to maintain key lifecycle policies.
-
-    Args:
-        db: Database session
-
-    Returns:
-        Dict with cleanup statistics
-    """
-    logger.info("Starting key lifecycle cleanup")
-
-    stats = {
-        "compliance_keys_destroyed": cleanup_expired_compliance_keys(db),
-        "message_keys_destroyed": cleanup_expired_message_keys(db),
-        "edit_history_entries_removed": cleanup_expired_edit_history(db),
-        "timestamp": datetime.now().isoformat()
-    }
-
-    logger.info(f"Key lifecycle cleanup completed: {stats}")
-    return stats
-
-
-def get_key_lifecycle_config() -> dict:
-    """
-    Get current key lifecycle configuration.
-
-    Returns:
-        Dict with current configuration values
-    """
-    return {
-        "compliance_key_retention_days": COMPLIANCE_KEY_RETENTION_DAYS,
-        "message_key_retention_days": MESSAGE_KEY_RETENTION_DAYS,
-        "default_compliance_retention": DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS,
-        "default_message_retention": DEFAULT_MESSAGE_KEY_RETENTION_DAYS
-    }

backend/services/messaging/main.py

@@ -51,6 +51,18 @@ def _initialize_compliance_key():
     """
     global _COMPLIANCE_PUBLIC_KEY_B64
 
+    try:
+        from services.shared.message_retention import get_message_retention
+    except ImportError:
+        from backend.services.shared.message_retention import get_message_retention  # type: ignore
+
+    if get_message_retention().never_store_compliance_mek():
+        _COMPLIANCE_PUBLIC_KEY_B64 = ""
+        logger.info(
+            "Compliance MEK not stored (MESSAGE_RETENTION_DAYS=-1); COMPLIANCE_PUBLIC_KEY optional"
+        )
+        return
+
     env_key = os.getenv("COMPLIANCE_PUBLIC_KEY", "").strip()
     if not env_key:
         raise RuntimeError(
@@ -151,6 +163,13 @@ except ImportError:
 if add_security_middleware:
     add_security_middleware(app)
 
+try:
+    from services.shared.inter_service_rate_limit import attach_internal_service_rate_limit
+except ImportError:
+    from backend.services.shared.inter_service_rate_limit import attach_internal_service_rate_limit  # type: ignore
+
+_internal_limiter = attach_internal_service_rate_limit(app, default_limit="5000/minute")
+
 # CORS configuration for inter-service communication
 app.add_middleware(
     CORSMiddleware,
@@ -202,6 +221,7 @@ class ProcessMessageWithFilesRequest(ProcessMessageRequest):
 # ============================================================================
 
 @app.get("/health", response_model=None)
+@_internal_limiter.exempt
 async def health_check():
     """Health check endpoint for messaging service."""
     return {"status": "healthy", "service": "messaging"}

backend/services/messaging/processor.py

@@ -30,6 +30,14 @@ from .encryption import (
 logger = logging.getLogger("uvicorn.error")
 
 
+def _store_compliance_wrapped_mek() -> bool:
+    try:
+        from services.shared.message_retention import get_message_retention
+    except ImportError:
+        from backend.services.shared.message_retention import get_message_retention  # type: ignore
+    return not get_message_retention().never_store_compliance_mek()
+
+
 def process_encrypted_message(
     client_public_key_b64: str,
     transport_nonce_b64: str,
@@ -101,32 +109,50 @@ def process_encrypted_message(
         # Use HKDF with recipient public key bytes as input to derive wrap keys
         # This is deterministic and doesn't require storing ephemeral keys
         import base64
-        compliance_key_bytes = base64.b64decode(compliance_public_key_b64)
         sender_key_bytes = base64.b64decode(sender_public_key_b64)
         recipient_key_bytes = base64.b64decode(recipient_public_key_b64)
 
-        logger.info(f"🔑 Deriving wrap keys for sender={sender_public_key_b64[:20]}... recipient={recipient_public_key_b64[:20]}...")
+        logger.info(
+            "🔑 Deriving wrap keys for sender=%s... recipient=%s...",
+            sender_public_key_b64[:20],
+            recipient_public_key_b64[:20],
+        )
 
-        compliance_wrap_key = derive_key_from_shared_secret(compliance_key_bytes, "compliance_wrap_key")
         sender_wrap_key = derive_key_from_shared_secret(sender_key_bytes, "sender_wrap_key")
         recipient_wrap_key = derive_key_from_shared_secret(recipient_key_bytes, "recipient_wrap_key")
 
-        logger.info("✅ Wrap keys derived successfully")
+        logger.info("✅ Sender/recipient wrap keys derived successfully")
 
-        # Step 4b: Wrap MEK for each recipient
+        if _store_compliance_wrapped_mek():
+            if not (compliance_public_key_b64 or "").strip():
+                raise ValueError(
+                    "compliance public key required when MESSAGE_RETENTION_DAYS is not -1"
+                )
+            compliance_key_bytes = base64.b64decode(compliance_public_key_b64)
+            compliance_wrap_key = derive_key_from_shared_secret(
+                compliance_key_bytes, "compliance_wrap_key"
+            )
             compliance_wrapped_mek = wrap_mek(mek, compliance_wrap_key)
+            logger.info(
+                "🔐 Compliance MEK: %s... (%s chars)",
+                compliance_wrapped_mek[:30],
+                len(compliance_wrapped_mek),
+            )
+        else:
+            compliance_wrapped_mek = None
+            logger.info("CRYPTO: Compliance MEK not stored (MESSAGE_RETENTION_DAYS=-1)")
+
         sender_wrapped_mek = wrap_mek(mek, sender_wrap_key)
         recipient_wrapped_mek = wrap_mek(mek, recipient_wrap_key)
 
         logger.info(f"🔐 MEK wrapping complete:")
-        logger.info(f"   Compliance MEK: {compliance_wrapped_mek[:30]}... ({len(compliance_wrapped_mek)} chars)")
         logger.info(f"   Sender MEK: {sender_wrapped_mek[:30]}... ({len(sender_wrapped_mek)} chars)")
         logger.info(f"   Recipient MEK: {recipient_wrapped_mek[:30]}... ({len(recipient_wrapped_mek)} chars)")
 
         duration = time.time() - start_time
         logger.info(
-            "CRYPTO: Successfully processed message with 3 MEK wraps (compliance/sender/recipient) in %.2fms",
+            "CRYPTO: Successfully processed message with MEK wraps in %.2fms",
-            duration * 1000
+            duration * 1000,
         )
 
         # Get the transport public key for storage with the message
@@ -251,15 +277,25 @@ def process_encrypted_message_and_files(
         files_out.append(entry)
 
     # Derive wrap keys deterministically (same as existing flow)
-    compliance_key_bytes = base64.b64decode(compliance_public_key_b64)
     sender_key_bytes = base64.b64decode(sender_public_key_b64)
     recipient_key_bytes = base64.b64decode(recipient_public_key_b64)
 
-    compliance_wrap_key = derive_key_from_shared_secret(compliance_key_bytes, "compliance_wrap_key")
     sender_wrap_key = derive_key_from_shared_secret(sender_key_bytes, "sender_wrap_key")
     recipient_wrap_key = derive_key_from_shared_secret(recipient_key_bytes, "recipient_wrap_key")
 
+    if _store_compliance_wrapped_mek():
+        if not (compliance_public_key_b64 or "").strip():
+            raise ValueError(
+                "compliance public key required when MESSAGE_RETENTION_DAYS is not -1"
+            )
+        compliance_key_bytes = base64.b64decode(compliance_public_key_b64)
+        compliance_wrap_key = derive_key_from_shared_secret(
+            compliance_key_bytes, "compliance_wrap_key"
+        )
         compliance_wrapped_mek = wrap_mek(mek, compliance_wrap_key)
+    else:
+        compliance_wrapped_mek = None
+
     sender_wrapped_mek = wrap_mek(mek, sender_wrap_key)
     recipient_wrapped_mek = wrap_mek(mek, recipient_wrap_key)
 

backend/services/shared/inter_service_rate_limit.py

@@ -0,0 +1,51 @@
+"""
+Per-IP rate limits for internal FastAPI apps (messaging, file_storage).
+
+Complements the main service's endpoint-specific limits. Uses a generous default
+because traffic is mostly from the main backend (single Docker bridge IP).
+"""
+
+from __future__ import annotations
+
+from fastapi import FastAPI, Request
+from slowapi import Limiter, _rate_limit_exceeded_handler
+from slowapi.errors import RateLimitExceeded
+from slowapi.middleware import SlowAPIMiddleware
+from slowapi.util import get_remote_address
+
+
+def _client_ip_key(request: Request) -> str:
+    if request is None:
+        return "unknown"
+    headers = request.headers
+    real = (headers.get("x-real-ip") or headers.get("X-Real-IP") or "").strip()
+    if real:
+        return real
+    forwarded = headers.get("x-forwarded-for") or headers.get("X-Forwarded-For")
+    if forwarded:
+        first = forwarded.split(",")[0].strip()
+        if first:
+            return first
+    if request.client and request.client.host:
+        return request.client.host
+    return get_remote_address(request)
+
+
+def attach_internal_service_rate_limit(
+    app: FastAPI,
+    *,
+    default_limit: str = "6000/minute",
+) -> Limiter:
+    """
+    Register SlowAPI on ``app`` with a default limit for all routes.
+    Use ``@limiter.exempt`` on ``/health`` (and similar) so probes are not throttled.
+    """
+    limiter = Limiter(
+        key_func=_client_ip_key,
+        default_limits=[default_limit],
+        storage_uri="memory://",
+    )
+    app.state.limiter = limiter
+    app.add_middleware(SlowAPIMiddleware)
+    app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
+    return limiter

backend/services/shared/message_retention.py

@@ -0,0 +1,106 @@
+"""
+Single MESSAGE_RETENTION_DAYS policy (required env, no in-code default).
+
+- Positive float: age-based cleanup after that many days (same cutoff for compliance MEK,
+  soft-deleted DM keys, and edit-history rows). Value may be an arithmetic expression.
+- 0: retain forever (no time-based cleanup; compliance MEK is still stored when a public key is configured).
+- -1: do not store compliance-wrapped MEK; no time-based cleanup (same as 0 for expiry).
+"""
+
+from __future__ import annotations
+
+import ast
+import math
+import os
+from dataclasses import dataclass
+from datetime import timedelta
+MESSAGE_RETENTION_DAYS = "MESSAGE_RETENTION_DAYS"
+
+_state: MessageRetentionState | None = None
+
+
+@dataclass(frozen=True)
+class MessageRetentionState:
+    """Parsed MESSAGE_RETENTION_DAYS (days, after evaluating optional expression)."""
+
+    days: float
+
+    def never_store_compliance_mek(self) -> bool:
+        return self.days == -1.0
+
+    def cleanup_enabled(self) -> bool:
+        return self.days > 0.0
+
+    def retention_timedelta(self) -> timedelta:
+        return timedelta(days=self.days)
+
+
+def _eval_numeric(node: ast.AST) -> float:
+    if isinstance(node, ast.Constant):
+        if isinstance(node.value, bool):
+            raise ValueError("MESSAGE_RETENTION_DAYS expression must be numeric")
+        if isinstance(node.value, (int, float)):
+            return float(node.value)
+        raise ValueError("MESSAGE_RETENTION_DAYS expression must be numeric")
+    if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
+        return -_eval_numeric(node.operand)
+    if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.UAdd):
+        return _eval_numeric(node.operand)
+    if isinstance(node, ast.BinOp):
+        left = _eval_numeric(node.left)
+        right = _eval_numeric(node.right)
+        if isinstance(node.op, ast.Add):
+            return left + right
+        if isinstance(node.op, ast.Sub):
+            return left - right
+        if isinstance(node.op, ast.Mult):
+            return left * right
+        if isinstance(node.op, ast.Div):
+            return left / right
+        if isinstance(node.op, ast.FloorDiv):
+            return left // right
+        if isinstance(node.op, ast.Mod):
+            return left % right
+        if isinstance(node.op, ast.Pow):
+            return left ** right
+        raise ValueError("Unsupported operator in MESSAGE_RETENTION_DAYS")
+    if isinstance(node, ast.Num):  # py<3.8 compatibility
+        return float(node.n)
+    raise ValueError("Unsupported syntax in MESSAGE_RETENTION_DAYS (only numbers and + - * / // % **)")
+
+
+def eval_message_retention_expression(raw: str) -> float:
+    s = raw.strip()
+    if not s:
+        raise ValueError("MESSAGE_RETENTION_DAYS must not be empty")
+    tree = ast.parse(s, mode="eval")
+    if not isinstance(tree, ast.Expression):
+        raise ValueError("Invalid MESSAGE_RETENTION_DAYS expression")
+    value = _eval_numeric(tree.body)
+    if math.isnan(value) or math.isinf(value):
+        raise ValueError("MESSAGE_RETENTION_DAYS must be finite")
+    if value < 0 and value != -1.0:
+        raise ValueError("MESSAGE_RETENTION_DAYS must be >= 0, or exactly -1")
+    return value
+
+
+def load_message_retention_from_env() -> MessageRetentionState:
+    raw = os.getenv(MESSAGE_RETENTION_DAYS)
+    if raw is None or not str(raw).strip():
+        raise ValueError(
+            "MESSAGE_RETENTION_DAYS environment variable must be set "
+            "(float days; expressions like 1/24/60*5 allowed; 0 = retain forever; -1 = do not store compliance MEK)"
+        )
+    return MessageRetentionState(days=eval_message_retention_expression(str(raw)))
+
+
+def get_message_retention() -> MessageRetentionState:
+    global _state
+    if _state is None:
+        _state = load_message_retention_from_env()
+    return _state
+
+
+def reset_message_retention_cache_for_tests() -> None:
+    global _state
+    _state = None

deployment/README.md

@@ -36,9 +36,11 @@ This directory contains the Docker configuration for the 3-service compliance ar
 
 ## Security Features
 
-- **Network Isolation**: Messaging and file storage services have NO external network access
+- **Network Isolation**: Messaging and file storage services attach only to the internal `services` network (`internal: true`) — no path to the public internet. PostgreSQL is on `services` only (not on `public`), so other `public`-only containers cannot reach the DB over Docker DNS; the host still uses the published `127.0.0.1:5432` port map.
+- **Inter-service rate limits**: The messaging and file_storage apps use SlowAPI with a high per-IP default (`5000/minute`) plus an exempt `/health` route; traffic is mostly from the main service. The main API keeps finer per-route limits.
+- **Firewall note**: Isolation is enforced with Docker networks (not iptables inside containers). Optional **gVisor / runsc** remains a manual host-level step (see plan); it is not automated here.
 - **Database Separation**: Each service has its own schema with minimal required permissions
-- **Secure File Storage**: File storage uses restricted permissions and user isolation
+- **Secure File Storage**: File storage uses restricted permissions and user isolation (stored files `chmod 600`, dirs `700`)
 - **Ephemeral Keys**: Messaging service generates temporary keys (never persisted)
 
 ## Environment Variables Required
@@ -61,7 +63,7 @@ VAPID_PRIVATE_KEY=generated_vapid_private_key
 COMPLIANCE_PUBLIC_KEY=base64_encoded_public_key
 ```
 
-The main backend **requires** Firebase for Android push (FCM). It is not generated into `.env`: `docker-compose.yml` sets `FIREBASE_CERT` and read-only-mounts `backend/firebase-cert.json` from the repo. Place your Firebase service account JSON at `backend/firebase-cert.json` before `docker compose up` (gitignored; excluded from the image build via the repo-root `.dockerignore`).
+The main backend **requires** Firebase for Android push (FCM). It is not generated into `.env`. The code loads `backend/firebase-cert.json` (path fixed relative to the backend tree); `docker-compose.yml` read-only-mounts that file into the container. Place your Firebase service account JSON at `backend/firebase-cert.json` before `docker compose up` (gitignored; excluded from the image build via the repo-root `.dockerignore`).
 
 ## Deployment Commands
 
@@ -85,10 +87,10 @@ For local development, set `SERVICE_MODE=development` to run all services in a s
 
 ## Network Architecture
 
-- **public**: External client access (main service, frontend, reverse proxy)
+- **public**: External client access (main service, frontend, reverse proxy). Main is also on `services` so it can reach Postgres, messaging, and file_storage.
-- **services**: Internal service communication only (database, messaging, file storage)
+- **services**: Internal bridge (`internal: true`). Postgres, messaging, file_storage, and main. The **frontend** is on both `public` and `services` so the Node server can reach `main` and `file_storage` (`FILE_STORAGE_HOST`) for SSR/proxy paths without exposing those backends on `public` directly.
-- Messaging and file storage services have NO external network access
+- Messaging and file_storage are **not** on `public` and cannot reach the internet.
-- All inter-service communication is HTTP-based with proper authentication
+- Inter-service traffic is HTTP with shared middleware (request size cap, rate limits on internal apps).
 
 ## Database Schema Separation
 

scripts/generate:env.sh

@@ -2,15 +2,14 @@
 # =============================================================================
 # _ENV_TEMPLATE: one KEY=value per line. Use <set> for stdin prompts. Use
 # <gen:…> only where a dedicated step is needed. Any $(command) here runs when
-# this script executes (after cd "$ROOT"). Piped stdin order: five <set> lines
+# this script executes (after cd "$ROOT"). Piped stdin order: four <set> lines
-# (TURN_USERNAME, TURN_SECRET, DEPLOYMENT_SERVER, FIREBASE_CERT, RELEASES_TOKEN),
+# (TURN_USERNAME, TURN_SECRET, DEPLOYMENT_SERVER, RELEASES_TOKEN),
 # then commit (y/n), then deployment output directory (blank = deployment), then
 # writes <dir>/.env and <dir>/compliance_keypair.txt (default dir: deployment); then
 # if each target exists, backup prompt [Y/n] (Enter = yes; only n/no skips).
 # Nothing is written until commit=y (including compliance_keypair.txt). Backups after commit=y, default yes.
-# Backups use deployment/.env.backup.<6-char sha256 prefix>.bak (git-style); same
+# Backups use <original-path>.<6-char sha256>.bak (same contents reuse one file). If that
-# contents reuse one file. If that name exists with different content, full hash is used.
+# name exists with different content, full 64-char hash is used before .bak.
-# Each written file uses <path>.backup.<short>.bak beside the target (same hash rules).
 # Template is read from fd 3 so stdin stays free.
 # =============================================================================
 set -euo pipefail
@@ -32,7 +31,6 @@ COMPLIANCE_PUBLIC_KEY=<gen:compliance>
 TURN_USERNAME=<set>
 TURN_SECRET=<set>
 DEPLOYMENT_SERVER=<set>
-FIREBASE_CERT=<set>
 POSTGRES_PASSWORD=$(openssl rand -hex 8 </dev/null)
 MAIN_DB_PASSWORD=$(openssl rand -hex 8 </dev/null)
 MESSAGING_DB_PASSWORD=$(openssl rand -hex 8 </dev/null)
@@ -147,20 +145,19 @@ prompt_set() {
   done
 }
 
-# $2 = backup path prefix without .<hash>.bak (use "${src}.backup")
+# Backup path: {src}.{short-hash}.bak, or {src}.{full-hash}.bak on short-hash collision
 _do_backup_copy() {
   local src="$1"
-  local dest_prefix="$2"
   local full short dest
   full="$(openssl dgst -sha256 -r <"$src" | awk '{print $1}')"
   short="${full:0:6}"
-  dest="${dest_prefix}.${short}.bak"
+  dest="${src}.${short}.bak"
   if [[ -f "$dest" ]]; then
     if cmp -s "$src" "$dest"; then
       print_kv_row "backup" "$GRAY" "backup_unchanged" "$dest"
       return 0
     fi
-    dest="${dest_prefix}.${full}.bak"
+    dest="${src}.${full}.bak"
     if [[ -f "$dest" ]] && cmp -s "$src" "$dest"; then
       print_kv_row "backup" "$GRAY" "backup_unchanged" "$dest"
       return 0
@@ -295,11 +292,11 @@ ENV_PATH="${DEPLOY_OUTPUT_DIR}/.env"
 COMPLIANCE_TXT="${DEPLOY_OUTPUT_DIR}/compliance_keypair.txt"
 
 if [[ -f "$ENV_PATH" ]] && read_yes_default_yes "File exists: ${ENV_PATH}. Create backup before overwrite? [Y/n]: "; then
-  _do_backup_copy "$ENV_PATH" "${ENV_PATH}.backup"
+  _do_backup_copy "$ENV_PATH"
 fi
 
 if [[ -f "$COMPLIANCE_TXT" ]] && read_yes_default_yes "File exists: ${COMPLIANCE_TXT}. Create backup before overwrite? [Y/n]: "; then
-  _do_backup_copy "$COMPLIANCE_TXT" "${COMPLIANCE_TXT}.backup"
+  _do_backup_copy "$COMPLIANCE_TXT"
 fi
 
 mkdir -p "$(dirname "$ENV_PATH")"

scripts/generate_compliance_keypair.py

@@ -73,11 +73,21 @@ def main():
         action="store_true",
         help="Output only the public key (for scripts)"
     )
+    parser.add_argument(
+        "--emit-key-lines",
+        action="store_true",
+        help="Print private key line then public key line to stdout only (no file; for generate:env.sh)",
+    )
 
     args = parser.parse_args()
 
     private_b64, public_b64 = generate_compliance_keypair()
 
+    if args.emit_key_lines:
+        print(private_b64)
+        print(public_b64)
+        return
+
     if args.public_only:
         # Output only public key for script integration
         print(public_b64)