mirror of
https://github.com/fromchat-messenger/web.git
synced 2026-09-22 19:15:08 +03:00
Implement compliance key lifecycle on main service, add message retention and rate limits
This commit is contained in:
@@ -1,279 +0,0 @@
|
||||
"""
|
||||
Key Lifecycle Management for Compliance and Security.
|
||||
|
||||
This module handles automatic destruction of compliance keys, selective key destruction
|
||||
on message deletion, and configurable retention policies for cryptographic keys.
|
||||
|
||||
Key Features:
|
||||
- Automatic compliance key destruction (default: 6 months)
|
||||
- Selective key destruction on message deletion (default: 6 months)
|
||||
- Configurable retention policies
|
||||
- Background cleanup jobs for expired keys
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from typing import List, Optional
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..main.models import DMEnvelope, MessageEditHistory, DMEditHistory
|
||||
from .encryption import generate_nonce, TRANSPORT_NONCE_SIZE
|
||||
|
||||
logger = logging.getLogger("uvicorn.error")
|
||||
|
||||
# Default retention periods (in days)
|
||||
DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS = 180 # 6 months
|
||||
DEFAULT_MESSAGE_KEY_RETENTION_DAYS = 180 # 6 months for deleted messages
|
||||
|
||||
# Environment variable overrides
|
||||
COMPLIANCE_KEY_RETENTION_DAYS = int(os.getenv("COMPLIANCE_KEY_RETENTION_DAYS", DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS))
|
||||
MESSAGE_KEY_RETENTION_DAYS = int(os.getenv("MESSAGE_KEY_RETENTION_DAYS", DEFAULT_MESSAGE_KEY_RETENTION_DAYS))
|
||||
|
||||
|
||||
def get_compliance_key_retention_period() -> timedelta:
|
||||
"""Get the retention period for compliance keys."""
|
||||
return timedelta(days=COMPLIANCE_KEY_RETENTION_DAYS)
|
||||
|
||||
|
||||
def get_message_key_retention_period() -> timedelta:
|
||||
"""Get the retention period for message keys after deletion."""
|
||||
return timedelta(days=MESSAGE_KEY_RETENTION_DAYS)
|
||||
|
||||
|
||||
def destroy_compliance_keys_for_message(db: Session, message_id: int) -> int:
|
||||
"""
|
||||
Destroy compliance keys for a specific message.
|
||||
|
||||
This removes the compliance_wrapped_mek_b64 from DM envelopes,
|
||||
making the message permanently inaccessible for compliance purposes.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
message_id: ID of the message to destroy compliance keys for
|
||||
|
||||
Returns:
|
||||
Number of envelopes affected
|
||||
"""
|
||||
try:
|
||||
# Find all DM envelopes for this message
|
||||
envelopes = db.query(DMEnvelope).filter(DMEnvelope.id == message_id).all()
|
||||
|
||||
destroyed_count = 0
|
||||
for envelope in envelopes:
|
||||
if envelope.compliance_wrapped_mek_b64:
|
||||
envelope.compliance_wrapped_mek_b64 = None
|
||||
destroyed_count += 1
|
||||
|
||||
if destroyed_count > 0:
|
||||
db.commit()
|
||||
logger.info(f"Destroyed compliance keys for {destroyed_count} DM envelopes (message_id={message_id})")
|
||||
|
||||
return destroyed_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to destroy compliance keys for message {message_id}: {e}")
|
||||
db.rollback()
|
||||
return 0
|
||||
|
||||
|
||||
def destroy_compliance_keys_for_dm_envelope(db: Session, dm_envelope_id: int) -> bool:
|
||||
"""
|
||||
Destroy compliance key for a specific DM envelope.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
dm_envelope_id: ID of the DM envelope
|
||||
|
||||
Returns:
|
||||
True if key was destroyed, False otherwise
|
||||
"""
|
||||
try:
|
||||
envelope = db.query(DMEnvelope).filter(DMEnvelope.id == dm_envelope_id).first()
|
||||
if envelope and envelope.compliance_wrapped_mek_b64:
|
||||
envelope.compliance_wrapped_mek_b64 = None
|
||||
db.commit()
|
||||
logger.info(f"Destroyed compliance key for DM envelope {dm_envelope_id}")
|
||||
return True
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to destroy compliance key for DM envelope {dm_envelope_id}: {e}")
|
||||
db.rollback()
|
||||
return False
|
||||
|
||||
|
||||
def cleanup_expired_compliance_keys(db: Session) -> int:
|
||||
"""
|
||||
Clean up expired compliance keys based on retention policy.
|
||||
|
||||
This removes compliance_wrapped_mek_b64 from DM envelopes that are older
|
||||
than the retention period, making them permanently inaccessible for compliance.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Number of keys destroyed
|
||||
"""
|
||||
try:
|
||||
cutoff_date = datetime.now() - get_compliance_key_retention_period()
|
||||
|
||||
# Find DM envelopes older than retention period that still have compliance keys
|
||||
expired_envelopes = db.query(DMEnvelope).filter(
|
||||
DMEnvelope.timestamp < cutoff_date,
|
||||
DMEnvelope.compliance_wrapped_mek_b64.isnot(None)
|
||||
).all()
|
||||
|
||||
destroyed_count = 0
|
||||
for envelope in expired_envelopes:
|
||||
envelope.compliance_wrapped_mek_b64 = None
|
||||
destroyed_count += 1
|
||||
|
||||
if destroyed_count > 0:
|
||||
db.commit()
|
||||
logger.info(f"Cleaned up {destroyed_count} expired compliance keys (retention: {COMPLIANCE_KEY_RETENTION_DAYS} days)")
|
||||
|
||||
return destroyed_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cleanup expired compliance keys: {e}")
|
||||
db.rollback()
|
||||
return 0
|
||||
|
||||
|
||||
def cleanup_expired_message_keys(db: Session) -> int:
|
||||
"""
|
||||
Clean up message keys for deleted messages after retention period.
|
||||
|
||||
This removes sender and recipient wrapped keys from DM envelopes that have been
|
||||
soft-deleted and are past the retention period, making them completely inaccessible
|
||||
except through compliance access (which preserves the compliance key).
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Number of keys destroyed
|
||||
"""
|
||||
try:
|
||||
from datetime import datetime, timedelta
|
||||
from ..main.models import DMEnvelope
|
||||
|
||||
# Calculate cutoff date for expired messages
|
||||
cutoff_date = datetime.now() - get_message_key_retention_period()
|
||||
|
||||
# Find soft-deleted messages past retention period
|
||||
expired_messages = db.query(DMEnvelope).filter(
|
||||
DMEnvelope.deleted_at.is_not(None),
|
||||
DMEnvelope.deleted_at < cutoff_date
|
||||
).all()
|
||||
|
||||
if not expired_messages:
|
||||
logger.info("Message key cleanup: No expired deleted messages to process")
|
||||
return 0
|
||||
|
||||
keys_destroyed = 0
|
||||
|
||||
for message in expired_messages:
|
||||
# Destroy sender and recipient keys (compliance key remains for legal access)
|
||||
message.sender_wrapped_mek_b64 = ""
|
||||
message.recipient_wrapped_mek_b64 = ""
|
||||
keys_destroyed += 2
|
||||
|
||||
logger.info(
|
||||
"Destroyed keys for soft-deleted message id=%s (deleted %s)",
|
||||
message.id,
|
||||
message.deleted_at.isoformat()
|
||||
)
|
||||
|
||||
db.commit()
|
||||
logger.info("Message key cleanup: Destroyed %d keys across %d messages",
|
||||
keys_destroyed, len(expired_messages))
|
||||
|
||||
return keys_destroyed
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cleanup expired message keys: {e}")
|
||||
db.rollback()
|
||||
return 0
|
||||
|
||||
|
||||
def cleanup_expired_edit_history(db: Session) -> int:
|
||||
"""
|
||||
Clean up old edit history entries based on retention policy.
|
||||
|
||||
This removes edit history entries that are older than the compliance
|
||||
retention period.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Number of edit history entries removed
|
||||
"""
|
||||
try:
|
||||
cutoff_date = datetime.now() - get_compliance_key_retention_period()
|
||||
|
||||
# Clean up public message edit history
|
||||
public_deleted = db.query(MessageEditHistory).filter(
|
||||
MessageEditHistory.edited_at < cutoff_date
|
||||
).delete(synchronize_session=False)
|
||||
|
||||
# Clean up DM edit history
|
||||
dm_deleted = db.query(DMEditHistory).filter(
|
||||
DMEditHistory.edited_at < cutoff_date
|
||||
).delete(synchronize_session=False)
|
||||
|
||||
total_deleted = public_deleted + dm_deleted
|
||||
|
||||
if total_deleted > 0:
|
||||
db.commit()
|
||||
logger.info(f"Cleaned up {total_deleted} expired edit history entries (retention: {COMPLIANCE_KEY_RETENTION_DAYS} days)")
|
||||
|
||||
return total_deleted
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cleanup expired edit history: {e}")
|
||||
db.rollback()
|
||||
return 0
|
||||
|
||||
|
||||
def run_key_lifecycle_cleanup(db: Session) -> dict:
|
||||
"""
|
||||
Run all key lifecycle cleanup operations.
|
||||
|
||||
This should be called periodically (e.g., daily) to maintain key lifecycle policies.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Dict with cleanup statistics
|
||||
"""
|
||||
logger.info("Starting key lifecycle cleanup")
|
||||
|
||||
stats = {
|
||||
"compliance_keys_destroyed": cleanup_expired_compliance_keys(db),
|
||||
"message_keys_destroyed": cleanup_expired_message_keys(db),
|
||||
"edit_history_entries_removed": cleanup_expired_edit_history(db),
|
||||
"timestamp": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
logger.info(f"Key lifecycle cleanup completed: {stats}")
|
||||
return stats
|
||||
|
||||
|
||||
def get_key_lifecycle_config() -> dict:
|
||||
"""
|
||||
Get current key lifecycle configuration.
|
||||
|
||||
Returns:
|
||||
Dict with current configuration values
|
||||
"""
|
||||
return {
|
||||
"compliance_key_retention_days": COMPLIANCE_KEY_RETENTION_DAYS,
|
||||
"message_key_retention_days": MESSAGE_KEY_RETENTION_DAYS,
|
||||
"default_compliance_retention": DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS,
|
||||
"default_message_retention": DEFAULT_MESSAGE_KEY_RETENTION_DAYS
|
||||
}
|
||||
@@ -50,7 +50,19 @@ def _initialize_compliance_key():
|
||||
The private key never exists on the server - all decryption is done offline.
|
||||
"""
|
||||
global _COMPLIANCE_PUBLIC_KEY_B64
|
||||
|
||||
|
||||
try:
|
||||
from services.shared.message_retention import get_message_retention
|
||||
except ImportError:
|
||||
from backend.services.shared.message_retention import get_message_retention # type: ignore
|
||||
|
||||
if get_message_retention().never_store_compliance_mek():
|
||||
_COMPLIANCE_PUBLIC_KEY_B64 = ""
|
||||
logger.info(
|
||||
"Compliance MEK not stored (MESSAGE_RETENTION_DAYS=-1); COMPLIANCE_PUBLIC_KEY optional"
|
||||
)
|
||||
return
|
||||
|
||||
env_key = os.getenv("COMPLIANCE_PUBLIC_KEY", "").strip()
|
||||
if not env_key:
|
||||
raise RuntimeError(
|
||||
@@ -58,7 +70,7 @@ def _initialize_compliance_key():
|
||||
"Generate offline on an air-gapped machine: "
|
||||
"X25519 private key → export public key (base64) → set as env var"
|
||||
)
|
||||
|
||||
|
||||
_COMPLIANCE_PUBLIC_KEY_B64 = env_key
|
||||
logger.info("Loaded compliance public key from COMPLIANCE_PUBLIC_KEY environment variable")
|
||||
|
||||
@@ -151,6 +163,13 @@ except ImportError:
|
||||
if add_security_middleware:
|
||||
add_security_middleware(app)
|
||||
|
||||
try:
|
||||
from services.shared.inter_service_rate_limit import attach_internal_service_rate_limit
|
||||
except ImportError:
|
||||
from backend.services.shared.inter_service_rate_limit import attach_internal_service_rate_limit # type: ignore
|
||||
|
||||
_internal_limiter = attach_internal_service_rate_limit(app, default_limit="5000/minute")
|
||||
|
||||
# CORS configuration for inter-service communication
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
@@ -202,6 +221,7 @@ class ProcessMessageWithFilesRequest(ProcessMessageRequest):
|
||||
# ============================================================================
|
||||
|
||||
@app.get("/health", response_model=None)
|
||||
@_internal_limiter.exempt
|
||||
async def health_check():
|
||||
"""Health check endpoint for messaging service."""
|
||||
return {"status": "healthy", "service": "messaging"}
|
||||
|
||||
@@ -30,6 +30,14 @@ from .encryption import (
|
||||
logger = logging.getLogger("uvicorn.error")
|
||||
|
||||
|
||||
def _store_compliance_wrapped_mek() -> bool:
|
||||
try:
|
||||
from services.shared.message_retention import get_message_retention
|
||||
except ImportError:
|
||||
from backend.services.shared.message_retention import get_message_retention # type: ignore
|
||||
return not get_message_retention().never_store_compliance_mek()
|
||||
|
||||
|
||||
def process_encrypted_message(
|
||||
client_public_key_b64: str,
|
||||
transport_nonce_b64: str,
|
||||
@@ -101,32 +109,50 @@ def process_encrypted_message(
|
||||
# Use HKDF with recipient public key bytes as input to derive wrap keys
|
||||
# This is deterministic and doesn't require storing ephemeral keys
|
||||
import base64
|
||||
compliance_key_bytes = base64.b64decode(compliance_public_key_b64)
|
||||
sender_key_bytes = base64.b64decode(sender_public_key_b64)
|
||||
recipient_key_bytes = base64.b64decode(recipient_public_key_b64)
|
||||
|
||||
logger.info(f"🔑 Deriving wrap keys for sender={sender_public_key_b64[:20]}... recipient={recipient_public_key_b64[:20]}...")
|
||||
logger.info(
|
||||
"🔑 Deriving wrap keys for sender=%s... recipient=%s...",
|
||||
sender_public_key_b64[:20],
|
||||
recipient_public_key_b64[:20],
|
||||
)
|
||||
|
||||
compliance_wrap_key = derive_key_from_shared_secret(compliance_key_bytes, "compliance_wrap_key")
|
||||
sender_wrap_key = derive_key_from_shared_secret(sender_key_bytes, "sender_wrap_key")
|
||||
recipient_wrap_key = derive_key_from_shared_secret(recipient_key_bytes, "recipient_wrap_key")
|
||||
|
||||
logger.info("✅ Wrap keys derived successfully")
|
||||
logger.info("✅ Sender/recipient wrap keys derived successfully")
|
||||
|
||||
if _store_compliance_wrapped_mek():
|
||||
if not (compliance_public_key_b64 or "").strip():
|
||||
raise ValueError(
|
||||
"compliance public key required when MESSAGE_RETENTION_DAYS is not -1"
|
||||
)
|
||||
compliance_key_bytes = base64.b64decode(compliance_public_key_b64)
|
||||
compliance_wrap_key = derive_key_from_shared_secret(
|
||||
compliance_key_bytes, "compliance_wrap_key"
|
||||
)
|
||||
compliance_wrapped_mek = wrap_mek(mek, compliance_wrap_key)
|
||||
logger.info(
|
||||
"🔐 Compliance MEK: %s... (%s chars)",
|
||||
compliance_wrapped_mek[:30],
|
||||
len(compliance_wrapped_mek),
|
||||
)
|
||||
else:
|
||||
compliance_wrapped_mek = None
|
||||
logger.info("CRYPTO: Compliance MEK not stored (MESSAGE_RETENTION_DAYS=-1)")
|
||||
|
||||
# Step 4b: Wrap MEK for each recipient
|
||||
compliance_wrapped_mek = wrap_mek(mek, compliance_wrap_key)
|
||||
sender_wrapped_mek = wrap_mek(mek, sender_wrap_key)
|
||||
recipient_wrapped_mek = wrap_mek(mek, recipient_wrap_key)
|
||||
|
||||
logger.info(f"🔐 MEK wrapping complete:")
|
||||
logger.info(f" Compliance MEK: {compliance_wrapped_mek[:30]}... ({len(compliance_wrapped_mek)} chars)")
|
||||
logger.info(f" Sender MEK: {sender_wrapped_mek[:30]}... ({len(sender_wrapped_mek)} chars)")
|
||||
logger.info(f" Recipient MEK: {recipient_wrapped_mek[:30]}... ({len(recipient_wrapped_mek)} chars)")
|
||||
|
||||
duration = time.time() - start_time
|
||||
logger.info(
|
||||
"CRYPTO: Successfully processed message with 3 MEK wraps (compliance/sender/recipient) in %.2fms",
|
||||
duration * 1000
|
||||
"CRYPTO: Successfully processed message with MEK wraps in %.2fms",
|
||||
duration * 1000,
|
||||
)
|
||||
|
||||
# Get the transport public key for storage with the message
|
||||
@@ -251,15 +277,25 @@ def process_encrypted_message_and_files(
|
||||
files_out.append(entry)
|
||||
|
||||
# Derive wrap keys deterministically (same as existing flow)
|
||||
compliance_key_bytes = base64.b64decode(compliance_public_key_b64)
|
||||
sender_key_bytes = base64.b64decode(sender_public_key_b64)
|
||||
recipient_key_bytes = base64.b64decode(recipient_public_key_b64)
|
||||
|
||||
compliance_wrap_key = derive_key_from_shared_secret(compliance_key_bytes, "compliance_wrap_key")
|
||||
sender_wrap_key = derive_key_from_shared_secret(sender_key_bytes, "sender_wrap_key")
|
||||
recipient_wrap_key = derive_key_from_shared_secret(recipient_key_bytes, "recipient_wrap_key")
|
||||
|
||||
compliance_wrapped_mek = wrap_mek(mek, compliance_wrap_key)
|
||||
if _store_compliance_wrapped_mek():
|
||||
if not (compliance_public_key_b64 or "").strip():
|
||||
raise ValueError(
|
||||
"compliance public key required when MESSAGE_RETENTION_DAYS is not -1"
|
||||
)
|
||||
compliance_key_bytes = base64.b64decode(compliance_public_key_b64)
|
||||
compliance_wrap_key = derive_key_from_shared_secret(
|
||||
compliance_key_bytes, "compliance_wrap_key"
|
||||
)
|
||||
compliance_wrapped_mek = wrap_mek(mek, compliance_wrap_key)
|
||||
else:
|
||||
compliance_wrapped_mek = None
|
||||
|
||||
sender_wrapped_mek = wrap_mek(mek, sender_wrap_key)
|
||||
recipient_wrapped_mek = wrap_mek(mek, recipient_wrap_key)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user