Add multi-service Docker setup, Postgres migrations, and messaging API cleanup

backend/services/main/constants.py

@@ -1,7 +1,7 @@
 import os
 
-# Database is always in backend/data/ relative to project root
-DATABASE_URL = "sqlite:///" + os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "data", "database.db")
+# Database URL from environment (Docker) or fallback to SQLite (development)
+DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///" + os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "data", "database.db"))
 JWT_ALGORITHM = "HS256"
 # Token inactivity expiration - token expires if not used for this duration
 TOKEN_INACTIVITY_EXPIRE_HOURS = 30 * 24  # 30 days of inactivity

backend/services/main/main.py

@@ -7,6 +7,7 @@ import subprocess
 import sys
 import os
 import logging
+from sqlalchemy.orm.exc import DetachedInstanceError
 
 # Import from same directory
 from .routes import account, messaging, profile, push, webrtc, devices, moderation, download, keys, envelope_messaging
@@ -166,18 +167,55 @@ if not _running_in_docker():
         logger.warning(f"Failed to mount internal services for development: {e}")
 
 
+def _get_username_for_log(user) -> str | None:
+    """
+    Safely extract username for access logs.
+
+    If the ORM instance is detached, we transparently open a short-lived session,
+    reload the user by ID and read the username from that fresh instance.
+    Logging must never break request handling.
+    """
+    if user is None:
+        return None
+
+    # Fast path: instance is still bound to a session.
+    try:
+        return getattr(user, "username", None)
+    except DetachedInstanceError:
+        # Session is gone; try to reload user by primary key.
+        try:
+            user_id = getattr(user, "id", None)
+        except Exception:
+            user_id = None
+
+        if not user_id:
+            return None
+
+        try:
+            with SessionLocal() as db:
+                fresh = db.query(User).filter(User.id == user_id).first()
+                return getattr(fresh, "username", None) if fresh is not None else None
+        except Exception:
+            return None
+    except Exception:
+        # Fall back to no user information if anything else goes wrong.
+        return None
+
+
 @app.middleware("http")
 async def access_logging_middleware(request: Request, call_next):
     # Log incoming request and Authorization header presence for debugging auth issues
-    try:
-        auth_header = request.headers.get("authorization")
-        if auth_header:
-            short = auth_header[:20] + "..." if len(auth_header) > 20 else auth_header
-            logger.info("Incoming request %s %s Authorization=%s", request.method, request.url.path, short)
-        else:
-            logger.info("Incoming request %s %s Authorization=NONE", request.method, request.url.path)
-    except Exception:
-        pass
+    # Skip logging for health check requests
+    if request.url.path != "/health":
+        try:
+            auth_header = request.headers.get("authorization")
+            if auth_header:
+                short = auth_header[:20] + "..." if len(auth_header) > 20 else auth_header
+                logger.info("Incoming request %s %s Authorization=%s", request.method, request.url.path, short)
+            else:
+                logger.info("Incoming request %s %s Authorization=NONE", request.method, request.url.path)
+        except Exception:
+            pass
     start = time.perf_counter()
     try:
         response = await call_next(request)
@@ -189,7 +227,7 @@ async def access_logging_middleware(request: Request, call_next):
             method=request.method,
             path=request.url.path,
             status="error",
-            user=getattr(user, "username", None),
+            user=_get_username_for_log(user),
             ip=get_client_ip(request),
             duration=f"{duration:.3f}s",
             error=str(exc),
@@ -203,7 +241,7 @@ async def access_logging_middleware(request: Request, call_next):
             method=request.method,
             path=request.url.path,
             status=response.status_code,
-            user=getattr(user, "username", None),
+            user=_get_username_for_log(user),
             ip=get_client_ip(request),
             duration=f"{duration:.3f}s",
         )
@@ -252,6 +290,12 @@ app.include_router(download.router)
 app.include_router(keys.router)
 
 
+@app.get("/health")
+async def health_check():
+    """Health check endpoint for Docker health checks."""
+    return {"status": "healthy", "service": "main"}
+
+
 @app.get("/key/public")
 async def key_public_proxy():
     """
@@ -261,9 +305,7 @@ async def key_public_proxy():
     return await keys.get_public_key()
 
 
-@app.post("/key/invalidate")
-async def key_invalidate_proxy():
-    """
-    Proxy endpoint to invalidate messaging ephemeral key.
-    """
-    return await keys.invalidate_key()
+if __name__ == "__main__":
+    import uvicorn
+    port = int(os.getenv("PORT", "8300"))
+    uvicorn.run(app, host="0.0.0.0", port=port)

backend/services/main/migration.py

@@ -69,6 +69,15 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+
+def _ensure_all_model_tables():
+    """Create any model tables that do not exist (e.g. dm_edit_history added after migrations)."""
+    engine = _create_engine_with_retry()
+    Base = _load_models_base()
+    Base.metadata.create_all(bind=engine)
+    logger.info("Ensured all model tables exist.")
+
+
 def run_migrations():
     """
     Run database migrations using Alembic.
@@ -128,7 +137,14 @@ def run_migrations():
             engine = _create_engine_with_retry()
             with engine.connect() as connection:
                 from sqlalchemy import text
-                result = connection.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name != 'alembic_version'"))
+                # Check for PostgreSQL or SQLite
+                if 'postgresql' in DATABASE_URL.lower():
+                    result = connection.execute(text("""
+                        SELECT tablename as name FROM pg_tables
+                        WHERE schemaname = 'public' AND tablename != 'alembic_version'
+                    """))
+                else:
+                    result = connection.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name != 'alembic_version'"))
                 existing_tables = result.fetchall()
                 
                 if existing_tables:
@@ -204,6 +220,7 @@ def run_migrations():
         try:
             command.upgrade(alembic_cfg, "head")
             logger.info("Database migrations completed successfully.")
+            _ensure_all_model_tables()
         except Exception as upgrade_error:
             if "Can't locate revision identified by 'direct_creation'" in str(upgrade_error):
                 logger.info("Found 'direct_creation' revision - resetting migration state...")
@@ -237,6 +254,7 @@ def run_migrations():
                 # Try upgrade again
                 command.upgrade(alembic_cfg, "head")
                 logger.info("Database migrations completed successfully after reset.")
+                _ensure_all_model_tables()
             elif "no such table" in str(upgrade_error).lower():
                 logger.info("Database tables missing - resetting migration state...")
                 # Clear the alembic_version table and start fresh
@@ -249,6 +267,7 @@ def run_migrations():
                 # Try upgrade again
                 command.upgrade(alembic_cfg, "head")
                 logger.info("Database migrations completed successfully after reset.")
+                _ensure_all_model_tables()
             else:
                 raise upgrade_error
         
@@ -286,6 +305,7 @@ def run_migrations():
                 
                 # Try upgrade again
                 command.upgrade(alembic_cfg, "head")
+                _ensure_all_model_tables()
                 logger.info("Automated recovery completed successfully.")
             else:
                 # No migration files, create fresh ones
@@ -294,6 +314,7 @@ def run_migrations():
                 
                 # Run the migration
                 command.upgrade(alembic_cfg, "head")
+                _ensure_all_model_tables()
                 logger.info("Automated recovery completed successfully.")
             
         except Exception as recovery_error:
@@ -600,6 +621,8 @@ def _create_database_directly():
                 else:
                     # Table doesn't exist, create it
                     logger.info(f"Creating table {table_name}")
+                    from sqlalchemy.schema import CreateTable
+                    connection.execute(CreateTable(Base.metadata.tables[table_name]))
         
         # Create alembic_version table manually
         connection.execute(text("""
@@ -623,29 +646,45 @@ def _create_database_directly():
                 revision_match = re.search(r"revision: str = '([^']+)'", content)
                 if revision_match:
                     revision_id = revision_match.group(1)
-                    connection.execute(text(f"INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('{revision_id}')"))
+                    if 'postgresql' in DATABASE_URL.lower():
+                        connection.execute(text(f"INSERT INTO alembic_version (version_num) VALUES ('{revision_id}') ON CONFLICT DO NOTHING"))
+                    else:
+                        connection.execute(text(f"INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('{revision_id}')"))
                 else:
-                    connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')"))
+                    if 'postgresql' in DATABASE_URL.lower():
+                        connection.execute(text("INSERT INTO alembic_version (version_num) VALUES ('direct_creation') ON CONFLICT DO NOTHING"))
+                    else:
+                        connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')"))
         else:
-            connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')"))
+            if 'postgresql' in DATABASE_URL.lower():
+                connection.execute(text("INSERT INTO alembic_version (version_num) VALUES ('direct_creation') ON CONFLICT DO NOTHING"))
+            else:
+                connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')"))
         
         connection.commit()
 
 
 def _get_sql_type(column):
     """Get SQL type for direct SQL execution."""
-    type_name = column.type.__class__.__name__
-    
-    if type_name == 'String':
-        return f"VARCHAR({column.type.length})"
-    elif type_name == 'Integer':
+    from sqlalchemy import String, Integer, Text, Boolean, DateTime
+    import os
+
+    # Check if we're using PostgreSQL
+    is_postgres = 'postgresql' in os.getenv('DATABASE_URL', '').lower()
+
+    if isinstance(column.type, String):
+        if column.type.length:
+            return f"VARCHAR({column.type.length})"
+        else:
+            return "TEXT"
+    elif isinstance(column.type, Integer):
         return "INTEGER"
-    elif type_name == 'Text':
+    elif isinstance(column.type, Text):
         return "TEXT"
-    elif type_name == 'Boolean':
+    elif isinstance(column.type, Boolean):
         return "BOOLEAN"
-    elif type_name == 'DateTime':
-        return "DATETIME"
+    elif isinstance(column.type, DateTime):
+        return "TIMESTAMP" if is_postgres else "DATETIME"
     else:
         return "TEXT"  # fallback
 

backend/services/main/models.py

@@ -86,6 +86,7 @@ class DMEnvelope(Base):
     timestamp = Column(DateTime, default=datetime.now)
     is_edited = Column(Boolean, default=False)
     created_at = Column(DateTime, default=datetime.now)
+    deleted_at = Column(DateTime, nullable=True)  # Soft delete timestamp
     files = relationship("DMFile", back_populates="message", cascade="all, delete-orphan", lazy="select")
     reactions = relationship("DMReaction", cascade="all, delete-orphan", lazy="select")
 
@@ -241,8 +242,6 @@ class DMEditHistoryResponse(BaseModel):
     dm_envelope_id: int
     previous_ciphertext_b64: str
     previous_iv_b64: str
-    previous_sender_wrapped_mek_b64: str
-    previous_recipient_wrapped_mek_b64: str
     previous_compliance_wrapped_mek_b64: str
     edited_at: str
     edited_by_username: str
@@ -374,8 +373,6 @@ class DMEditHistory(Base):
     dm_envelope_id = Column(Integer, ForeignKey("dm_envelope.id"), nullable=False)  # Match existing DB schema
     previous_ciphertext_b64 = Column(Text, nullable=False)  # Encrypted content before this edit
     previous_iv_b64 = Column(Text, nullable=False)  # IV for previous content
-    previous_sender_wrapped_mek_b64 = Column(Text, nullable=False)  # MEK wrapped for sender before edit
-    previous_recipient_wrapped_mek_b64 = Column(Text, nullable=False)  # MEK wrapped for recipient before edit
     previous_compliance_wrapped_mek_b64 = Column(Text, nullable=False)  # MEK wrapped for compliance before edit
     edited_at = Column(DateTime, default=datetime.now, nullable=False, index=True)
     edited_by = Column(Integer, ForeignKey("user.id"), nullable=False)  # Match existing DB schema

backend/services/main/service_calls.py

@@ -171,6 +171,8 @@ async def store_encrypted_file(
     encrypted_file_data_b64: str,
     filename: str,
     content_type: str = "application/octet-stream",
+    sender_id: int = None,
+    recipient_id: int = None,
     timeout: float = 30.0,
 ) -> Dict[str, Any]:
     """
@@ -205,10 +207,17 @@ async def store_encrypted_file(
     try:
         try:
             import httpx
+            allowed_user_ids = []
+            if sender_id is not None:
+                allowed_user_ids.append(sender_id)
+            if recipient_id is not None:
+                allowed_user_ids.append(recipient_id)
+
             payload = {
                 "filename": filename,
                 "data_b64": encrypted_file_data_b64,
                 "content_type": content_type,
+                "allowed_user_ids": allowed_user_ids,
             }
             async with httpx.AsyncClient(timeout=timeout) as client:
                 r = await client.post(url, json=payload)
@@ -216,10 +225,17 @@ async def store_encrypted_file(
                 return r.json()
         except Exception:
             from urllib import request
+            allowed_user_ids = []
+            if sender_id is not None:
+                allowed_user_ids.append(sender_id)
+            if recipient_id is not None:
+                allowed_user_ids.append(recipient_id)
+
             payload = {
                 "filename": filename,
                 "data_b64": encrypted_file_data_b64,
                 "content_type": content_type,
+                "allowed_user_ids": allowed_user_ids,
             }
             req = request.Request(url, method="POST")
             req.data = json.dumps(payload).encode("utf-8")

backend/services/messaging/key_lifecycle.py

@@ -6,7 +6,7 @@ on message deletion, and configurable retention policies for cryptographic keys.
 
 Key Features:
 - Automatic compliance key destruction (default: 6 months)
-- Selective key destruction on message deletion
+- Selective key destruction on message deletion (default: 6 months)
 - Configurable retention policies
 - Background cleanup jobs for expired keys
 """
@@ -24,7 +24,7 @@ logger = logging.getLogger("uvicorn.error")
 
 # Default retention periods (in days)
 DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS = 180  # 6 months
-DEFAULT_MESSAGE_KEY_RETENTION_DAYS = 30      # 30 days for deleted messages
+DEFAULT_MESSAGE_KEY_RETENTION_DAYS = 180     # 6 months for deleted messages
 
 # Environment variable overrides
 COMPLIANCE_KEY_RETENTION_DAYS = int(os.getenv("COMPLIANCE_KEY_RETENTION_DAYS", DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS))
@@ -147,7 +147,8 @@ def cleanup_expired_message_keys(db: Session) -> int:
     Clean up message keys for deleted messages after retention period.
 
     This removes sender and recipient wrapped keys from DM envelopes that have been
-    deleted and are past the retention period, making them completely inaccessible.
+    soft-deleted and are past the retention period, making them completely inaccessible
+    except through compliance access (which preserves the compliance key).
 
     Args:
         db: Database session
@@ -156,16 +157,45 @@ def cleanup_expired_message_keys(db: Session) -> int:
         Number of keys destroyed
     """
     try:
-        # Note: We don't have a direct "deleted" flag on DMEnvelope, so this would need
-        # to be implemented when message deletion is added. For now, this is a placeholder.
+        from datetime import datetime, timedelta
+        from ..main.models import DMEnvelope
 
-        # This would typically work with a deletion timestamp or flag on the envelope
-        # For now, return 0 as we don't have deleted message tracking yet
-        logger.info("Message key cleanup: No deleted messages to process")
-        return 0
+        # Calculate cutoff date for expired messages
+        cutoff_date = datetime.now() - get_message_key_retention_period()
+
+        # Find soft-deleted messages past retention period
+        expired_messages = db.query(DMEnvelope).filter(
+            DMEnvelope.deleted_at.is_not(None),
+            DMEnvelope.deleted_at < cutoff_date
+        ).all()
+
+        if not expired_messages:
+            logger.info("Message key cleanup: No expired deleted messages to process")
+            return 0
+
+        keys_destroyed = 0
+
+        for message in expired_messages:
+            # Destroy sender and recipient keys (compliance key remains for legal access)
+            message.sender_wrapped_mek_b64 = ""
+            message.recipient_wrapped_mek_b64 = ""
+            keys_destroyed += 2
+
+            logger.info(
+                "Destroyed keys for soft-deleted message id=%s (deleted %s)",
+                message.id,
+                message.deleted_at.isoformat()
+            )
+
+        db.commit()
+        logger.info("Message key cleanup: Destroyed %d keys across %d messages",
+                   keys_destroyed, len(expired_messages))
+
+        return keys_destroyed
 
     except Exception as e:
         logger.error(f"Failed to cleanup expired message keys: {e}")
+        db.rollback()
         return 0
 
 

backend/services/messaging/main.py

@@ -386,7 +386,6 @@ async def process_message_with_files_http(request: ProcessMessageWithFilesReques
         logger.exception("Failed to process message with files: %s", e)
         raise
 
-
 if __name__ == "__main__":
     import uvicorn
     port = int(os.getenv("PORT", "8301"))

backend/services/shared/middleware.py

@@ -5,6 +5,7 @@ Provides:
 - Request size limiting (max 5GB)
 - Input validation and sanitization
 - Comprehensive audit logging
+- Health check access log filtering
 """
 
 import logging
@@ -15,6 +16,16 @@ from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.responses import Response
 
 logger = logging.getLogger("uvicorn.error")
+access_logger = logging.getLogger("uvicorn.access")
+
+
+class HealthCheckFilter(logging.Filter):
+    """Filter to suppress access logs for health check requests."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        """Return False to suppress logs containing health check requests."""
+        message = record.getMessage()
+        return "GET /health HTTP/" not in message
 
 # Maximum request size: 5GB
 MAX_REQUEST_SIZE = 5 * 1024 * 1024 * 1024  # 5GB in bytes
@@ -48,61 +59,17 @@ class RequestSizeLimitMiddleware(BaseHTTPMiddleware):
         return await call_next(request)
 
 
-class AuditLoggingMiddleware(BaseHTTPMiddleware):
-    """Middleware for comprehensive audit logging of all requests."""
-
-    async def dispatch(self, request: Request, call_next: Callable) -> Response:
-        """Log request and response details."""
-        start_time = time.time()
-        
-        # Log request
-        client_ip = request.client.host if request.client else "unknown"
-        method = request.method
-        path = request.url.path
-        
-        logger.info(
-            "REQUEST: %s %s from %s",
-            method,
-            path,
-            client_ip,
-        )
-
-        try:
-            response = await call_next(request)
-            
-            # Log response
-            duration = time.time() - start_time
-            logger.info(
-                "RESPONSE: %s %s -> %d in %.2fms",
-                method,
-                path,
-                response.status_code,
-                duration * 1000,
-            )
-            
-            return response
-            
-        except Exception as e:
-            duration = time.time() - start_time
-            logger.exception(
-                "ERROR: %s %s failed after %.2fms: %s",
-                method,
-                path,
-                duration * 1000,
-                e,
-            )
-            raise
+# Apply health check filter to access logger
+if not any(isinstance(f, HealthCheckFilter) for f in access_logger.filters):
+    access_logger.addFilter(HealthCheckFilter())
 
 
 def add_security_middleware(app: FastAPI):
     """
     Add all security and audit middleware to FastAPI app.
-    
+
     Args:
         app: FastAPI application instance
     """
     # Request size limiting (inner, checked first)
     app.add_middleware(RequestSizeLimitMiddleware)
-    
-    # Audit logging (outer, logs everything)
-    app.add_middleware(AuditLoggingMiddleware)