Add multi-service Docker setup, Postgres migrations, and messaging API cleanup

backend/services/main/constants.py

@@ -1,7 +1,7 @@
 import os
 
-# Database is always in backend/data/ relative to project root
-DATABASE_URL = "sqlite:///" + os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "data", "database.db")
+# Database URL from environment (Docker) or fallback to SQLite (development)
+DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///" + os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "data", "database.db"))
 JWT_ALGORITHM = "HS256"
 # Token inactivity expiration - token expires if not used for this duration
 TOKEN_INACTIVITY_EXPIRE_HOURS = 30 * 24  # 30 days of inactivity

backend/services/main/main.py

@@ -7,6 +7,7 @@ import subprocess
 import sys
 import os
 import logging
+from sqlalchemy.orm.exc import DetachedInstanceError
 
 # Import from same directory
 from .routes import account, messaging, profile, push, webrtc, devices, moderation, download, keys, envelope_messaging
@@ -166,18 +167,55 @@ if not _running_in_docker():
         logger.warning(f"Failed to mount internal services for development: {e}")
 
 
+def _get_username_for_log(user) -> str | None:
+    """
+    Safely extract username for access logs.
+
+    If the ORM instance is detached, we transparently open a short-lived session,
+    reload the user by ID and read the username from that fresh instance.
+    Logging must never break request handling.
+    """
+    if user is None:
+        return None
+
+    # Fast path: instance is still bound to a session.
+    try:
+        return getattr(user, "username", None)
+    except DetachedInstanceError:
+        # Session is gone; try to reload user by primary key.
+        try:
+            user_id = getattr(user, "id", None)
+        except Exception:
+            user_id = None
+
+        if not user_id:
+            return None
+
+        try:
+            with SessionLocal() as db:
+                fresh = db.query(User).filter(User.id == user_id).first()
+                return getattr(fresh, "username", None) if fresh is not None else None
+        except Exception:
+            return None
+    except Exception:
+        # Fall back to no user information if anything else goes wrong.
+        return None
+
+
 @app.middleware("http")
 async def access_logging_middleware(request: Request, call_next):
     # Log incoming request and Authorization header presence for debugging auth issues
-    try:
-        auth_header = request.headers.get("authorization")
-        if auth_header:
-            short = auth_header[:20] + "..." if len(auth_header) > 20 else auth_header
-            logger.info("Incoming request %s %s Authorization=%s", request.method, request.url.path, short)
-        else:
-            logger.info("Incoming request %s %s Authorization=NONE", request.method, request.url.path)
-    except Exception:
-        pass
+    # Skip logging for health check requests
+    if request.url.path != "/health":
+        try:
+            auth_header = request.headers.get("authorization")
+            if auth_header:
+                short = auth_header[:20] + "..." if len(auth_header) > 20 else auth_header
+                logger.info("Incoming request %s %s Authorization=%s", request.method, request.url.path, short)
+            else:
+                logger.info("Incoming request %s %s Authorization=NONE", request.method, request.url.path)
+        except Exception:
+            pass
     start = time.perf_counter()
     try:
         response = await call_next(request)
@@ -189,7 +227,7 @@ async def access_logging_middleware(request: Request, call_next):
             method=request.method,
             path=request.url.path,
             status="error",
-            user=getattr(user, "username", None),
+            user=_get_username_for_log(user),
             ip=get_client_ip(request),
             duration=f"{duration:.3f}s",
             error=str(exc),
@@ -203,7 +241,7 @@ async def access_logging_middleware(request: Request, call_next):
             method=request.method,
             path=request.url.path,
             status=response.status_code,
-            user=getattr(user, "username", None),
+            user=_get_username_for_log(user),
             ip=get_client_ip(request),
             duration=f"{duration:.3f}s",
         )
@@ -252,6 +290,12 @@ app.include_router(download.router)
 app.include_router(keys.router)
 
 
+@app.get("/health")
+async def health_check():
+    """Health check endpoint for Docker health checks."""
+    return {"status": "healthy", "service": "main"}
+
+
 @app.get("/key/public")
 async def key_public_proxy():
     """
@@ -261,9 +305,7 @@ async def key_public_proxy():
     return await keys.get_public_key()
 
 
-@app.post("/key/invalidate")
-async def key_invalidate_proxy():
-    """
-    Proxy endpoint to invalidate messaging ephemeral key.
-    """
-    return await keys.invalidate_key()
+if __name__ == "__main__":
+    import uvicorn
+    port = int(os.getenv("PORT", "8300"))
+    uvicorn.run(app, host="0.0.0.0", port=port)

backend/services/main/migration.py

@@ -69,6 +69,15 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+
+def _ensure_all_model_tables():
+    """Create any model tables that do not exist (e.g. dm_edit_history added after migrations)."""
+    engine = _create_engine_with_retry()
+    Base = _load_models_base()
+    Base.metadata.create_all(bind=engine)
+    logger.info("Ensured all model tables exist.")
+
+
 def run_migrations():
     """
     Run database migrations using Alembic.
@@ -128,7 +137,14 @@ def run_migrations():
             engine = _create_engine_with_retry()
             with engine.connect() as connection:
                 from sqlalchemy import text
-                result = connection.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name != 'alembic_version'"))
+                # Check for PostgreSQL or SQLite
+                if 'postgresql' in DATABASE_URL.lower():
+                    result = connection.execute(text("""
+                        SELECT tablename as name FROM pg_tables
+                        WHERE schemaname = 'public' AND tablename != 'alembic_version'
+                    """))
+                else:
+                    result = connection.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name != 'alembic_version'"))
                 existing_tables = result.fetchall()
                 
                 if existing_tables:
@@ -204,6 +220,7 @@ def run_migrations():
         try:
             command.upgrade(alembic_cfg, "head")
             logger.info("Database migrations completed successfully.")
+            _ensure_all_model_tables()
         except Exception as upgrade_error:
             if "Can't locate revision identified by 'direct_creation'" in str(upgrade_error):
                 logger.info("Found 'direct_creation' revision - resetting migration state...")
@@ -237,6 +254,7 @@ def run_migrations():
                 # Try upgrade again
                 command.upgrade(alembic_cfg, "head")
                 logger.info("Database migrations completed successfully after reset.")
+                _ensure_all_model_tables()
             elif "no such table" in str(upgrade_error).lower():
                 logger.info("Database tables missing - resetting migration state...")
                 # Clear the alembic_version table and start fresh
@@ -249,6 +267,7 @@ def run_migrations():
                 # Try upgrade again
                 command.upgrade(alembic_cfg, "head")
                 logger.info("Database migrations completed successfully after reset.")
+                _ensure_all_model_tables()
             else:
                 raise upgrade_error
         
@@ -286,6 +305,7 @@ def run_migrations():
                 
                 # Try upgrade again
                 command.upgrade(alembic_cfg, "head")
+                _ensure_all_model_tables()
                 logger.info("Automated recovery completed successfully.")
             else:
                 # No migration files, create fresh ones
@@ -294,6 +314,7 @@ def run_migrations():
                 
                 # Run the migration
                 command.upgrade(alembic_cfg, "head")
+                _ensure_all_model_tables()
                 logger.info("Automated recovery completed successfully.")
             
         except Exception as recovery_error:
@@ -600,6 +621,8 @@ def _create_database_directly():
                 else:
                     # Table doesn't exist, create it
                     logger.info(f"Creating table {table_name}")
+                    from sqlalchemy.schema import CreateTable
+                    connection.execute(CreateTable(Base.metadata.tables[table_name]))
         
         # Create alembic_version table manually
         connection.execute(text("""
@@ -623,29 +646,45 @@ def _create_database_directly():
                 revision_match = re.search(r"revision: str = '([^']+)'", content)
                 if revision_match:
                     revision_id = revision_match.group(1)
-                    connection.execute(text(f"INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('{revision_id}')"))
+                    if 'postgresql' in DATABASE_URL.lower():
+                        connection.execute(text(f"INSERT INTO alembic_version (version_num) VALUES ('{revision_id}') ON CONFLICT DO NOTHING"))
+                    else:
+                        connection.execute(text(f"INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('{revision_id}')"))
                 else:
-                    connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')"))
+                    if 'postgresql' in DATABASE_URL.lower():
+                        connection.execute(text("INSERT INTO alembic_version (version_num) VALUES ('direct_creation') ON CONFLICT DO NOTHING"))
+                    else:
+                        connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')"))
         else:
-            connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')"))
+            if 'postgresql' in DATABASE_URL.lower():
+                connection.execute(text("INSERT INTO alembic_version (version_num) VALUES ('direct_creation') ON CONFLICT DO NOTHING"))
+            else:
+                connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')"))
         
         connection.commit()
 
 
 def _get_sql_type(column):
     """Get SQL type for direct SQL execution."""
-    type_name = column.type.__class__.__name__
-    
-    if type_name == 'String':
-        return f"VARCHAR({column.type.length})"
-    elif type_name == 'Integer':
+    from sqlalchemy import String, Integer, Text, Boolean, DateTime
+    import os
+
+    # Check if we're using PostgreSQL
+    is_postgres = 'postgresql' in os.getenv('DATABASE_URL', '').lower()
+
+    if isinstance(column.type, String):
+        if column.type.length:
+            return f"VARCHAR({column.type.length})"
+        else:
+            return "TEXT"
+    elif isinstance(column.type, Integer):
         return "INTEGER"
-    elif type_name == 'Text':
+    elif isinstance(column.type, Text):
         return "TEXT"
-    elif type_name == 'Boolean':
+    elif isinstance(column.type, Boolean):
         return "BOOLEAN"
-    elif type_name == 'DateTime':
-        return "DATETIME"
+    elif isinstance(column.type, DateTime):
+        return "TIMESTAMP" if is_postgres else "DATETIME"
     else:
         return "TEXT"  # fallback
 

backend/services/main/models.py

@@ -86,6 +86,7 @@ class DMEnvelope(Base):
     timestamp = Column(DateTime, default=datetime.now)
     is_edited = Column(Boolean, default=False)
     created_at = Column(DateTime, default=datetime.now)
+    deleted_at = Column(DateTime, nullable=True)  # Soft delete timestamp
     files = relationship("DMFile", back_populates="message", cascade="all, delete-orphan", lazy="select")
     reactions = relationship("DMReaction", cascade="all, delete-orphan", lazy="select")
 
@@ -241,8 +242,6 @@ class DMEditHistoryResponse(BaseModel):
     dm_envelope_id: int
     previous_ciphertext_b64: str
     previous_iv_b64: str
-    previous_sender_wrapped_mek_b64: str
-    previous_recipient_wrapped_mek_b64: str
     previous_compliance_wrapped_mek_b64: str
     edited_at: str
     edited_by_username: str
@@ -374,8 +373,6 @@ class DMEditHistory(Base):
     dm_envelope_id = Column(Integer, ForeignKey("dm_envelope.id"), nullable=False)  # Match existing DB schema
     previous_ciphertext_b64 = Column(Text, nullable=False)  # Encrypted content before this edit
     previous_iv_b64 = Column(Text, nullable=False)  # IV for previous content
-    previous_sender_wrapped_mek_b64 = Column(Text, nullable=False)  # MEK wrapped for sender before edit
-    previous_recipient_wrapped_mek_b64 = Column(Text, nullable=False)  # MEK wrapped for recipient before edit
     previous_compliance_wrapped_mek_b64 = Column(Text, nullable=False)  # MEK wrapped for compliance before edit
     edited_at = Column(DateTime, default=datetime.now, nullable=False, index=True)
     edited_by = Column(Integer, ForeignKey("user.id"), nullable=False)  # Match existing DB schema

backend/services/main/service_calls.py

@@ -171,6 +171,8 @@ async def store_encrypted_file(
     encrypted_file_data_b64: str,
     filename: str,
     content_type: str = "application/octet-stream",
+    sender_id: int = None,
+    recipient_id: int = None,
     timeout: float = 30.0,
 ) -> Dict[str, Any]:
     """
@@ -205,10 +207,17 @@ async def store_encrypted_file(
     try:
         try:
             import httpx
+            allowed_user_ids = []
+            if sender_id is not None:
+                allowed_user_ids.append(sender_id)
+            if recipient_id is not None:
+                allowed_user_ids.append(recipient_id)
+
             payload = {
                 "filename": filename,
                 "data_b64": encrypted_file_data_b64,
                 "content_type": content_type,
+                "allowed_user_ids": allowed_user_ids,
             }
             async with httpx.AsyncClient(timeout=timeout) as client:
                 r = await client.post(url, json=payload)
@@ -216,10 +225,17 @@ async def store_encrypted_file(
                 return r.json()
         except Exception:
             from urllib import request
+            allowed_user_ids = []
+            if sender_id is not None:
+                allowed_user_ids.append(sender_id)
+            if recipient_id is not None:
+                allowed_user_ids.append(recipient_id)
+
             payload = {
                 "filename": filename,
                 "data_b64": encrypted_file_data_b64,
                 "content_type": content_type,
+                "allowed_user_ids": allowed_user_ids,
             }
             req = request.Request(url, method="POST")
             req.data = json.dumps(payload).encode("utf-8")