diff --git a/.cursor/commands/security-audit.md b/.cursor/commands/security-audit.md index f33471c..9eb0008 100644 --- a/.cursor/commands/security-audit.md +++ b/.cursor/commands/security-audit.md @@ -8,7 +8,6 @@ Perform a comprehensive security audit of the FromChat application codebase. - React/TypeScript frontend - Python FastAPI backend -- End-to-end encryption for DMs and calls - Caddy reverse proxy with security headers - WebSocket support for real-time features - Electron support for desktop app @@ -122,7 +121,7 @@ Provide a **clean, concise report** with: - CORS configuration in backend/app.py - Password validation in backend/validation.py - JWT token generation and validation -- E2E encryption implementation (NaCl, AES-GCM) +- Encryption implementation (NaCl, AES-GCM) - File upload sanitization - Authorization checks on sensitive endpoints - Rate limiting configuration diff --git a/.cursor/rules/general.mdc b/.cursor/rules/general.mdc index 9cadaa7..577c96e 100644 --- a/.cursor/rules/general.mdc +++ b/.cursor/rules/general.mdc @@ -21,6 +21,7 @@ When working with this project, follow these rules: ## File Operations - If possible, try to update files in a single edit when making multiple changes. - Do NOT "cd" to the project directory. +- NEVER edit/delete/regenerate .env files without explicit permission. ## Testing & Validation - Do NOT "test the implementation" when you are done. The only exception is when you diff --git a/.gitignore b/.gitignore index 98b7a25..171cd8f 100644 --- a/.gitignore +++ b/.gitignore @@ -581,3 +581,5 @@ backend/alembic/** tmp compliance_keypair.txt backend/files +*.db-wal +*.db-shm diff --git a/backend/requirements.txt b/backend/requirements.txt index 0a78322..45666b4 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -2,6 +2,7 @@ PyJWT>=2.8.0 fastapi[standard]>=0.116.1 pydantic>=2.11.7 sqlalchemy>=2.0.43 +psycopg2-binary>=2.9.9 bcrypt>=4.3.0 websockets>=15.0.1 Pillow>=10.0.0 @@ -14,4 +15,5 @@ user-agents>=2.2.0 httpx>=0.27.2 rich>=13.9.4 slowapi>=0.1.9 -firebase_admin>=7.1.0 \ No newline at end of file +firebase_admin>=7.1.0 +PyNaCl>=1.5.0 \ No newline at end of file diff --git a/backend/services/main/constants.py b/backend/services/main/constants.py index 57c2aac..6f8f8c1 100644 --- a/backend/services/main/constants.py +++ b/backend/services/main/constants.py @@ -1,7 +1,7 @@ import os -# Database is always in backend/data/ relative to project root -DATABASE_URL = "sqlite:///" + os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "data", "database.db") +# Database URL from environment (Docker) or fallback to SQLite (development) +DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///" + os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "data", "database.db")) JWT_ALGORITHM = "HS256" # Token inactivity expiration - token expires if not used for this duration TOKEN_INACTIVITY_EXPIRE_HOURS = 30 * 24 # 30 days of inactivity diff --git a/backend/services/main/main.py b/backend/services/main/main.py index 5ec3be3..64d13b5 100644 --- a/backend/services/main/main.py +++ b/backend/services/main/main.py @@ -7,6 +7,7 @@ import subprocess import sys import os import logging +from sqlalchemy.orm.exc import DetachedInstanceError # Import from same directory from .routes import account, messaging, profile, push, webrtc, devices, moderation, download, keys, envelope_messaging @@ -166,18 +167,55 @@ if not _running_in_docker(): logger.warning(f"Failed to mount internal services for development: {e}") +def _get_username_for_log(user) -> str | None: + """ + Safely extract username for access logs. + + If the ORM instance is detached, we transparently open a short-lived session, + reload the user by ID and read the username from that fresh instance. + Logging must never break request handling. + """ + if user is None: + return None + + # Fast path: instance is still bound to a session. + try: + return getattr(user, "username", None) + except DetachedInstanceError: + # Session is gone; try to reload user by primary key. + try: + user_id = getattr(user, "id", None) + except Exception: + user_id = None + + if not user_id: + return None + + try: + with SessionLocal() as db: + fresh = db.query(User).filter(User.id == user_id).first() + return getattr(fresh, "username", None) if fresh is not None else None + except Exception: + return None + except Exception: + # Fall back to no user information if anything else goes wrong. + return None + + @app.middleware("http") async def access_logging_middleware(request: Request, call_next): # Log incoming request and Authorization header presence for debugging auth issues - try: - auth_header = request.headers.get("authorization") - if auth_header: - short = auth_header[:20] + "..." if len(auth_header) > 20 else auth_header - logger.info("Incoming request %s %s Authorization=%s", request.method, request.url.path, short) - else: - logger.info("Incoming request %s %s Authorization=NONE", request.method, request.url.path) - except Exception: - pass + # Skip logging for health check requests + if request.url.path != "/health": + try: + auth_header = request.headers.get("authorization") + if auth_header: + short = auth_header[:20] + "..." if len(auth_header) > 20 else auth_header + logger.info("Incoming request %s %s Authorization=%s", request.method, request.url.path, short) + else: + logger.info("Incoming request %s %s Authorization=NONE", request.method, request.url.path) + except Exception: + pass start = time.perf_counter() try: response = await call_next(request) @@ -189,7 +227,7 @@ async def access_logging_middleware(request: Request, call_next): method=request.method, path=request.url.path, status="error", - user=getattr(user, "username", None), + user=_get_username_for_log(user), ip=get_client_ip(request), duration=f"{duration:.3f}s", error=str(exc), @@ -203,7 +241,7 @@ async def access_logging_middleware(request: Request, call_next): method=request.method, path=request.url.path, status=response.status_code, - user=getattr(user, "username", None), + user=_get_username_for_log(user), ip=get_client_ip(request), duration=f"{duration:.3f}s", ) @@ -252,6 +290,12 @@ app.include_router(download.router) app.include_router(keys.router) +@app.get("/health") +async def health_check(): + """Health check endpoint for Docker health checks.""" + return {"status": "healthy", "service": "main"} + + @app.get("/key/public") async def key_public_proxy(): """ @@ -261,9 +305,7 @@ async def key_public_proxy(): return await keys.get_public_key() -@app.post("/key/invalidate") -async def key_invalidate_proxy(): - """ - Proxy endpoint to invalidate messaging ephemeral key. - """ - return await keys.invalidate_key() +if __name__ == "__main__": + import uvicorn + port = int(os.getenv("PORT", "8300")) + uvicorn.run(app, host="0.0.0.0", port=port) diff --git a/backend/services/main/migration.py b/backend/services/main/migration.py index 260a03b..030a56e 100644 --- a/backend/services/main/migration.py +++ b/backend/services/main/migration.py @@ -69,6 +69,15 @@ import logging logger = logging.getLogger(__name__) + +def _ensure_all_model_tables(): + """Create any model tables that do not exist (e.g. dm_edit_history added after migrations).""" + engine = _create_engine_with_retry() + Base = _load_models_base() + Base.metadata.create_all(bind=engine) + logger.info("Ensured all model tables exist.") + + def run_migrations(): """ Run database migrations using Alembic. @@ -128,7 +137,14 @@ def run_migrations(): engine = _create_engine_with_retry() with engine.connect() as connection: from sqlalchemy import text - result = connection.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name != 'alembic_version'")) + # Check for PostgreSQL or SQLite + if 'postgresql' in DATABASE_URL.lower(): + result = connection.execute(text(""" + SELECT tablename as name FROM pg_tables + WHERE schemaname = 'public' AND tablename != 'alembic_version' + """)) + else: + result = connection.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name != 'alembic_version'")) existing_tables = result.fetchall() if existing_tables: @@ -204,6 +220,7 @@ def run_migrations(): try: command.upgrade(alembic_cfg, "head") logger.info("Database migrations completed successfully.") + _ensure_all_model_tables() except Exception as upgrade_error: if "Can't locate revision identified by 'direct_creation'" in str(upgrade_error): logger.info("Found 'direct_creation' revision - resetting migration state...") @@ -237,6 +254,7 @@ def run_migrations(): # Try upgrade again command.upgrade(alembic_cfg, "head") logger.info("Database migrations completed successfully after reset.") + _ensure_all_model_tables() elif "no such table" in str(upgrade_error).lower(): logger.info("Database tables missing - resetting migration state...") # Clear the alembic_version table and start fresh @@ -249,6 +267,7 @@ def run_migrations(): # Try upgrade again command.upgrade(alembic_cfg, "head") logger.info("Database migrations completed successfully after reset.") + _ensure_all_model_tables() else: raise upgrade_error @@ -286,6 +305,7 @@ def run_migrations(): # Try upgrade again command.upgrade(alembic_cfg, "head") + _ensure_all_model_tables() logger.info("Automated recovery completed successfully.") else: # No migration files, create fresh ones @@ -294,6 +314,7 @@ def run_migrations(): # Run the migration command.upgrade(alembic_cfg, "head") + _ensure_all_model_tables() logger.info("Automated recovery completed successfully.") except Exception as recovery_error: @@ -600,6 +621,8 @@ def _create_database_directly(): else: # Table doesn't exist, create it logger.info(f"Creating table {table_name}") + from sqlalchemy.schema import CreateTable + connection.execute(CreateTable(Base.metadata.tables[table_name])) # Create alembic_version table manually connection.execute(text(""" @@ -623,29 +646,45 @@ def _create_database_directly(): revision_match = re.search(r"revision: str = '([^']+)'", content) if revision_match: revision_id = revision_match.group(1) - connection.execute(text(f"INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('{revision_id}')")) + if 'postgresql' in DATABASE_URL.lower(): + connection.execute(text(f"INSERT INTO alembic_version (version_num) VALUES ('{revision_id}') ON CONFLICT DO NOTHING")) + else: + connection.execute(text(f"INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('{revision_id}')")) else: - connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')")) + if 'postgresql' in DATABASE_URL.lower(): + connection.execute(text("INSERT INTO alembic_version (version_num) VALUES ('direct_creation') ON CONFLICT DO NOTHING")) + else: + connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')")) else: - connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')")) + if 'postgresql' in DATABASE_URL.lower(): + connection.execute(text("INSERT INTO alembic_version (version_num) VALUES ('direct_creation') ON CONFLICT DO NOTHING")) + else: + connection.execute(text("INSERT OR IGNORE INTO alembic_version (version_num) VALUES ('direct_creation')")) connection.commit() def _get_sql_type(column): """Get SQL type for direct SQL execution.""" - type_name = column.type.__class__.__name__ - - if type_name == 'String': - return f"VARCHAR({column.type.length})" - elif type_name == 'Integer': + from sqlalchemy import String, Integer, Text, Boolean, DateTime + import os + + # Check if we're using PostgreSQL + is_postgres = 'postgresql' in os.getenv('DATABASE_URL', '').lower() + + if isinstance(column.type, String): + if column.type.length: + return f"VARCHAR({column.type.length})" + else: + return "TEXT" + elif isinstance(column.type, Integer): return "INTEGER" - elif type_name == 'Text': + elif isinstance(column.type, Text): return "TEXT" - elif type_name == 'Boolean': + elif isinstance(column.type, Boolean): return "BOOLEAN" - elif type_name == 'DateTime': - return "DATETIME" + elif isinstance(column.type, DateTime): + return "TIMESTAMP" if is_postgres else "DATETIME" else: return "TEXT" # fallback diff --git a/backend/services/main/models.py b/backend/services/main/models.py index 364f54f..d4ecbfe 100644 --- a/backend/services/main/models.py +++ b/backend/services/main/models.py @@ -86,6 +86,7 @@ class DMEnvelope(Base): timestamp = Column(DateTime, default=datetime.now) is_edited = Column(Boolean, default=False) created_at = Column(DateTime, default=datetime.now) + deleted_at = Column(DateTime, nullable=True) # Soft delete timestamp files = relationship("DMFile", back_populates="message", cascade="all, delete-orphan", lazy="select") reactions = relationship("DMReaction", cascade="all, delete-orphan", lazy="select") @@ -241,8 +242,6 @@ class DMEditHistoryResponse(BaseModel): dm_envelope_id: int previous_ciphertext_b64: str previous_iv_b64: str - previous_sender_wrapped_mek_b64: str - previous_recipient_wrapped_mek_b64: str previous_compliance_wrapped_mek_b64: str edited_at: str edited_by_username: str @@ -374,8 +373,6 @@ class DMEditHistory(Base): dm_envelope_id = Column(Integer, ForeignKey("dm_envelope.id"), nullable=False) # Match existing DB schema previous_ciphertext_b64 = Column(Text, nullable=False) # Encrypted content before this edit previous_iv_b64 = Column(Text, nullable=False) # IV for previous content - previous_sender_wrapped_mek_b64 = Column(Text, nullable=False) # MEK wrapped for sender before edit - previous_recipient_wrapped_mek_b64 = Column(Text, nullable=False) # MEK wrapped for recipient before edit previous_compliance_wrapped_mek_b64 = Column(Text, nullable=False) # MEK wrapped for compliance before edit edited_at = Column(DateTime, default=datetime.now, nullable=False, index=True) edited_by = Column(Integer, ForeignKey("user.id"), nullable=False) # Match existing DB schema diff --git a/backend/services/main/service_calls.py b/backend/services/main/service_calls.py index 07e619e..da26a22 100644 --- a/backend/services/main/service_calls.py +++ b/backend/services/main/service_calls.py @@ -171,6 +171,8 @@ async def store_encrypted_file( encrypted_file_data_b64: str, filename: str, content_type: str = "application/octet-stream", + sender_id: int = None, + recipient_id: int = None, timeout: float = 30.0, ) -> Dict[str, Any]: """ @@ -205,10 +207,17 @@ async def store_encrypted_file( try: try: import httpx + allowed_user_ids = [] + if sender_id is not None: + allowed_user_ids.append(sender_id) + if recipient_id is not None: + allowed_user_ids.append(recipient_id) + payload = { "filename": filename, "data_b64": encrypted_file_data_b64, "content_type": content_type, + "allowed_user_ids": allowed_user_ids, } async with httpx.AsyncClient(timeout=timeout) as client: r = await client.post(url, json=payload) @@ -216,10 +225,17 @@ async def store_encrypted_file( return r.json() except Exception: from urllib import request + allowed_user_ids = [] + if sender_id is not None: + allowed_user_ids.append(sender_id) + if recipient_id is not None: + allowed_user_ids.append(recipient_id) + payload = { "filename": filename, "data_b64": encrypted_file_data_b64, "content_type": content_type, + "allowed_user_ids": allowed_user_ids, } req = request.Request(url, method="POST") req.data = json.dumps(payload).encode("utf-8") diff --git a/backend/services/messaging/key_lifecycle.py b/backend/services/messaging/key_lifecycle.py index a61b6b3..04e75d4 100644 --- a/backend/services/messaging/key_lifecycle.py +++ b/backend/services/messaging/key_lifecycle.py @@ -6,7 +6,7 @@ on message deletion, and configurable retention policies for cryptographic keys. Key Features: - Automatic compliance key destruction (default: 6 months) -- Selective key destruction on message deletion +- Selective key destruction on message deletion (default: 6 months) - Configurable retention policies - Background cleanup jobs for expired keys """ @@ -24,7 +24,7 @@ logger = logging.getLogger("uvicorn.error") # Default retention periods (in days) DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS = 180 # 6 months -DEFAULT_MESSAGE_KEY_RETENTION_DAYS = 30 # 30 days for deleted messages +DEFAULT_MESSAGE_KEY_RETENTION_DAYS = 180 # 6 months for deleted messages # Environment variable overrides COMPLIANCE_KEY_RETENTION_DAYS = int(os.getenv("COMPLIANCE_KEY_RETENTION_DAYS", DEFAULT_COMPLIANCE_KEY_RETENTION_DAYS)) @@ -147,7 +147,8 @@ def cleanup_expired_message_keys(db: Session) -> int: Clean up message keys for deleted messages after retention period. This removes sender and recipient wrapped keys from DM envelopes that have been - deleted and are past the retention period, making them completely inaccessible. + soft-deleted and are past the retention period, making them completely inaccessible + except through compliance access (which preserves the compliance key). Args: db: Database session @@ -156,16 +157,45 @@ def cleanup_expired_message_keys(db: Session) -> int: Number of keys destroyed """ try: - # Note: We don't have a direct "deleted" flag on DMEnvelope, so this would need - # to be implemented when message deletion is added. For now, this is a placeholder. + from datetime import datetime, timedelta + from ..main.models import DMEnvelope - # This would typically work with a deletion timestamp or flag on the envelope - # For now, return 0 as we don't have deleted message tracking yet - logger.info("Message key cleanup: No deleted messages to process") - return 0 + # Calculate cutoff date for expired messages + cutoff_date = datetime.now() - get_message_key_retention_period() + + # Find soft-deleted messages past retention period + expired_messages = db.query(DMEnvelope).filter( + DMEnvelope.deleted_at.is_not(None), + DMEnvelope.deleted_at < cutoff_date + ).all() + + if not expired_messages: + logger.info("Message key cleanup: No expired deleted messages to process") + return 0 + + keys_destroyed = 0 + + for message in expired_messages: + # Destroy sender and recipient keys (compliance key remains for legal access) + message.sender_wrapped_mek_b64 = "" + message.recipient_wrapped_mek_b64 = "" + keys_destroyed += 2 + + logger.info( + "Destroyed keys for soft-deleted message id=%s (deleted %s)", + message.id, + message.deleted_at.isoformat() + ) + + db.commit() + logger.info("Message key cleanup: Destroyed %d keys across %d messages", + keys_destroyed, len(expired_messages)) + + return keys_destroyed except Exception as e: logger.error(f"Failed to cleanup expired message keys: {e}") + db.rollback() return 0 diff --git a/backend/services/messaging/main.py b/backend/services/messaging/main.py index 364ff5b..5bb43da 100644 --- a/backend/services/messaging/main.py +++ b/backend/services/messaging/main.py @@ -386,7 +386,6 @@ async def process_message_with_files_http(request: ProcessMessageWithFilesReques logger.exception("Failed to process message with files: %s", e) raise - if __name__ == "__main__": import uvicorn port = int(os.getenv("PORT", "8301")) diff --git a/backend/services/shared/middleware.py b/backend/services/shared/middleware.py index 61660de..b913dad 100644 --- a/backend/services/shared/middleware.py +++ b/backend/services/shared/middleware.py @@ -5,6 +5,7 @@ Provides: - Request size limiting (max 5GB) - Input validation and sanitization - Comprehensive audit logging +- Health check access log filtering """ import logging @@ -15,6 +16,16 @@ from starlette.middleware.base import BaseHTTPMiddleware from starlette.responses import Response logger = logging.getLogger("uvicorn.error") +access_logger = logging.getLogger("uvicorn.access") + + +class HealthCheckFilter(logging.Filter): + """Filter to suppress access logs for health check requests.""" + + def filter(self, record: logging.LogRecord) -> bool: + """Return False to suppress logs containing health check requests.""" + message = record.getMessage() + return "GET /health HTTP/" not in message # Maximum request size: 5GB MAX_REQUEST_SIZE = 5 * 1024 * 1024 * 1024 # 5GB in bytes @@ -48,61 +59,17 @@ class RequestSizeLimitMiddleware(BaseHTTPMiddleware): return await call_next(request) -class AuditLoggingMiddleware(BaseHTTPMiddleware): - """Middleware for comprehensive audit logging of all requests.""" - - async def dispatch(self, request: Request, call_next: Callable) -> Response: - """Log request and response details.""" - start_time = time.time() - - # Log request - client_ip = request.client.host if request.client else "unknown" - method = request.method - path = request.url.path - - logger.info( - "REQUEST: %s %s from %s", - method, - path, - client_ip, - ) - - try: - response = await call_next(request) - - # Log response - duration = time.time() - start_time - logger.info( - "RESPONSE: %s %s -> %d in %.2fms", - method, - path, - response.status_code, - duration * 1000, - ) - - return response - - except Exception as e: - duration = time.time() - start_time - logger.exception( - "ERROR: %s %s failed after %.2fms: %s", - method, - path, - duration * 1000, - e, - ) - raise +# Apply health check filter to access logger +if not any(isinstance(f, HealthCheckFilter) for f in access_logger.filters): + access_logger.addFilter(HealthCheckFilter()) def add_security_middleware(app: FastAPI): """ Add all security and audit middleware to FastAPI app. - + Args: app: FastAPI application instance """ # Request size limiting (inner, checked first) app.add_middleware(RequestSizeLimitMiddleware) - - # Audit logging (outer, logs everything) - app.add_middleware(AuditLoggingMiddleware) diff --git a/deployment/Dockerfile b/deployment/Dockerfile new file mode 100644 index 0000000..43205a1 --- /dev/null +++ b/deployment/Dockerfile @@ -0,0 +1,111 @@ +# ============================================================================ +# COMPLIANCE ARCHITECTURE - Unified Dockerfile +# ============================================================================ +# Base stage with common dependencies for all services + +FROM python:3.12-slim AS base + +# Create common directories +RUN mkdir -p /app && \ + useradd -u 1000 -m app && \ + useradd -u 1001 -m messaging && \ + useradd -u 1002 -m -s /bin/false filestorage + +# Set working directory +WORKDIR /app + +# Copy health check script +COPY --chown=app:app deployment/healthcheck.py /usr/local/bin/healthcheck.py +RUN chmod +x /usr/local/bin/healthcheck.py + +# Copy and install Python dependencies with pip cache +COPY --chown=app:app backend/requirements.txt . +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --no-cache-dir -r requirements.txt + +# ============================================================================ +# MAIN SERVICE - User-facing operations +# ============================================================================ +FROM base AS main + +# Copy main service code +COPY --chown=app:app backend/services/main/ ./services/main/ +COPY --chown=app:app backend/services/shared/ ./services/shared/ +COPY --chown=app:app backend/alembic/ ./alembic/ +COPY --chown=app:app backend/alembic.ini ./ + +# Create data directories for main service +RUN mkdir -p /app/data /app/logs /app/alembic/versions && \ + chown -R app:app /app/data /app/logs /app/alembic + +# Switch to non-root user +USER app + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD python3 /usr/local/bin/healthcheck.py + +# Expose port +EXPOSE ${PORT:-8300} + +# Run main service +CMD ["python", "-m", "services.main.main"] + +# ============================================================================ +# MESSAGING SERVICE - Secure cryptographic processing +# ============================================================================ +FROM base AS messaging + +# Copy messaging service code +COPY --chown=messaging:messaging backend/services/messaging/ ./services/messaging/ +COPY --chown=messaging:messaging backend/services/shared/ ./services/shared/ + +# Create directories with restricted permissions +RUN mkdir -p /app/logs && \ + chown -R messaging:messaging /app && \ + chmod 700 /app + +# Switch to non-root user +USER messaging + +# Health check - only accessible internally +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD python3 /usr/local/bin/healthcheck.py + +# Expose port (internal only) +EXPOSE ${PORT:-8301} + +# Run messaging service +CMD ["python", "-m", "services.messaging.main"] + +# ============================================================================ +# FILE STORAGE SERVICE - Secure file storage with execution prevention +# ============================================================================ +FROM base AS file_storage + +# Copy file storage service code +COPY --chown=filestorage:filestorage backend/services/file_storage/ ./services/file_storage/ +COPY --chown=filestorage:filestorage backend/services/shared/ ./services/shared/ +COPY --chown=filestorage:filestorage backend/services/main/db.py ./services/main/ +COPY --chown=filestorage:filestorage backend/services/main/dependencies.py ./services/main/ +COPY --chown=filestorage:filestorage backend/services/main/models.py ./services/main/ +COPY --chown=filestorage:filestorage backend/services/main/constants.py ./services/main/ +COPY --chown=filestorage:filestorage backend/services/main/utils.py ./services/main/ + +# Create secure file storage directories +RUN mkdir -p /app/files /app/logs && \ + chown -R filestorage:filestorage /app && \ + chmod 700 /app + +# Switch to non-root user +USER filestorage + +# Health check - only accessible internally +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD python3 /usr/local/bin/healthcheck.py + +# Expose port (internal only) +EXPOSE ${PORT:-8302} + +# Run file storage service with permission fix +CMD ["sh", "-c", "chown -R filestorage:filestorage /app/files /app/logs 2>/dev/null || true && exec python -m services.file_storage.main"] \ No newline at end of file diff --git a/deployment/README.md b/deployment/README.md new file mode 100644 index 0000000..3a39ef7 --- /dev/null +++ b/deployment/README.md @@ -0,0 +1,98 @@ +# FromChat Compliance Architecture - Docker Deployment + +This directory contains the Docker configuration for the 3-service compliance architecture. + +## Architecture Overview + +``` +┌─────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Clients │────│ Main Service │────│ Messaging │ +│ │ │ (Port 8300) │ │ Service │ +│ Web/Apps │ │ │ │ (Port 8301) │ +│ │ │ • User auth │ │ • Encryption │ +└─────────────┘ │ • WebSocket │ │ • Compliance │ + │ • API proxy │ │ • No ext access │ + └─────────────────┘ └─────────────────┘ + │ │ + │ │ + ▼ ▼ + ┌─────────────────┐ ┌─────────────────┐ + │ File Storage │ │ PostgreSQL │ + │ Service │ │ Database │ + │ (Port 8302) │ │ • Main schema │ + │ • Secure files │ │ • Messaging │ + │ • No ext access │ │ • File schema │ + └─────────────────┘ └─────────────────┘ +``` + +## Docker Build Optimization + +- **Unified Dockerfile**: Single Dockerfile with multi-stage builds for all services +- **Shared Base**: Common Python dependencies cached in base stage +- **Zero System Dependencies**: No gcc, curl, or system packages - pure Python +- **Python Health Checks**: Built-in health monitoring using urllib +- **Aggressive Caching**: Pip cache and layer optimization +- **Security**: Non-root users, restricted permissions per service + +## Security Features + +- **Network Isolation**: Messaging and file storage services have NO external network access +- **Database Separation**: Each service has its own schema with minimal required permissions +- **Secure File Storage**: File storage uses restricted permissions and user isolation +- **Ephemeral Keys**: Messaging service generates temporary keys (never persisted) + +## Environment Variables Required + +Create a `.env` file in this directory with the following variables: + +```bash +# Database +POSTGRES_PASSWORD=your_secure_postgres_password +MAIN_DB_PASSWORD=separate_password_for_main_service +MESSAGING_DB_PASSWORD=separate_password_for_messaging +FILE_STORAGE_DB_PASSWORD=separate_password_for_file_storage + +# Security +JWT_SECRET=your_jwt_secret_key +VAPID_PUBLIC_KEY=generated_vapid_public_key +VAPID_PRIVATE_KEY=generated_vapid_private_key +FIREBASE_CERT='{"type":"service_account",...}' + +# Compliance (public key only - private key stays offline) +COMPLIANCE_PUBLIC_KEY=base64_encoded_public_key +``` + +## Deployment Commands + +```bash +# Start all services +docker compose up -d + +# View logs +docker compose logs -f + +# Stop services +docker compose down + +# Rebuild and restart +docker compose up -d --build +``` + +## Development Mode + +For local development, set `SERVICE_MODE=development` to run all services in a single Python process instead of containers. + +## Network Architecture + +- **public**: External client access (main service, frontend, reverse proxy) +- **services**: Internal service communication only (database, messaging, file storage) +- Messaging and file storage services have NO external network access +- All inter-service communication is HTTP-based with proper authentication + +## Database Schema Separation + +- `fromchat_main`: User data, authentication, profiles +- `fromchat_messaging`: Encrypted messages, keys, compliance data +- `fromchat_files`: File metadata, storage references + +Each service has minimal required database permissions for security isolation. \ No newline at end of file diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 79ce64f..6e850ac 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -196,7 +196,6 @@ services: - db:/var/lib/postgresql/data networks: - services - - public healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 10s diff --git a/deployment/frontend/Dockerfile b/deployment/frontend/Dockerfile index 8bad32e..fb4e243 100644 --- a/deployment/frontend/Dockerfile +++ b/deployment/frontend/Dockerfile @@ -33,7 +33,10 @@ RUN npm run build # 3. Put it all together FROM node:24-slim -# 3.1. Non-root user +# 3.1. Install curl for health checks +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* + +# 3.2. Non-root user RUN useradd -u 1001 app && \ mkdir -p /app && \ chown -R app /app && \ diff --git a/deployment/frontend/server.ts b/deployment/frontend/server.ts index a24cb02..1faa187 100644 --- a/deployment/frontend/server.ts +++ b/deployment/frontend/server.ts @@ -5,6 +5,7 @@ import { resolve } from 'path'; const app = express(); const port = process.env.PORT || 3000; const backendHost = process.env.BACKEND_HOST || "http://localhost:8300"; +const fileStorageHost = process.env.FILE_STORAGE_HOST || "http://localhost:8302"; const filePath = process.env.STATIC_FILE_PATH || "."; // API proxy middleware @@ -15,6 +16,12 @@ app.use('/api', createProxyMiddleware({ ws: true })); +// File serving proxy middleware +app.use('/uploads/files', createProxyMiddleware({ + target: fileStorageHost, + changeOrigin: true +})); + // Serve static files app.use(express.static(resolve(filePath))); diff --git a/deployment/healthcheck.py b/deployment/healthcheck.py new file mode 100644 index 0000000..213af11 --- /dev/null +++ b/deployment/healthcheck.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +""" +Simple health check script using built-in urllib +Replaces curl dependency in Docker health checks +""" + +import sys +import urllib.request +import os + +def main(): + port = os.getenv('PORT', '8300') + url = f'http://localhost:{port}/health' + + try: + with urllib.request.urlopen(url, timeout=10) as response: + if response.status == 200: + print("OK") + sys.exit(0) + else: + print(f"HTTP {response.status}") + sys.exit(1) + except Exception as e: + print(f"FAILED: {e}") + sys.exit(1) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/frontend/src/core/api/chats/dm.ts b/frontend/src/core/api/chats/dm.ts index cedb5a4..dec7c9d 100644 --- a/frontend/src/core/api/chats/dm.ts +++ b/frontend/src/core/api/chats/dm.ts @@ -40,19 +40,9 @@ export async function decrypt(envelope: DmEnvelope, userId?: number): Promise = { content, is_edited: true, files }; - this.updateMessage(id, updates); - } catch (e) { + const { id, senderId, recipientId, iv_b64, ciphertext_b64, wrapped_mek_b64, timestamp } = response.data; + if (!wrapped_mek_b64) { this.updateMessage(id, { is_edited: true }); + } else { + try { + const plaintext = await api.chats.dm.decrypt( + { + id, + senderId: senderId ?? 0, + recipientId: recipientId ?? 0, + iv_b64: iv_b64 ?? "", + ciphertext_b64: ciphertext_b64 ?? "", + wrapped_mek_b64, + timestamp: timestamp ?? new Date().toISOString() + }, + this.currentUser.currentUser?.id + ); + let content = plaintext; + let files: Message["files"] | undefined = undefined; + try { + const obj = JSON.parse(plaintext) as EncryptedMessageJson; + if (obj.type === "text" && obj.data) { + content = obj.data.content; + files = obj.data.files; + } + } catch {} + const updates: Partial = { content, is_edited: true, files }; + this.updateMessage(id, updates); + } catch (e) { + this.updateMessage(id, { is_edited: true }); + } } } if (response.type === "dmDeleted" && this.dmData) { diff --git a/scripts/generate:env.sh b/scripts/generate:env.sh index 6937ee9..a3fd21a 100755 --- a/scripts/generate:env.sh +++ b/scripts/generate:env.sh @@ -2,7 +2,7 @@ echo > deployment/.env -./.venv/bin/python3 backend/generate_vapid_keys.py >> deployment/.env +./.venv/bin/python3 backend/services/main/generate_vapid_keys.py >> deployment/.env cat >> deployment/.env <