Add streaming encryption support

backend/services/messaging/encryption.py

@@ -11,6 +11,8 @@ Handles:
 import os
 import base64
 import logging
+from pathlib import Path
+from typing import BinaryIO
 from cryptography.hazmat.primitives.ciphers.aead import AESGCM
 from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey, X25519PublicKey
 from cryptography.hazmat.primitives.kdf.hkdf import HKDF
@@ -25,6 +27,13 @@ TRANSPORT_NONCE_SIZE = 24  # For X25519 transport encryption (PyNaCl Box/XSalsa2
 MEK_NONCE_SIZE = 12        # For AES-GCM content encryption
 MEK_SIZE = 32              # Message Encryption Key size
 
+# Client streaming transport format (chunked AES-256-GCM): FCAE | version | frames…
+FCAE_MAGIC = b"FCAE"
+FCAE_VERSION = 1
+FCAE_PREFIX_BYTES = len(FCAE_MAGIC) + 1
+FCAE_FRAME_LENGTH_BYTES = 4
+TRANSPORT_FILE_KEY_CONTEXT = "fromchat_transport_file_v1"
+
 
 def generate_mek() -> bytes:
     """Generate a random Message Encryption Key (32 bytes)."""
@@ -123,6 +132,106 @@ def decrypt_transport_message(
         raise
 
 
+def is_fcae_transport_blob(prefix: bytes) -> bool:
+    return len(prefix) >= len(FCAE_MAGIC) and prefix[: len(FCAE_MAGIC)] == FCAE_MAGIC
+
+
+def derive_transport_file_aes_key(
+    client_public_key_b64: str,
+    ephemeral_private_key: X25519PrivateKey,
+) -> bytes:
+    client_public_bytes = base64.b64decode(client_public_key_b64)
+    server_private_bytes = ephemeral_private_key.private_bytes_raw()
+    shared = sodium.crypto_box_beforenm(client_public_bytes, server_private_bytes)
+    return derive_key_from_shared_secret(shared, TRANSPORT_FILE_KEY_CONTEXT)
+
+
+def _read_fcae_frame_payload(source: BinaryIO) -> tuple[bytes, bytes] | None:
+    length_bytes = source.read(FCAE_FRAME_LENGTH_BYTES)
+    if not length_bytes:
+        return None
+    if len(length_bytes) < FCAE_FRAME_LENGTH_BYTES:
+        raise ValueError("Truncated FCAE frame length")
+    frame_len = int.from_bytes(length_bytes, byteorder="big", signed=False)
+    if frame_len <= MEK_NONCE_SIZE:
+        raise ValueError("Invalid FCAE frame length")
+    frame = source.read(frame_len)
+    if len(frame) < frame_len:
+        raise ValueError("Truncated FCAE frame")
+    iv = frame[:MEK_NONCE_SIZE]
+    ciphertext = frame[MEK_NONCE_SIZE:]
+    return iv, ciphertext
+
+
+def _decrypt_fcae_transport_stream_io(
+    client_public_key_b64: str,
+    source: BinaryIO,
+    ephemeral_private_key: X25519PrivateKey,
+) -> bytes:
+    prefix = source.read(FCAE_PREFIX_BYTES)
+    if len(prefix) < FCAE_PREFIX_BYTES:
+        raise ValueError("FCAE blob is too short")
+    if not is_fcae_transport_blob(prefix):
+        raise ValueError("Not an FCAE transport blob")
+    if prefix[4] != FCAE_VERSION:
+        raise ValueError("Unsupported FCAE version")
+    aes_key = derive_transport_file_aes_key(client_public_key_b64, ephemeral_private_key)
+    cipher = AESGCM(aes_key)
+    parts: list[bytes] = []
+    while True:
+        frame = _read_fcae_frame_payload(source)
+        if frame is None:
+            break
+        iv, ciphertext = frame
+        parts.append(cipher.decrypt(iv, ciphertext, None))
+    return b"".join(parts)
+
+
+def decrypt_fcae_transport_blob_to_file(
+    client_public_key_b64: str,
+    encrypted_path: Path,
+    ephemeral_private_key: X25519PrivateKey,
+    output_path: Path,
+) -> int:
+    """Stream-decrypt FCAE transport ciphertext from disk to a plaintext file."""
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    total_out = 0
+    with open(encrypted_path, "rb") as enc, open(output_path, "wb") as out:
+        prefix = enc.read(FCAE_PREFIX_BYTES)
+        if not is_fcae_transport_blob(prefix):
+            raise ValueError("Not an FCAE transport blob")
+        if prefix[4] != FCAE_VERSION:
+            raise ValueError("Unsupported FCAE version")
+        aes_key = derive_transport_file_aes_key(client_public_key_b64, ephemeral_private_key)
+        cipher = AESGCM(aes_key)
+        while True:
+            frame = _read_fcae_frame_payload(enc)
+            if frame is None:
+                break
+            iv, ciphertext = frame
+            plain = cipher.decrypt(iv, ciphertext, None)
+            out.write(plain)
+            total_out += len(plain)
+    return total_out
+
+
+def encrypt_message_to_file(plaintext_path: Path, mek: bytes, output_path: Path) -> str:
+    """AES-GCM encrypt a file on disk; returns nonce_b64. Ciphertext written to output_path."""
+    from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+
+    nonce = generate_nonce(MEK_NONCE_SIZE)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    encryptor = Cipher(algorithms.AES(mek), modes.GCM(nonce)).encryptor()
+    with open(plaintext_path, "rb") as src, open(output_path, "wb") as dst:
+        while True:
+            chunk = src.read(1024 * 1024)
+            if not chunk:
+                break
+            dst.write(encryptor.update(chunk))
+        dst.write(encryptor.finalize())
+    return base64.b64encode(nonce).decode("utf-8")
+
+
 def decrypt_transport_blob(
     client_public_key_b64: str,
     encrypted_blob: bytes,
@@ -146,6 +255,15 @@ def decrypt_transport_blob(
     Returns:
         Decrypted plaintext bytes.
     """
+    if is_fcae_transport_blob(encrypted_blob):
+        import io
+
+        return _decrypt_fcae_transport_stream_io(
+            client_public_key_b64,
+            io.BytesIO(encrypted_blob),
+            ephemeral_private_key,
+        )
+
     if len(encrypted_blob) < nonce_size + 16:
         # crypto_box has a MAC; ciphertext must have at least some overhead.
         raise ValueError("Encrypted blob is too short to contain nonce + ciphertext")

backend/services/messaging/main.py

@@ -15,7 +15,9 @@ import sys
 import time
 import base64
 import os
-from typing import Dict, Any
+import tempfile
+from pathlib import Path
+from typing import Dict, Any, Union
 from fastapi import FastAPI, HTTPException, status
 from nacl.exceptions import CryptoError
 from fastapi.middleware.cors import CORSMiddleware
@@ -27,7 +29,14 @@ logger = logging.getLogger("uvicorn.error")
 _B64_DECODE_KW = {"validate": True} if sys.version_info >= (3, 11) else {}
 
 # Import encryption modules
-from .encryption import generate_nonce, TRANSPORT_NONCE_SIZE, decrypt_transport_blob, decrypt_transport_message
+from .encryption import (
+    generate_nonce,
+    TRANSPORT_NONCE_SIZE,
+    decrypt_transport_blob,
+    decrypt_transport_message,
+    is_fcae_transport_blob,
+    decrypt_fcae_transport_blob_to_file,
+)
 from .processor import process_encrypted_message, process_encrypted_message_and_files
 
 try:
@@ -374,40 +383,84 @@ async def process_message_with_files(
     )
 
     plaintext_files: list[bytes] = []
+    plaintext_file_paths: list[Path | None] = []
     filenames: list[str] = []
-    for idx, tf in enumerate(transport_files):
-        enc_b64 = tf.get("encrypted_file_data_b64", "")
-        try:
-            transport_blob = base64.b64decode(enc_b64, **_B64_DECODE_KW)
-        except Exception as e:
-            logger.error("Invalid base64 for transport file index=%s filename=%r: %s", idx, tf.get("filename"), e)
-            raise
-        try:
-            plaintext_files.append(
-                decrypt_transport_blob(
-                    client_public_key_b64=client_public_key_b64,
-                    encrypted_blob=transport_blob,
-                    ephemeral_private_key=private_key,
-                )
-            )
-        except Exception as e:
-            logger.error(
-                "Transport file decrypt failed index=%s filename=%r (check same ephemeral as message): %s",
-                idx,
-                tf.get("filename"),
-                e,
-            )
-            raise
-        filenames.append(tf.get("filename", "file"))
+    temp_paths: list[Path] = []
+    try:
+        for idx, tf in enumerate(transport_files):
+            enc_path = (tf.get("encrypted_file_path") or "").strip()
+            if enc_path:
+                blob_path = Path(enc_path)
+                if not blob_path.is_file():
+                    raise ValueError(f"Transport file path missing index={idx}")
+                prefix = blob_path.read_bytes()[: len(b"FCAE") + 1]
+                if is_fcae_transport_blob(prefix):
+                    plain_tmp = Path(tempfile.mkstemp(prefix="fcae-plain-", suffix=".bin")[1])
+                    temp_paths.append(plain_tmp)
+                    decrypt_fcae_transport_blob_to_file(
+                        client_public_key_b64=client_public_key_b64,
+                        encrypted_path=blob_path,
+                        ephemeral_private_key=private_key,
+                        output_path=plain_tmp,
+                    )
+                    plaintext_files.append(b"")
+                    plaintext_file_paths.append(plain_tmp)
+                else:
+                    transport_blob = blob_path.read_bytes()
+                    plaintext_files.append(
+                        decrypt_transport_blob(
+                            client_public_key_b64=client_public_key_b64,
+                            encrypted_blob=transport_blob,
+                            ephemeral_private_key=private_key,
+                        )
+                    )
+                    plaintext_file_paths.append(None)
+            else:
+                enc_b64 = tf.get("encrypted_file_data_b64", "")
+                try:
+                    transport_blob = base64.b64decode(enc_b64, **_B64_DECODE_KW)
+                except Exception as e:
+                    logger.error(
+                        "Invalid base64 for transport file index=%s filename=%r: %s",
+                        idx,
+                        tf.get("filename"),
+                        e,
+                    )
+                    raise
+                try:
+                    plaintext_files.append(
+                        decrypt_transport_blob(
+                            client_public_key_b64=client_public_key_b64,
+                            encrypted_blob=transport_blob,
+                            ephemeral_private_key=private_key,
+                        )
+                    )
+                except Exception as e:
+                    logger.error(
+                        "Transport file decrypt failed index=%s filename=%r (check same ephemeral as message): %s",
+                        idx,
+                        tf.get("filename"),
+                        e,
+                    )
+                    raise
+                plaintext_file_paths.append(None)
+            filenames.append(tf.get("filename", "file"))
 
-    return process_encrypted_message_and_files(
-        plaintext_message=plaintext_message,
-        plaintext_files=plaintext_files,
-        filenames=filenames,
-        compliance_public_key_b64=compliance_public_key_b64,
-        sender_public_key_b64=sender_public_key_b64,
-        recipient_public_key_b64=recipient_public_key_b64,
-    )
+        return process_encrypted_message_and_files(
+            plaintext_message=plaintext_message,
+            plaintext_files=plaintext_files,
+            filenames=filenames,
+            plaintext_file_paths=plaintext_file_paths,
+            compliance_public_key_b64=compliance_public_key_b64,
+            sender_public_key_b64=sender_public_key_b64,
+            recipient_public_key_b64=recipient_public_key_b64,
+        )
+    finally:
+        for p in temp_paths:
+            try:
+                p.unlink(missing_ok=True)
+            except Exception:
+                pass
 
 
 @app.post("/process-with-files", response_model=None)

backend/services/messaging/processor.py

@@ -22,6 +22,7 @@ from .encryption import (
     decrypt_transport_message,
     generate_mek,
     encrypt_message,
+    encrypt_message_to_file,
     wrap_mek,
     derive_shared_secret,
     derive_key_from_shared_secret,
@@ -205,6 +206,9 @@ def _generate_thumbnail(image_bytes: bytes) -> tuple[str | None, list[int]]:
         return (None, [1, 1])
 
 
+_LARGE_FILE_THUMB_BYTES = 32 * 1024 * 1024
+
+
 def process_encrypted_message_and_files(
     plaintext_message: bytes,
     plaintext_files: list[bytes],
@@ -212,6 +216,7 @@ def process_encrypted_message_and_files(
     compliance_public_key_b64: str,
     sender_public_key_b64: str,
     recipient_public_key_b64: str,
+    plaintext_file_paths: list[Path | None] | None = None,
 ) -> Dict[str, Any]:
     """
     Process a message and its attached files using a single MEK.
@@ -230,6 +235,10 @@ def process_encrypted_message_and_files(
     if len(filenames) != len(plaintext_files):
         filenames = [f"file_{i}" for i in range(len(plaintext_files))]
 
+    paths = plaintext_file_paths or [None] * len(plaintext_files)
+    if len(paths) < len(plaintext_files):
+        paths = paths + [None] * (len(plaintext_files) - len(paths))
+
     # One MEK for everything in this envelope
     mek = generate_mek()
 
@@ -239,11 +248,24 @@ def process_encrypted_message_and_files(
     file_sizes: list[int] = []
     for i, f_bytes in enumerate(plaintext_files):
         name = filenames[i] if i < len(filenames) else ""
-        file_sizes.append(len(f_bytes))
-        if Path(name).suffix.lower() in _IMAGE_EXTENSIONS:
+        path = paths[i]
+        size = int(path.stat().st_size) if path is not None else len(f_bytes)
+        file_sizes.append(size)
+        if (
+            path is None
+            and Path(name).suffix.lower() in _IMAGE_EXTENSIONS
+        ):
             thumb_b64, wh = _generate_thumbnail(f_bytes)
             file_thumbnails.append(thumb_b64 or "")
             file_aspect_ratios.append(wh)
+        elif (
+            path is not None
+            and size <= _LARGE_FILE_THUMB_BYTES
+            and Path(name).suffix.lower() in _IMAGE_EXTENSIONS
+        ):
+            thumb_b64, wh = _generate_thumbnail(path.read_bytes())
+            file_thumbnails.append(thumb_b64 or "")
+            file_aspect_ratios.append(wh)
         else:
             file_thumbnails.append("")
             file_aspect_ratios.append([1, 1])
@@ -270,10 +292,17 @@ def process_encrypted_message_and_files(
 
     # Encrypt files (same MEK, per-file nonce)
     files_out: list[Dict[str, Any]] = []
+    import tempfile
+
     for i, f_bytes in enumerate(plaintext_files):
-        f_nonce, f_ciphertext = encrypt_message(f_bytes, mek)
-        entry: Dict[str, Any] = {"nonce": f_nonce, "ciphertext": f_ciphertext}
-        files_out.append(entry)
+        path = paths[i]
+        if path is not None:
+            enc_tmp = Path(tempfile.mkstemp(prefix="mek-enc-", suffix=".bin")[1])
+            f_nonce = encrypt_message_to_file(path, mek, enc_tmp)
+            files_out.append({"nonce": f_nonce, "ciphertext_path": str(enc_tmp)})
+        else:
+            f_nonce, f_ciphertext = encrypt_message(f_bytes, mek)
+            files_out.append({"nonce": f_nonce, "ciphertext": f_ciphertext})
 
     # Derive wrap keys deterministically (same as existing flow)
     sender_key_bytes = base64.b64decode(sender_public_key_b64)