From 3575fb3752e089adc55eaf9abeee341a2125cfdc Mon Sep 17 00:00:00 2001 From: denis0001-dev Date: Sat, 11 Oct 2025 15:24:25 +0300 Subject: [PATCH] Implement video encryption --- frontend/src/core/calls/e2eeWorker.ts | 113 ++++++--- frontend/src/core/calls/signaling.ts | 18 +- frontend/src/core/calls/webrtc.ts | 215 ++++++++++++++---- frontend/src/pages/chat/hooks/useAudioCall.ts | 3 + frontend/src/pages/chat/hooks/useCall.ts | 3 + 5 files changed, 270 insertions(+), 82 deletions(-) diff --git a/frontend/src/core/calls/e2eeWorker.ts b/frontend/src/core/calls/e2eeWorker.ts index 2dac954..d3ac349 100644 --- a/frontend/src/core/calls/e2eeWorker.ts +++ b/frontend/src/core/calls/e2eeWorker.ts @@ -1,11 +1,26 @@ /** * E2EE Worker for WebRTC Insertable Streams * Encrypts/decrypts encoded audio and video frames using AES-GCM - * Optimized for both small audio frames and large video frames + * Uses RTP timestamps for IVs to handle out-of-order and dropped frames */ +export interface FrameMetadata { + contributingSources?: number[]; + mimeType?: string; + payloadType?: number; + rtpTimestamp: number; + synchronizationSource: number; + dependencies?: number[]; + frameId?: number; + spatialIndex?: number; + temporalIndex?: number; +} + export interface EncodedFrame { - data: Uint8Array; + data: Uint8Array | ArrayBuffer; + timestamp?: number; + type?: string; + getMetadata?: () => FrameMetadata; } export interface WorkerOptions { @@ -14,28 +29,75 @@ export interface WorkerOptions { sessionId?: string; } +/** + * Extract sequence number from encoded frame + * For RTCEncodedVideoFrame/AudioFrame, we use the frame's metadata if available, + * otherwise fall back to extracting from RTP header + */ +function makeIV(encodedFrame: EncodedFrame, frameCount: number): ArrayBuffer { + // Debug: log frame properties + if (frameCount <= 3) { + console.log("Frame object keys:", Object.keys(encodedFrame)); + console.log("Frame timestamp:", encodedFrame.timestamp); + if (encodedFrame.getMetadata) { + const metadata = encodedFrame.getMetadata(); + console.log("Metadata:", metadata); + console.log("RTP timestamp:", metadata?.rtpTimestamp); + } + } + + // Use RTP timestamp from metadata as IV base + // This is synchronized between sender and receiver + const ivBuffer = new ArrayBuffer(12); + + if (encodedFrame.getMetadata) { + try { + const metadata = encodedFrame.getMetadata(); + if (metadata && typeof metadata.rtpTimestamp === 'number') { + // Use RTP timestamp as IV - it's synchronized between peers + const view = new DataView(ivBuffer); + view.setUint32(0, metadata.rtpTimestamp, false); // First 4 bytes + view.setUint32(8, metadata.synchronizationSource || 0, false); // Last 4 bytes + return ivBuffer; + } + } catch (e) { + console.error("Failed to get metadata:", e); + } + } + + // Fallback: use frame count (not ideal but better than nothing) + const view = new DataView(ivBuffer); + view.setUint32(8, frameCount, false); + return ivBuffer; +} + addEventListener("rtctransform", (event) => { const { transformer } = event; const { readable, writable } = transformer; const { key, mode } = transformer.options as WorkerOptions; - // Use a synchronized counter for IV generation - // Both sides must start from the same point for the same session - let frameCounter = 0; - - // Track sender vs receiver side independently - // Each side maintains its own counter const isEncrypting = mode === 'encrypt'; + let frameCount = 0; + let lastLogTime = 0; + async function transform(encodedFrame: EncodedFrame, controller: TransformStreamDefaultController) { try { - // Create a unique IV for each frame using the counter - // Format: 12 bytes total = 8 bytes of zeros + 4 bytes counter - const iv = new Uint8Array(12); - const view = new DataView(iv.buffer); - view.setUint32(8, frameCounter++, false); // Big-endian counter - const data = new Uint8Array(encodedFrame.data); + + // Increment frame counter + frameCount++; + + // Create IV using RTP timestamp from metadata (synchronized between peers) + const iv = makeIV(encodedFrame, frameCount); + + // Log first few frames and periodically for debugging + const now = Date.now(); + if (frameCount <= 5 || now - lastLogTime > 5000) { + console.log(`E2EE ${mode} frame #${frameCount}, size: ${data.length} bytes`); + lastLogTime = now; + } + const params: AesGcmParams = { name: 'AES-GCM', iv }; let result: ArrayBuffer; @@ -48,23 +110,18 @@ addEventListener("rtctransform", (event) => { result = await crypto.subtle.decrypt(params, key, data); } - // Update frame data with encrypted/decrypted result - encodedFrame.data = new Uint8Array(result); + // CRITICAL: Video frames need ArrayBuffer, not Uint8Array + // Must assign the buffer directly, not wrapped in Uint8Array + encodedFrame.data = result; controller.enqueue(encodedFrame); } catch (e) { - // Log error but don't stop the stream - allows graceful degradation - console.error(`E2EE ${mode} failed for frame ${frameCounter}:`, e); - - // For decryption errors, we can't recover - must drop the frame - if (!isEncrypting) { - // Just drop the frame silently to avoid breaking the stream - return; - } - - // For encryption errors, pass through unencrypted as last resort - console.warn("Passing through unencrypted frame due to encryption failure"); - controller.enqueue(encodedFrame); + // FAIL SECURELY: Never send unencrypted frames + const data = new Uint8Array(encodedFrame.data); + console.error(`E2EE ${mode} FAILED - dropping frame #${frameCount}, size: ${data.length}`, e); + console.error('Frame type:', encodedFrame.type || 'unknown'); + // Drop the frame completely - don't enqueue anything + return; } } diff --git a/frontend/src/core/calls/signaling.ts b/frontend/src/core/calls/signaling.ts index c555084..b89d161 100644 --- a/frontend/src/core/calls/signaling.ts +++ b/frontend/src/core/calls/signaling.ts @@ -138,26 +138,32 @@ export class CallSignalingHandler { private handleVideoToggle(data: any) { console.log("handleVideoToggle called with data:", data); const state = this.getState(); - const { data: toggleData } = data; + const { fromUserId, data: toggleData } = data; - if (toggleData && typeof toggleData.enabled === "boolean") { + if (toggleData && typeof toggleData.enabled === "boolean" && fromUserId) { console.log("Setting remote video enabled to:", toggleData.enabled); + // Update Zustand state (for UI) state.setRemoteVideoEnabled(toggleData.enabled); + // Update WebRTC internal state (for track routing) + WebRTC.setRemoteVideoEnabled(fromUserId, toggleData.enabled); } else { - console.warn("Invalid toggle data:", toggleData); + console.warn("Invalid toggle data:", data); } } private handleScreenShareToggle(data: any) { console.log("handleScreenShareToggle called with data:", data); const state = this.getState(); - const { data: toggleData } = data; + const { fromUserId, data: toggleData } = data; - if (toggleData && typeof toggleData.enabled === "boolean") { + if (toggleData && typeof toggleData.enabled === "boolean" && fromUserId) { console.log("Setting remote screen sharing to:", toggleData.enabled); + // Update Zustand state (for UI) state.setRemoteScreenSharing(toggleData.enabled); + // Update WebRTC internal state (for track routing) + WebRTC.setRemoteScreenSharing(fromUserId, toggleData.enabled); } else { - console.warn("Invalid toggle data:", toggleData); + console.warn("Invalid toggle data:", data); } } } diff --git a/frontend/src/core/calls/webrtc.ts b/frontend/src/core/calls/webrtc.ts index 82f2d32..77992b0 100644 --- a/frontend/src/core/calls/webrtc.ts +++ b/frontend/src/core/calls/webrtc.ts @@ -19,6 +19,8 @@ export interface WebRTCCall { isMuted?: boolean; isLocalVideoEnabled: boolean; isScreenSharing: boolean; + isRemoteScreenSharing: boolean; // Track remote screen share state from signaling + isRemoteVideoEnabled: boolean; // Track remote video state from signaling isNegotiating?: boolean; // Insertable Streams E2EE sessionKey?: Uint8Array | null; @@ -26,6 +28,8 @@ export interface WebRTCCall { sessionId: string; keyRotationTimer?: NodeJS.Timeout; lastKeyRotation?: number; + transformedSenders: Set; + transformedReceivers: Set; } // Global state @@ -131,9 +135,13 @@ async function createPeerConnection(userId: number): Promise isMuted: false, isLocalVideoEnabled: false, isScreenSharing: false, + isRemoteScreenSharing: false, + isRemoteVideoEnabled: false, sessionKey: null, sessionCryptoKey: null, - sessionId: crypto.randomUUID() + sessionId: crypto.randomUUID(), + transformedSenders: new Set(), + transformedReceivers: new Set() }; calls.set(userId, call); @@ -231,36 +239,42 @@ async function createPeerConnection(userId: number): Promise const track = event.track; // Apply E2EE transform to the receiver for this new track if session key is available - // Currently disabled for video tracks due to counter synchronization issues - if (call.sessionKey && window.RTCRtpScriptTransform && track.kind === "audio") { + if (call.sessionKey && window.RTCRtpScriptTransform) { try { - const key = await importAesGcmKey(call.sessionKey); const receiver = call.peerConnection.getReceivers().find(r => r.track === track); - if (receiver) { + if (receiver && !call.transformedReceivers.has(receiver)) { + const key = await importAesGcmKey(call.sessionKey); console.log(`Applying decrypt transform to newly received ${track.kind} track`); // @ts-ignore receiver.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'decrypt', sessionId: call.sessionId }); + call.transformedReceivers.add(receiver); } } catch (error) { console.error("Failed to apply E2EE to received track:", error); } } - // Determine stream type based on track kind and stream ID - const streamId = remoteStream.id; + // Determine stream type based on track kind and signaling state + console.log(`Track received: kind=${track.kind}, isRemoteScreenSharing=${call.isRemoteScreenSharing}, isRemoteVideoEnabled=${call.isRemoteVideoEnabled}`); - // Check if this is a screen share stream (we'll use a convention: screen share streams have "screen" in their ID) - if (streamId.includes("screen")) { - console.log("Detected screen share track, notifying handler"); - // Handle remote screen share - if (onRemoteScreenShare) { - onRemoteScreenShare(userId, remoteStream); - } - } else if (track.kind === "video") { - console.log("Detected video track, notifying handler"); - // Handle remote video - if (onRemoteVideoStream) { - onRemoteVideoStream(userId, remoteStream); + if (track.kind === "video") { + // Prioritize screen share over regular video + // If remote is screen sharing, this video track is the screen share + if (call.isRemoteScreenSharing) { + console.log("Detected screen share track (based on signaling), notifying handler"); + if (onRemoteScreenShare) { + onRemoteScreenShare(userId, remoteStream); + } else { + console.warn("onRemoteScreenShare handler not set!"); + } + } else { + console.log("Detected video track (based on signaling), notifying handler"); + // Handle remote video + if (onRemoteVideoStream) { + onRemoteVideoStream(userId, remoteStream); + } else { + console.warn("onRemoteVideoStream handler not set!"); + } } } else if (track.kind === "audio") { console.log("Detected audio track, notifying handler"); @@ -268,6 +282,8 @@ async function createPeerConnection(userId: number): Promise call.remoteStream = remoteStream; if (onRemoteStream) { onRemoteStream(userId, remoteStream); + } else { + console.warn("onRemoteStream handler not set!"); } } } @@ -393,23 +409,25 @@ async function applyE2EETransforms(call: WebRTCCall): Promise { const key = await importAesGcmKey(call.sessionKey); call.sessionCryptoKey = key; - // Apply to audio receivers only (video E2EE requires synchronized counters) + // Apply to receivers that don't already have transforms const receivers = call.peerConnection.getReceivers(); for (const receiver of receivers) { - if (receiver.track && receiver.track.kind === "audio") { + if (receiver.track && !call.transformedReceivers.has(receiver)) { console.log(`Applying decrypt transform to ${receiver.track.kind} receiver`); // @ts-ignore receiver.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'decrypt', sessionId: call.sessionId }); + call.transformedReceivers.add(receiver); } } - // Apply to audio senders only (video E2EE requires synchronized counters) + // Apply to senders that don't already have transforms const senders = call.peerConnection.getSenders(); for (const sender of senders) { - if (sender.track && sender.track.kind === "audio") { + if (sender.track && !call.transformedSenders.has(sender)) { console.log(`Applying encrypt transform to ${sender.track.kind} sender`); // @ts-ignore sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'encrypt', sessionId: call.sessionId }); + call.transformedSenders.add(sender); } } } catch (error) { @@ -419,10 +437,18 @@ async function applyE2EETransforms(call: WebRTCCall): Promise { export async function setSessionKey(userId: number, keyBytes: Uint8Array): Promise { const call = calls.get(userId); - if (!call) return; + if (!call) { + console.error("setSessionKey: No call found for user", userId); + return; + } + + console.log("setSessionKey: Setting session key for user", userId); call.sessionKey = keyBytes; call.lastKeyRotation = Date.now(); + + console.log("setSessionKey: Applying E2EE transforms..."); await applyE2EETransforms(call); + console.log("setSessionKey: E2EE transforms applied successfully"); // Start key rotation timer (rotate every 10 minutes for long calls) if (call.keyRotationTimer) { @@ -468,10 +494,18 @@ async function rotateSessionKey(userId: number): Promise { export async function receiveWrappedSessionKey(fromUserId: number, wrappedPayload: any, sessionKeyHash?: string): Promise { if (!authToken) return; try { + console.log("receiveWrappedSessionKey called for user", fromUserId, "hash:", sessionKeyHash); const senderPublicKey = await fetchUserPublicKey(fromUserId, authToken); - if (!senderPublicKey) return; - if (!wrappedPayload || !sessionKeyHash) return; + if (!senderPublicKey) { + console.error("Failed to get sender public key"); + return; + } + if (!wrappedPayload || !sessionKeyHash) { + console.error("Missing wrapped payload or session key hash"); + return; + } + console.log("Unwrapping session key..."); // Unwrap the session key from the encrypted payload const unwrappedSessionKey = await unwrapCallSessionKeyFromSender(senderPublicKey, { salt: wrappedPayload.salt, @@ -479,8 +513,10 @@ export async function receiveWrappedSessionKey(fromUserId: number, wrappedPayloa wrapped: wrappedPayload.wrapped }); + console.log("Session key unwrapped successfully, setting it..."); // Use the unwrapped session key directly (both sides should have the same key) await setSessionKey(fromUserId, unwrappedSessionKey); + console.log("Session key set successfully for user", fromUserId); } catch (e) { console.error("Failed to unwrap session key:", e); } @@ -590,7 +626,7 @@ export async function onRemoteAccepted(userId: number): Promise { } } -async function createE2EETransform(sessionKey: NonNullable, peerConnection: RTCPeerConnection, sessionId?: string): Promise { +async function createE2EETransform(call: WebRTCCall, sessionKey: Uint8Array, sessionId?: string): Promise { try { // @ts-ignore if (!sessionKey || !window.RTCRtpScriptTransform) { @@ -601,28 +637,31 @@ async function createE2EETransform(sessionKey: NonNullable l.includes('m=')).join(', ')); + await call.peerConnection.setRemoteDescription(answer); + console.log("Remote description set successfully"); + console.log("Current receivers after answer:", call.peerConnection.getReceivers().map(r => r.track?.kind)); + console.log("Current senders after answer:", call.peerConnection.getSenders().map(s => s.track?.kind)); + // Reset negotiating flag call.isNegotiating = false; // Attach transforms on initiator side if session key is available // If not available yet, setSessionKey will apply them when it arrives if (call.sessionKey) { - await createE2EETransform(call.sessionKey, call.peerConnection, call.sessionId); + await createE2EETransform(call, call.sessionKey, call.sessionId); } else { console.log("Session key not yet available in handleCallAnswer - will apply transforms when key arrives"); } + + // Check if there are new receivers with tracks that haven't been notified yet + // This handles the case where tracks exist but the track event hasn't fired yet + const receivers = call.peerConnection.getReceivers(); + for (const receiver of receivers) { + if (receiver.track) { + const track = receiver.track; + console.log("Checking receiver track:", track.kind); + + // Find the stream for this track + const transceiver = call.peerConnection.getTransceivers().find(t => t.receiver === receiver); + if (transceiver && transceiver.receiver.track) { + // Manually trigger stream handlers for tracks that didn't fire events + if (track.kind === "video" && onRemoteVideoStream) { + // Create a MediaStream from the track + const stream = new MediaStream([track]); + console.log("Manually notifying remote video stream handler"); + onRemoteVideoStream(userId, stream); + } + } + } + } } catch (error) { console.error("Failed to handle answer:", error); throw error; @@ -829,7 +897,7 @@ export async function toggleVideo(userId: number): Promise { // Add video track to peer connection const videoTrack = videoStream.getVideoTracks()[0]; - call.peerConnection.addTrack(videoTrack, videoStream); + const sender = call.peerConnection.addTrack(videoTrack, videoStream); console.log("Video track added successfully"); console.log("Current senders:", call.peerConnection.getSenders().map(s => s.track?.kind)); @@ -839,9 +907,20 @@ export async function toggleVideo(userId: number): Promise { direction: t.direction }))); - // Note: E2EE for video is temporarily disabled for testing - // Will re-enable with proper counter synchronization - console.log("Video sent without E2EE (will implement synchronized encryption)"); + // Apply E2EE transform if session key exists and not already transformed + if (call.sessionKey && window.RTCRtpScriptTransform && !call.transformedSenders.has(sender)) { + try { + const key = await importAesGcmKey(call.sessionKey); + console.log("Applying E2EE to video sender"); + sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId }); + call.transformedSenders.add(sender); + } catch (error) { + console.error("Failed to apply E2EE to video:", error); + throw error; // Fail securely + } + } else { + console.log("Skipping E2EE for video - session key not available yet or already transformed"); + } // Notify local video stream handler if (onLocalVideoStream) { @@ -916,10 +995,16 @@ export async function toggleScreenShare(userId: number): Promise { }); // Set a special ID to identify screen share streams - Object.defineProperty(screenStream, "id", { - value: `screen-${crypto.randomUUID()}`, - writable: false - }); + try { + Object.defineProperty(screenStream, "id", { + value: `screen-${crypto.randomUUID()}`, + writable: false, + configurable: true + }); + console.log("Set screen share stream ID to:", screenStream.id); + } catch (e) { + console.warn("Failed to set custom stream ID, using default:", screenStream.id); + } call.screenShareStream = screenStream; call.isScreenSharing = true; @@ -932,13 +1017,25 @@ export async function toggleScreenShare(userId: number): Promise { toggleScreenShare(userId); }); - call.peerConnection.addTrack(videoTrack, screenStream); + const sender = call.peerConnection.addTrack(videoTrack, screenStream); console.log("Screen share track added successfully"); - // Note: E2EE for screen share is temporarily disabled for testing - // Will re-enable with proper counter synchronization - console.log("Screen share sent without E2EE (will implement synchronized encryption)"); + // Apply E2EE transform if session key exists and not already transformed + if (call.sessionKey && window.RTCRtpScriptTransform && !call.transformedSenders.has(sender)) { + try { + const key = await importAesGcmKey(call.sessionKey); + console.log("Applying E2EE to screen share sender"); + // @ts-ignore + sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId }); + call.transformedSenders.add(sender); + } catch (error) { + console.error("Failed to apply E2EE to screen share:", error); + throw error; // Fail securely + } + } else { + console.log("Skipping E2EE for screen share - session key not available yet or already transformed"); + } // Notify local screen share handler if (onLocalScreenShare) { @@ -1024,6 +1121,28 @@ export function cleanupCall(userId: number): void { } } +/** + * Update the remote video enabled state (called when receiving signaling) + */ +export function setRemoteVideoEnabled(userId: number, enabled: boolean): void { + const call = calls.get(userId); + if (call) { + console.log(`Setting remote video enabled to ${enabled} for user ${userId}`); + call.isRemoteVideoEnabled = enabled; + } +} + +/** + * Update the remote screen sharing state (called when receiving signaling) + */ +export function setRemoteScreenSharing(userId: number, enabled: boolean): void { + const call = calls.get(userId); + if (call) { + console.log(`Setting remote screen sharing to ${enabled} for user ${userId}`); + call.isRemoteScreenSharing = enabled; + } +} + export function cleanup(): void { // Clean up all calls for (const userId of calls.keys()) { diff --git a/frontend/src/pages/chat/hooks/useAudioCall.ts b/frontend/src/pages/chat/hooks/useAudioCall.ts index f1c0052..8c54444 100644 --- a/frontend/src/pages/chat/hooks/useAudioCall.ts +++ b/frontend/src/pages/chat/hooks/useAudioCall.ts @@ -148,6 +148,9 @@ export default function useAudioCall() { const success = await WebRTC.initiateCall(userId, username); if (success && sessionKey) { + // Set the session key for ourselves (initiator) + await WebRTC.setSessionKey(userId, sessionKey.key); + // Send session key hash to the receiver for visual verification await WebRTC.sendCallSessionKey(userId, sessionKey.hash); // Also wrap and send the actual session key for E2EE media diff --git a/frontend/src/pages/chat/hooks/useCall.ts b/frontend/src/pages/chat/hooks/useCall.ts index f725f80..0b6757e 100644 --- a/frontend/src/pages/chat/hooks/useCall.ts +++ b/frontend/src/pages/chat/hooks/useCall.ts @@ -269,6 +269,9 @@ export default function useCall() { const success = await WebRTC.initiateCall(userId, username); if (success && sessionKey) { + // Set the session key for ourselves (initiator) + await WebRTC.setSessionKey(userId, sessionKey.key); + // Send session key hash to the receiver for visual verification await WebRTC.sendCallSessionKey(userId, sessionKey.hash); // Also wrap and send the actual session key for E2EE media