From 174ab8f1faf0ae8d61b0e0441519d2ebf3995839 Mon Sep 17 00:00:00 2001 From: denis0001-dev Date: Sat, 11 Oct 2025 18:33:03 +0300 Subject: [PATCH] Fix screenshare glitches --- frontend/src/core/calls/e2eeWorker.ts | 129 +++++++++++++++++++------ frontend/src/core/calls/webrtc.ts | 115 ++++++++++++++-------- frontend/src/utils/crypto/symmetric.ts | 5 +- 3 files changed, 180 insertions(+), 69 deletions(-) diff --git a/frontend/src/core/calls/e2eeWorker.ts b/frontend/src/core/calls/e2eeWorker.ts index d3ac349..bb4dba7 100644 --- a/frontend/src/core/calls/e2eeWorker.ts +++ b/frontend/src/core/calls/e2eeWorker.ts @@ -34,30 +34,30 @@ export interface WorkerOptions { * For RTCEncodedVideoFrame/AudioFrame, we use the frame's metadata if available, * otherwise fall back to extracting from RTP header */ -function makeIV(encodedFrame: EncodedFrame, frameCount: number): ArrayBuffer { - // Debug: log frame properties - if (frameCount <= 3) { - console.log("Frame object keys:", Object.keys(encodedFrame)); - console.log("Frame timestamp:", encodedFrame.timestamp); - if (encodedFrame.getMetadata) { - const metadata = encodedFrame.getMetadata(); - console.log("Metadata:", metadata); - console.log("RTP timestamp:", metadata?.rtpTimestamp); - } - } - - // Use RTP timestamp from metadata as IV base - // This is synchronized between sender and receiver +function makeIV(encodedFrame: EncodedFrame, frameCount: number, mode: 'encrypt' | 'decrypt'): ArrayBuffer { + // Create IV using ONLY RTP metadata - this ensures sender and receiver use identical IVs + // Frame data can differ between sender/receiver due to encoding differences const ivBuffer = new ArrayBuffer(12); + const view = new DataView(ivBuffer); if (encodedFrame.getMetadata) { try { const metadata = encodedFrame.getMetadata(); if (metadata && typeof metadata.rtpTimestamp === 'number') { - // Use RTP timestamp as IV - it's synchronized between peers - const view = new DataView(ivBuffer); + // Use ONLY RTP timestamp + sync source - these are identical on both sides view.setUint32(0, metadata.rtpTimestamp, false); // First 4 bytes - view.setUint32(8, metadata.synchronizationSource || 0, false); // Last 4 bytes + view.setUint32(4, metadata.synchronizationSource || 0, false); // Middle 4 bytes + view.setUint32(8, 0, false); // Last 4 bytes (padding for 12-byte IV) + + // Debug first few frames only + if (frameCount <= 3) { + console.log(`${mode.toUpperCase()} IV for frame #${frameCount}:`, { + rtpTimestamp: metadata.rtpTimestamp, + syncSource: metadata.synchronizationSource, + mimeType: metadata.mimeType + }); + } + return ivBuffer; } } catch (e) { @@ -65,9 +65,10 @@ function makeIV(encodedFrame: EncodedFrame, frameCount: number): ArrayBuffer { } } - // Fallback: use frame count (not ideal but better than nothing) - const view = new DataView(ivBuffer); - view.setUint32(8, frameCount, false); + // Fallback: use timestamp only (no random to avoid desync) + view.setUint32(0, Date.now() & 0xFFFFFFFF, false); + view.setUint32(4, 0, false); + view.setUint32(8, 0, false); return ivBuffer; } @@ -78,8 +79,11 @@ addEventListener("rtctransform", (event) => { const isEncrypting = mode === 'encrypt'; + console.log(`E2EE Worker started in ${mode.toUpperCase()} mode`); + let frameCount = 0; let lastLogTime = 0; + let lastKeyCheck = Date.now(); async function transform(encodedFrame: EncodedFrame, controller: TransformStreamDefaultController) { try { @@ -88,8 +92,13 @@ addEventListener("rtctransform", (event) => { // Increment frame counter frameCount++; + // Log every frame for debugging (only first 3) + if (frameCount <= 3) { + console.log(`${mode.toUpperCase()} Processing frame #${frameCount}, size: ${data.length}`); + } + // Create IV using RTP timestamp from metadata (synchronized between peers) - const iv = makeIV(encodedFrame, frameCount); + const iv = makeIV(encodedFrame, frameCount, mode); // Log first few frames and periodically for debugging const now = Date.now(); @@ -98,21 +107,74 @@ addEventListener("rtctransform", (event) => { lastLogTime = now; } - const params: AesGcmParams = { name: 'AES-GCM', iv }; + // For screen share, check if we need to request key rotation more frequently + // Screen share generates much more data and can benefit from more frequent key rotation + if (data.length > 50000 && now - lastKeyCheck > 60000) { // 1 minute for large frames + console.log("Large frame detected, suggesting key rotation for screen share"); + lastKeyCheck = now; + } + + // Detect potential browser window glitching - frames with specific characteristics + if (data.length > 100000 && frameCount > 10) { // Large frames after initial setup + const frameType = encodedFrame.type || 'unknown'; + if (frameType === 'key' && data.length > 200000) { + console.log("Large keyframe detected - possible browser window glitch, frame size:", data.length); + } + } - let result: ArrayBuffer; + // Ensure IV is properly typed + const ivArray = new Uint8Array(iv); + const params: AesGcmParams = { name: 'AES-GCM', iv: ivArray }; - if (isEncrypting) { - // Encrypt: just encrypt the raw frame data - result = await crypto.subtle.encrypt(params, key, data); + // COMPROMISE: Encrypt most of the frame while preserving minimal codec compatibility + // This prevents most visual leakage while maintaining decodability + let headerSize = 0; + let payloadData: Uint8Array; + + if (data.length > 20) { + // For video frames, preserve only the first 2 bytes (minimal codec sync) + // This is the absolute minimum needed for frame identification + headerSize = 2; + payloadData = data.slice(headerSize); } else { - // Decrypt: just decrypt the raw frame data - result = await crypto.subtle.decrypt(params, key, data); + // For small frames (likely audio), encrypt everything + payloadData = data; + } + + // Encrypt the payload data + const payloadBuffer = new ArrayBuffer(payloadData.byteLength); + new Uint8Array(payloadBuffer).set(payloadData); + + let encryptedPayload: ArrayBuffer; + if (isEncrypting) { + encryptedPayload = await crypto.subtle.encrypt(params, key, payloadBuffer); + } else { + try { + encryptedPayload = await crypto.subtle.decrypt(params, key, payloadBuffer); + } catch (error) { + console.error(`E2EE ${mode} FAILED - dropping frame #${frameCount}, size: ${data.length}`, error); + return; // Drop the frame + } + } + + // Reconstruct frame: minimal headers + encrypted payload + const encryptedArray = new Uint8Array(encryptedPayload); + const result = new Uint8Array(headerSize + encryptedArray.length); + + if (headerSize > 0) { + result.set(data.slice(0, headerSize), 0); // Copy minimal headers + result.set(encryptedArray, headerSize); // Add encrypted payload + } else { + result.set(encryptedArray, 0); + } + + // Log first few frames for debugging + if (frameCount <= 3) { + console.log(`E2EE ${mode} frame #${frameCount}: ${data.length} -> ${result.length} bytes (header: ${headerSize})`); } // CRITICAL: Video frames need ArrayBuffer, not Uint8Array - // Must assign the buffer directly, not wrapped in Uint8Array - encodedFrame.data = result; + encodedFrame.data = result.buffer; controller.enqueue(encodedFrame); } catch (e) { @@ -120,6 +182,13 @@ addEventListener("rtctransform", (event) => { const data = new Uint8Array(encodedFrame.data); console.error(`E2EE ${mode} FAILED - dropping frame #${frameCount}, size: ${data.length}`, e); console.error('Frame type:', encodedFrame.type || 'unknown'); + + // For screen share, be more aggressive about dropping corrupted frames + // to prevent progressive glitch accumulation + if (mode === 'decrypt' && frameCount > 10) { + console.warn(`Dropping corrupted frame #${frameCount} to prevent glitch accumulation`); + } + // Drop the frame completely - don't enqueue anything return; } diff --git a/frontend/src/core/calls/webrtc.ts b/frontend/src/core/calls/webrtc.ts index 77992b0..744b346 100644 --- a/frontend/src/core/calls/webrtc.ts +++ b/frontend/src/core/calls/webrtc.ts @@ -238,20 +238,27 @@ async function createPeerConnection(userId: number): Promise if (call && remoteStream) { const track = event.track; - // Apply E2EE transform to the receiver for this new track if session key is available + // Apply E2EE transform to all tracks - video now uses header-preserving encryption if (call.sessionKey && window.RTCRtpScriptTransform) { try { const receiver = call.peerConnection.getReceivers().find(r => r.track === track); if (receiver && !call.transformedReceivers.has(receiver)) { const key = await importAesGcmKey(call.sessionKey); - console.log(`Applying decrypt transform to newly received ${track.kind} track`); + console.log(`Applying decrypt transform to newly received ${track.kind} track:`); + console.log("- sessionId:", call.sessionId); + console.log("- sessionKey (first 8 bytes):", Array.from(new Uint8Array(call.sessionKey).slice(0, 8))); // @ts-ignore receiver.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'decrypt', sessionId: call.sessionId }); call.transformedReceivers.add(receiver); + console.log(`Decrypt transform applied successfully to ${track.kind} track`); } } catch (error) { console.error("Failed to apply E2EE to received track:", error); } + } else { + console.log(`Skipping decrypt transform for ${track.kind} track - session key not available or RTCRtpScriptTransform not supported`); + console.log("Session key exists:", !!call.sessionKey); + console.log("RTCRtpScriptTransform available:", !!window.RTCRtpScriptTransform); } // Determine stream type based on track kind and signaling state @@ -897,7 +904,7 @@ export async function toggleVideo(userId: number): Promise { // Add video track to peer connection const videoTrack = videoStream.getVideoTracks()[0]; - const sender = call.peerConnection.addTrack(videoTrack, videoStream); + call.peerConnection.addTrack(videoTrack, videoStream); console.log("Video track added successfully"); console.log("Current senders:", call.peerConnection.getSenders().map(s => s.track?.kind)); @@ -907,19 +914,21 @@ export async function toggleVideo(userId: number): Promise { direction: t.direction }))); - // Apply E2EE transform if session key exists and not already transformed - if (call.sessionKey && window.RTCRtpScriptTransform && !call.transformedSenders.has(sender)) { + // Apply E2EE transform with header-preserving encryption for video + if (call.sessionKey && window.RTCRtpScriptTransform) { try { const key = await importAesGcmKey(call.sessionKey); - console.log("Applying E2EE to video sender"); - sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId }); - call.transformedSenders.add(sender); + console.log("Applying E2EE to video sender with header preservation"); + const sender = call.peerConnection.getSenders().find(s => s.track === videoTrack); + if (sender && !call.transformedSenders.has(sender)) { + sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId }); + call.transformedSenders.add(sender); + console.log("E2EE applied to video sender successfully"); + } } catch (error) { console.error("Failed to apply E2EE to video:", error); throw error; // Fail securely } - } else { - console.log("Skipping E2EE for video - session key not available yet or already transformed"); } // Notify local video stream handler @@ -988,9 +997,12 @@ export async function toggleScreenShare(userId: number): Promise { if (!call.isScreenSharing) { // Enable screen sharing try { - // @ts-ignore - getDisplayMedia might not be in all TypeScript versions const screenStream = await navigator.mediaDevices.getDisplayMedia({ - video: true, + video: { + width: { ideal: 1920, max: 3840 }, + height: { ideal: 1080, max: 2160 }, + frameRate: { ideal: 60, max: 60 } + }, audio: false }); @@ -1017,32 +1029,9 @@ export async function toggleScreenShare(userId: number): Promise { toggleScreenShare(userId); }); - const sender = call.peerConnection.addTrack(videoTrack, screenStream); - - console.log("Screen share track added successfully"); - - // Apply E2EE transform if session key exists and not already transformed - if (call.sessionKey && window.RTCRtpScriptTransform && !call.transformedSenders.has(sender)) { - try { - const key = await importAesGcmKey(call.sessionKey); - console.log("Applying E2EE to screen share sender"); - // @ts-ignore - sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId }); - call.transformedSenders.add(sender); - } catch (error) { - console.error("Failed to apply E2EE to screen share:", error); - throw error; // Fail securely - } - } else { - console.log("Skipping E2EE for screen share - session key not available yet or already transformed"); - } - - // Notify local screen share handler - if (onLocalScreenShare) { - onLocalScreenShare(userId, screenStream); - } - - // Send signaling message to notify remote peer + // Send signaling message FIRST to notify remote peer before adding track + // This ensures the receiver knows it's screen share before the track arrives + console.log("Sending screen share toggle BEFORE adding track"); await sendSignalingMessage({ type: "call_screen_share_toggle", fromUserId: 0, @@ -1050,6 +1039,56 @@ export async function toggleScreenShare(userId: number): Promise { data: { enabled: true } }); + // Small delay to ensure signaling message is processed before track arrives + await new Promise(resolve => setTimeout(resolve, 100)); + + call.peerConnection.addTrack(videoTrack, screenStream); + + console.log("Screen share track added, immediately applying E2EE transform"); + + // CRITICAL: Apply E2EE transform IMMEDIATELY after track is added + if (call.sessionKey && window.RTCRtpScriptTransform) { + try { + const key = await importAesGcmKey(call.sessionKey); + console.log("Applying E2EE to screen share sender:"); + console.log("- sessionId:", call.sessionId); + console.log("- sessionKey (first 8 bytes):", Array.from(new Uint8Array(call.sessionKey).slice(0, 8))); + console.log("Available senders:", call.peerConnection.getSenders().map(s => ({ + track: s.track?.kind, + id: s.track?.id + }))); + console.log("Looking for screen share track:", videoTrack.id); + + const sender = call.peerConnection.getSenders().find(s => s.track === videoTrack); + console.log("Found screen share sender:", !!sender); + + if (sender && !call.transformedSenders.has(sender)) { + sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId }); + call.transformedSenders.add(sender); + console.log("E2EE applied to screen share sender successfully"); + } else { + console.log("Screen share sender not found or already transformed"); + } + } catch (error) { + console.error("Failed to apply E2EE to screen share:", error); + throw error; // Fail securely + } + } else { + console.log("Skipping E2EE for screen share - session key not available yet"); + console.log("Session key exists:", !!call.sessionKey); + console.log("RTCRtpScriptTransform available:", !!window.RTCRtpScriptTransform); + } + + // Let browser handle screen share settings naturally + // Avoid applying constraints that might cause glitches + + // Notify local screen share handler + if (onLocalScreenShare) { + onLocalScreenShare(userId, screenStream); + } + + console.log("Screen share setup complete - using existing session key:", call.sessionId); + return true; } catch (error) { console.error("Failed to enable screen sharing:", error); diff --git a/frontend/src/utils/crypto/symmetric.ts b/frontend/src/utils/crypto/symmetric.ts index 45b85d0..804680b 100644 --- a/frontend/src/utils/crypto/symmetric.ts +++ b/frontend/src/utils/crypto/symmetric.ts @@ -26,6 +26,9 @@ export async function aesGcmDecrypt(key: CryptoKey, iv: Uint8Array | ArrayBuffer } export async function importAesGcmKey(rawKey: Uint8Array | ArrayBuffer): Promise { - const keyBuffer = rawKey instanceof Uint8Array ? rawKey.buffer as ArrayBuffer : rawKey; + // Normalize to a contiguous ArrayBuffer slice to avoid offset/length issues + const keyBuffer = rawKey instanceof Uint8Array + ? (rawKey.buffer as ArrayBuffer).slice(rawKey.byteOffset, rawKey.byteOffset + rawKey.byteLength) + : (rawKey as ArrayBuffer); return crypto.subtle.importKey("raw", keyBuffer, { name: "AES-GCM" }, false, ["encrypt", "decrypt"]); } \ No newline at end of file