Fix screenshare glitches

frontend/src/core/calls/e2eeWorker.ts

@@ -34,30 +34,30 @@ export interface WorkerOptions {
  * For RTCEncodedVideoFrame/AudioFrame, we use the frame's metadata if available,
  * otherwise fall back to extracting from RTP header
  */
-function makeIV(encodedFrame: EncodedFrame, frameCount: number): ArrayBuffer {
+function makeIV(encodedFrame: EncodedFrame, frameCount: number, mode: 'encrypt' | 'decrypt'): ArrayBuffer {
-    // Debug: log frame properties
+    // Create IV using ONLY RTP metadata - this ensures sender and receiver use identical IVs
-    if (frameCount <= 3) {
+    // Frame data can differ between sender/receiver due to encoding differences
-        console.log("Frame object keys:", Object.keys(encodedFrame));
-        console.log("Frame timestamp:", encodedFrame.timestamp);
-        if (encodedFrame.getMetadata) {
-            const metadata = encodedFrame.getMetadata();
-            console.log("Metadata:", metadata);
-            console.log("RTP timestamp:", metadata?.rtpTimestamp);
-        }
-    }
-    
-    // Use RTP timestamp from metadata as IV base
-    // This is synchronized between sender and receiver
     const ivBuffer = new ArrayBuffer(12);
+    const view = new DataView(ivBuffer);
     
     if (encodedFrame.getMetadata) {
         try {
             const metadata = encodedFrame.getMetadata();
             if (metadata && typeof metadata.rtpTimestamp === 'number') {
-                // Use RTP timestamp as IV - it's synchronized between peers
+                // Use ONLY RTP timestamp + sync source - these are identical on both sides
-                const view = new DataView(ivBuffer);
                 view.setUint32(0, metadata.rtpTimestamp, false); // First 4 bytes
-                view.setUint32(8, metadata.synchronizationSource || 0, false); // Last 4 bytes
+                view.setUint32(4, metadata.synchronizationSource || 0, false); // Middle 4 bytes
+                view.setUint32(8, 0, false); // Last 4 bytes (padding for 12-byte IV)
+                
+                // Debug first few frames only
+                if (frameCount <= 3) {
+                    console.log(`${mode.toUpperCase()} IV for frame #${frameCount}:`, {
+                        rtpTimestamp: metadata.rtpTimestamp,
+                        syncSource: metadata.synchronizationSource,
+                        mimeType: metadata.mimeType
+                    });
+                }
+                
                 return ivBuffer;
             }
         } catch (e) {
@@ -65,9 +65,10 @@ function makeIV(encodedFrame: EncodedFrame, frameCount: number): ArrayBuffer {
         }
     }
     
-    // Fallback: use frame count (not ideal but better than nothing)
+    // Fallback: use timestamp only (no random to avoid desync)
-    const view = new DataView(ivBuffer);
+    view.setUint32(0, Date.now() & 0xFFFFFFFF, false);
-    view.setUint32(8, frameCount, false);
+    view.setUint32(4, 0, false);
+    view.setUint32(8, 0, false);
     return ivBuffer;
 }
 
@@ -78,8 +79,11 @@ addEventListener("rtctransform", (event) => {
     
     const isEncrypting = mode === 'encrypt';
     
+    console.log(`E2EE Worker started in ${mode.toUpperCase()} mode`);
+    
     let frameCount = 0;
     let lastLogTime = 0;
+    let lastKeyCheck = Date.now();
     
     async function transform(encodedFrame: EncodedFrame, controller: TransformStreamDefaultController<EncodedFrame>) {
         try {
@@ -88,8 +92,13 @@ addEventListener("rtctransform", (event) => {
             // Increment frame counter
             frameCount++;
             
+            // Log every frame for debugging (only first 3)
+            if (frameCount <= 3) {
+                console.log(`${mode.toUpperCase()} Processing frame #${frameCount}, size: ${data.length}`);
+            }
+            
             // Create IV using RTP timestamp from metadata (synchronized between peers)
-            const iv = makeIV(encodedFrame, frameCount);
+            const iv = makeIV(encodedFrame, frameCount, mode);
             
             // Log first few frames and periodically for debugging
             const now = Date.now();
@@ -98,21 +107,74 @@ addEventListener("rtctransform", (event) => {
                 lastLogTime = now;
             }
             
-            const params: AesGcmParams = { name: 'AES-GCM', iv };
+            // For screen share, check if we need to request key rotation more frequently
+            // Screen share generates much more data and can benefit from more frequent key rotation
+            if (data.length > 50000 && now - lastKeyCheck > 60000) { // 1 minute for large frames
+                console.log("Large frame detected, suggesting key rotation for screen share");
+                lastKeyCheck = now;
+            }
 
-            let result: ArrayBuffer;
+            // Detect potential browser window glitching - frames with specific characteristics
+            if (data.length > 100000 && frameCount > 10) { // Large frames after initial setup
+                const frameType = encodedFrame.type || 'unknown';
+                if (frameType === 'key' && data.length > 200000) {
+                    console.log("Large keyframe detected - possible browser window glitch, frame size:", data.length);
+                }
+            }
             
-            if (isEncrypting) {
+            // Ensure IV is properly typed
-                // Encrypt: just encrypt the raw frame data
+            const ivArray = new Uint8Array(iv);
-                result = await crypto.subtle.encrypt(params, key, data);
+            const params: AesGcmParams = { name: 'AES-GCM', iv: ivArray };
+            
+            // COMPROMISE: Encrypt most of the frame while preserving minimal codec compatibility
+            // This prevents most visual leakage while maintaining decodability
+            let headerSize = 0;
+            let payloadData: Uint8Array;
+            
+            if (data.length > 20) {
+                // For video frames, preserve only the first 2 bytes (minimal codec sync)
+                // This is the absolute minimum needed for frame identification
+                headerSize = 2;
+                payloadData = data.slice(headerSize);
             } else {
-                // Decrypt: just decrypt the raw frame data
+                // For small frames (likely audio), encrypt everything
-                result = await crypto.subtle.decrypt(params, key, data);
+                payloadData = data;
+            }
+            
+            // Encrypt the payload data
+            const payloadBuffer = new ArrayBuffer(payloadData.byteLength);
+            new Uint8Array(payloadBuffer).set(payloadData);
+            
+            let encryptedPayload: ArrayBuffer;
+            if (isEncrypting) {
+                encryptedPayload = await crypto.subtle.encrypt(params, key, payloadBuffer);
+            } else {
+                try {
+                    encryptedPayload = await crypto.subtle.decrypt(params, key, payloadBuffer);
+                } catch (error) {
+                    console.error(`E2EE ${mode} FAILED - dropping frame #${frameCount}, size: ${data.length}`, error);
+                    return; // Drop the frame
+                }
+            }
+            
+            // Reconstruct frame: minimal headers + encrypted payload
+            const encryptedArray = new Uint8Array(encryptedPayload);
+            const result = new Uint8Array(headerSize + encryptedArray.length);
+            
+            if (headerSize > 0) {
+                result.set(data.slice(0, headerSize), 0); // Copy minimal headers
+                result.set(encryptedArray, headerSize); // Add encrypted payload
+            } else {
+                result.set(encryptedArray, 0);
+            }
+            
+            // Log first few frames for debugging
+            if (frameCount <= 3) {
+                console.log(`E2EE ${mode} frame #${frameCount}: ${data.length} -> ${result.length} bytes (header: ${headerSize})`);
             }
             
             // CRITICAL: Video frames need ArrayBuffer, not Uint8Array
-            // Must assign the buffer directly, not wrapped in Uint8Array
+            encodedFrame.data = result.buffer;
-            encodedFrame.data = result;
             controller.enqueue(encodedFrame);
             
         } catch (e) {
@@ -120,6 +182,13 @@ addEventListener("rtctransform", (event) => {
             const data = new Uint8Array(encodedFrame.data);
             console.error(`E2EE ${mode} FAILED - dropping frame #${frameCount}, size: ${data.length}`, e);
             console.error('Frame type:', encodedFrame.type || 'unknown');
+            
+            // For screen share, be more aggressive about dropping corrupted frames
+            // to prevent progressive glitch accumulation
+            if (mode === 'decrypt' && frameCount > 10) {
+                console.warn(`Dropping corrupted frame #${frameCount} to prevent glitch accumulation`);
+            }
+            
             // Drop the frame completely - don't enqueue anything
             return;
         }

frontend/src/core/calls/webrtc.ts

@@ -238,20 +238,27 @@ async function createPeerConnection(userId: number): Promise<RTCPeerConnection>
         if (call && remoteStream) {
             const track = event.track;
             
-            // Apply E2EE transform to the receiver for this new track if session key is available
+            // Apply E2EE transform to all tracks - video now uses header-preserving encryption
             if (call.sessionKey && window.RTCRtpScriptTransform) {
                 try {
                     const receiver = call.peerConnection.getReceivers().find(r => r.track === track);
                     if (receiver && !call.transformedReceivers.has(receiver)) {
                         const key = await importAesGcmKey(call.sessionKey);
-                        console.log(`Applying decrypt transform to newly received ${track.kind} track`);
+                        console.log(`Applying decrypt transform to newly received ${track.kind} track:`);
+                        console.log("- sessionId:", call.sessionId);
+                        console.log("- sessionKey (first 8 bytes):", Array.from(new Uint8Array(call.sessionKey).slice(0, 8)));
                         // @ts-ignore
                         receiver.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'decrypt', sessionId: call.sessionId });
                         call.transformedReceivers.add(receiver);
+                        console.log(`Decrypt transform applied successfully to ${track.kind} track`);
                     }
                 } catch (error) {
                     console.error("Failed to apply E2EE to received track:", error);
                 }
+            } else {
+                console.log(`Skipping decrypt transform for ${track.kind} track - session key not available or RTCRtpScriptTransform not supported`);
+                console.log("Session key exists:", !!call.sessionKey);
+                console.log("RTCRtpScriptTransform available:", !!window.RTCRtpScriptTransform);
             }
             
             // Determine stream type based on track kind and signaling state
@@ -897,7 +904,7 @@ export async function toggleVideo(userId: number): Promise<boolean> {
 
             // Add video track to peer connection
             const videoTrack = videoStream.getVideoTracks()[0];
-            const sender = call.peerConnection.addTrack(videoTrack, videoStream);
+            call.peerConnection.addTrack(videoTrack, videoStream);
 
             console.log("Video track added successfully");
             console.log("Current senders:", call.peerConnection.getSenders().map(s => s.track?.kind));
@@ -907,19 +914,21 @@ export async function toggleVideo(userId: number): Promise<boolean> {
                 direction: t.direction
             })));
 
-            // Apply E2EE transform if session key exists and not already transformed
+            // Apply E2EE transform with header-preserving encryption for video
-            if (call.sessionKey && window.RTCRtpScriptTransform && !call.transformedSenders.has(sender)) {
+            if (call.sessionKey && window.RTCRtpScriptTransform) {
                 try {
                     const key = await importAesGcmKey(call.sessionKey);
-                    console.log("Applying E2EE to video sender");
+                    console.log("Applying E2EE to video sender with header preservation");
+                    const sender = call.peerConnection.getSenders().find(s => s.track === videoTrack);
+                    if (sender && !call.transformedSenders.has(sender)) {
                         sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId });
                         call.transformedSenders.add(sender);
+                        console.log("E2EE applied to video sender successfully");
+                    }
                 } catch (error) {
                     console.error("Failed to apply E2EE to video:", error);
                     throw error; // Fail securely
                 }
-            } else {
-                console.log("Skipping E2EE for video - session key not available yet or already transformed");
             }
 
             // Notify local video stream handler
@@ -988,9 +997,12 @@ export async function toggleScreenShare(userId: number): Promise<boolean> {
     if (!call.isScreenSharing) {
         // Enable screen sharing
         try {
-            // @ts-ignore - getDisplayMedia might not be in all TypeScript versions
             const screenStream = await navigator.mediaDevices.getDisplayMedia({
-                video: true,
+                video: {
+                    width: { ideal: 1920, max: 3840 },
+                    height: { ideal: 1080, max: 2160 },
+                    frameRate: { ideal: 60, max: 60 }
+                },
                 audio: false
             });
 
@@ -1017,32 +1029,9 @@ export async function toggleScreenShare(userId: number): Promise<boolean> {
                 toggleScreenShare(userId);
             });
 
-            const sender = call.peerConnection.addTrack(videoTrack, screenStream);
+            // Send signaling message FIRST to notify remote peer before adding track
-
+            // This ensures the receiver knows it's screen share before the track arrives
-            console.log("Screen share track added successfully");
+            console.log("Sending screen share toggle BEFORE adding track");
-
-            // Apply E2EE transform if session key exists and not already transformed
-            if (call.sessionKey && window.RTCRtpScriptTransform && !call.transformedSenders.has(sender)) {
-                try {
-                    const key = await importAesGcmKey(call.sessionKey);
-                    console.log("Applying E2EE to screen share sender");
-                    // @ts-ignore
-                    sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId });
-                    call.transformedSenders.add(sender);
-                } catch (error) {
-                    console.error("Failed to apply E2EE to screen share:", error);
-                    throw error; // Fail securely
-                }
-            } else {
-                console.log("Skipping E2EE for screen share - session key not available yet or already transformed");
-            }
-
-            // Notify local screen share handler
-            if (onLocalScreenShare) {
-                onLocalScreenShare(userId, screenStream);
-            }
-
-            // Send signaling message to notify remote peer
             await sendSignalingMessage({
                 type: "call_screen_share_toggle",
                 fromUserId: 0,
@@ -1050,6 +1039,56 @@ export async function toggleScreenShare(userId: number): Promise<boolean> {
                 data: { enabled: true }
             });
 
+            // Small delay to ensure signaling message is processed before track arrives
+            await new Promise(resolve => setTimeout(resolve, 100));
+
+            call.peerConnection.addTrack(videoTrack, screenStream);
+
+            console.log("Screen share track added, immediately applying E2EE transform");
+
+            // CRITICAL: Apply E2EE transform IMMEDIATELY after track is added
+            if (call.sessionKey && window.RTCRtpScriptTransform) {
+                try {
+                    const key = await importAesGcmKey(call.sessionKey);
+                    console.log("Applying E2EE to screen share sender:");
+                    console.log("- sessionId:", call.sessionId);
+                    console.log("- sessionKey (first 8 bytes):", Array.from(new Uint8Array(call.sessionKey).slice(0, 8)));
+                    console.log("Available senders:", call.peerConnection.getSenders().map(s => ({ 
+                        track: s.track?.kind, 
+                        id: s.track?.id 
+                    })));
+                    console.log("Looking for screen share track:", videoTrack.id);
+                    
+                    const sender = call.peerConnection.getSenders().find(s => s.track === videoTrack);
+                    console.log("Found screen share sender:", !!sender);
+                    
+                    if (sender && !call.transformedSenders.has(sender)) {
+                        sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId });
+                        call.transformedSenders.add(sender);
+                        console.log("E2EE applied to screen share sender successfully");
+                    } else {
+                        console.log("Screen share sender not found or already transformed");
+                    }
+                } catch (error) {
+                    console.error("Failed to apply E2EE to screen share:", error);
+                    throw error; // Fail securely
+                }
+            } else {
+                console.log("Skipping E2EE for screen share - session key not available yet");
+                console.log("Session key exists:", !!call.sessionKey);
+                console.log("RTCRtpScriptTransform available:", !!window.RTCRtpScriptTransform);
+            }
+
+            // Let browser handle screen share settings naturally
+            // Avoid applying constraints that might cause glitches
+
+            // Notify local screen share handler
+            if (onLocalScreenShare) {
+                onLocalScreenShare(userId, screenStream);
+            }
+
+            console.log("Screen share setup complete - using existing session key:", call.sessionId);
+
             return true;
         } catch (error) {
             console.error("Failed to enable screen sharing:", error);

frontend/src/utils/crypto/symmetric.ts

@@ -26,6 +26,9 @@ export async function aesGcmDecrypt(key: CryptoKey, iv: Uint8Array | ArrayBuffer
 }
 
 export async function importAesGcmKey(rawKey: Uint8Array | ArrayBuffer): Promise<CryptoKey> {
-	const keyBuffer = rawKey instanceof Uint8Array ? rawKey.buffer as ArrayBuffer : rawKey;
+	// Normalize to a contiguous ArrayBuffer slice to avoid offset/length issues
+	const keyBuffer = rawKey instanceof Uint8Array
+		? (rawKey.buffer as ArrayBuffer).slice(rawKey.byteOffset, rawKey.byteOffset + rawKey.byteLength)
+		: (rawKey as ArrayBuffer);
 	return crypto.subtle.importKey("raw", keyBuffer, { name: "AES-GCM" }, false, ["encrypt", "decrypt"]);
 }