mirror of
https://github.com/fromchat-messenger/web.git
synced 2026-09-22 19:15:08 +03:00
Fix screenshare glitches
This commit is contained in:
@@ -34,30 +34,30 @@ export interface WorkerOptions {
|
||||
* For RTCEncodedVideoFrame/AudioFrame, we use the frame's metadata if available,
|
||||
* otherwise fall back to extracting from RTP header
|
||||
*/
|
||||
function makeIV(encodedFrame: EncodedFrame, frameCount: number): ArrayBuffer {
|
||||
// Debug: log frame properties
|
||||
if (frameCount <= 3) {
|
||||
console.log("Frame object keys:", Object.keys(encodedFrame));
|
||||
console.log("Frame timestamp:", encodedFrame.timestamp);
|
||||
if (encodedFrame.getMetadata) {
|
||||
const metadata = encodedFrame.getMetadata();
|
||||
console.log("Metadata:", metadata);
|
||||
console.log("RTP timestamp:", metadata?.rtpTimestamp);
|
||||
}
|
||||
}
|
||||
|
||||
// Use RTP timestamp from metadata as IV base
|
||||
// This is synchronized between sender and receiver
|
||||
function makeIV(encodedFrame: EncodedFrame, frameCount: number, mode: 'encrypt' | 'decrypt'): ArrayBuffer {
|
||||
// Create IV using ONLY RTP metadata - this ensures sender and receiver use identical IVs
|
||||
// Frame data can differ between sender/receiver due to encoding differences
|
||||
const ivBuffer = new ArrayBuffer(12);
|
||||
const view = new DataView(ivBuffer);
|
||||
|
||||
if (encodedFrame.getMetadata) {
|
||||
try {
|
||||
const metadata = encodedFrame.getMetadata();
|
||||
if (metadata && typeof metadata.rtpTimestamp === 'number') {
|
||||
// Use RTP timestamp as IV - it's synchronized between peers
|
||||
const view = new DataView(ivBuffer);
|
||||
// Use ONLY RTP timestamp + sync source - these are identical on both sides
|
||||
view.setUint32(0, metadata.rtpTimestamp, false); // First 4 bytes
|
||||
view.setUint32(8, metadata.synchronizationSource || 0, false); // Last 4 bytes
|
||||
view.setUint32(4, metadata.synchronizationSource || 0, false); // Middle 4 bytes
|
||||
view.setUint32(8, 0, false); // Last 4 bytes (padding for 12-byte IV)
|
||||
|
||||
// Debug first few frames only
|
||||
if (frameCount <= 3) {
|
||||
console.log(`${mode.toUpperCase()} IV for frame #${frameCount}:`, {
|
||||
rtpTimestamp: metadata.rtpTimestamp,
|
||||
syncSource: metadata.synchronizationSource,
|
||||
mimeType: metadata.mimeType
|
||||
});
|
||||
}
|
||||
|
||||
return ivBuffer;
|
||||
}
|
||||
} catch (e) {
|
||||
@@ -65,9 +65,10 @@ function makeIV(encodedFrame: EncodedFrame, frameCount: number): ArrayBuffer {
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: use frame count (not ideal but better than nothing)
|
||||
const view = new DataView(ivBuffer);
|
||||
view.setUint32(8, frameCount, false);
|
||||
// Fallback: use timestamp only (no random to avoid desync)
|
||||
view.setUint32(0, Date.now() & 0xFFFFFFFF, false);
|
||||
view.setUint32(4, 0, false);
|
||||
view.setUint32(8, 0, false);
|
||||
return ivBuffer;
|
||||
}
|
||||
|
||||
@@ -78,8 +79,11 @@ addEventListener("rtctransform", (event) => {
|
||||
|
||||
const isEncrypting = mode === 'encrypt';
|
||||
|
||||
console.log(`E2EE Worker started in ${mode.toUpperCase()} mode`);
|
||||
|
||||
let frameCount = 0;
|
||||
let lastLogTime = 0;
|
||||
let lastKeyCheck = Date.now();
|
||||
|
||||
async function transform(encodedFrame: EncodedFrame, controller: TransformStreamDefaultController<EncodedFrame>) {
|
||||
try {
|
||||
@@ -88,8 +92,13 @@ addEventListener("rtctransform", (event) => {
|
||||
// Increment frame counter
|
||||
frameCount++;
|
||||
|
||||
// Log every frame for debugging (only first 3)
|
||||
if (frameCount <= 3) {
|
||||
console.log(`${mode.toUpperCase()} Processing frame #${frameCount}, size: ${data.length}`);
|
||||
}
|
||||
|
||||
// Create IV using RTP timestamp from metadata (synchronized between peers)
|
||||
const iv = makeIV(encodedFrame, frameCount);
|
||||
const iv = makeIV(encodedFrame, frameCount, mode);
|
||||
|
||||
// Log first few frames and periodically for debugging
|
||||
const now = Date.now();
|
||||
@@ -98,21 +107,74 @@ addEventListener("rtctransform", (event) => {
|
||||
lastLogTime = now;
|
||||
}
|
||||
|
||||
const params: AesGcmParams = { name: 'AES-GCM', iv };
|
||||
// For screen share, check if we need to request key rotation more frequently
|
||||
// Screen share generates much more data and can benefit from more frequent key rotation
|
||||
if (data.length > 50000 && now - lastKeyCheck > 60000) { // 1 minute for large frames
|
||||
console.log("Large frame detected, suggesting key rotation for screen share");
|
||||
lastKeyCheck = now;
|
||||
}
|
||||
|
||||
let result: ArrayBuffer;
|
||||
// Detect potential browser window glitching - frames with specific characteristics
|
||||
if (data.length > 100000 && frameCount > 10) { // Large frames after initial setup
|
||||
const frameType = encodedFrame.type || 'unknown';
|
||||
if (frameType === 'key' && data.length > 200000) {
|
||||
console.log("Large keyframe detected - possible browser window glitch, frame size:", data.length);
|
||||
}
|
||||
}
|
||||
|
||||
if (isEncrypting) {
|
||||
// Encrypt: just encrypt the raw frame data
|
||||
result = await crypto.subtle.encrypt(params, key, data);
|
||||
// Ensure IV is properly typed
|
||||
const ivArray = new Uint8Array(iv);
|
||||
const params: AesGcmParams = { name: 'AES-GCM', iv: ivArray };
|
||||
|
||||
// COMPROMISE: Encrypt most of the frame while preserving minimal codec compatibility
|
||||
// This prevents most visual leakage while maintaining decodability
|
||||
let headerSize = 0;
|
||||
let payloadData: Uint8Array;
|
||||
|
||||
if (data.length > 20) {
|
||||
// For video frames, preserve only the first 2 bytes (minimal codec sync)
|
||||
// This is the absolute minimum needed for frame identification
|
||||
headerSize = 2;
|
||||
payloadData = data.slice(headerSize);
|
||||
} else {
|
||||
// Decrypt: just decrypt the raw frame data
|
||||
result = await crypto.subtle.decrypt(params, key, data);
|
||||
// For small frames (likely audio), encrypt everything
|
||||
payloadData = data;
|
||||
}
|
||||
|
||||
// Encrypt the payload data
|
||||
const payloadBuffer = new ArrayBuffer(payloadData.byteLength);
|
||||
new Uint8Array(payloadBuffer).set(payloadData);
|
||||
|
||||
let encryptedPayload: ArrayBuffer;
|
||||
if (isEncrypting) {
|
||||
encryptedPayload = await crypto.subtle.encrypt(params, key, payloadBuffer);
|
||||
} else {
|
||||
try {
|
||||
encryptedPayload = await crypto.subtle.decrypt(params, key, payloadBuffer);
|
||||
} catch (error) {
|
||||
console.error(`E2EE ${mode} FAILED - dropping frame #${frameCount}, size: ${data.length}`, error);
|
||||
return; // Drop the frame
|
||||
}
|
||||
}
|
||||
|
||||
// Reconstruct frame: minimal headers + encrypted payload
|
||||
const encryptedArray = new Uint8Array(encryptedPayload);
|
||||
const result = new Uint8Array(headerSize + encryptedArray.length);
|
||||
|
||||
if (headerSize > 0) {
|
||||
result.set(data.slice(0, headerSize), 0); // Copy minimal headers
|
||||
result.set(encryptedArray, headerSize); // Add encrypted payload
|
||||
} else {
|
||||
result.set(encryptedArray, 0);
|
||||
}
|
||||
|
||||
// Log first few frames for debugging
|
||||
if (frameCount <= 3) {
|
||||
console.log(`E2EE ${mode} frame #${frameCount}: ${data.length} -> ${result.length} bytes (header: ${headerSize})`);
|
||||
}
|
||||
|
||||
// CRITICAL: Video frames need ArrayBuffer, not Uint8Array
|
||||
// Must assign the buffer directly, not wrapped in Uint8Array
|
||||
encodedFrame.data = result;
|
||||
encodedFrame.data = result.buffer;
|
||||
controller.enqueue(encodedFrame);
|
||||
|
||||
} catch (e) {
|
||||
@@ -120,6 +182,13 @@ addEventListener("rtctransform", (event) => {
|
||||
const data = new Uint8Array(encodedFrame.data);
|
||||
console.error(`E2EE ${mode} FAILED - dropping frame #${frameCount}, size: ${data.length}`, e);
|
||||
console.error('Frame type:', encodedFrame.type || 'unknown');
|
||||
|
||||
// For screen share, be more aggressive about dropping corrupted frames
|
||||
// to prevent progressive glitch accumulation
|
||||
if (mode === 'decrypt' && frameCount > 10) {
|
||||
console.warn(`Dropping corrupted frame #${frameCount} to prevent glitch accumulation`);
|
||||
}
|
||||
|
||||
// Drop the frame completely - don't enqueue anything
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -238,20 +238,27 @@ async function createPeerConnection(userId: number): Promise<RTCPeerConnection>
|
||||
if (call && remoteStream) {
|
||||
const track = event.track;
|
||||
|
||||
// Apply E2EE transform to the receiver for this new track if session key is available
|
||||
// Apply E2EE transform to all tracks - video now uses header-preserving encryption
|
||||
if (call.sessionKey && window.RTCRtpScriptTransform) {
|
||||
try {
|
||||
const receiver = call.peerConnection.getReceivers().find(r => r.track === track);
|
||||
if (receiver && !call.transformedReceivers.has(receiver)) {
|
||||
const key = await importAesGcmKey(call.sessionKey);
|
||||
console.log(`Applying decrypt transform to newly received ${track.kind} track`);
|
||||
console.log(`Applying decrypt transform to newly received ${track.kind} track:`);
|
||||
console.log("- sessionId:", call.sessionId);
|
||||
console.log("- sessionKey (first 8 bytes):", Array.from(new Uint8Array(call.sessionKey).slice(0, 8)));
|
||||
// @ts-ignore
|
||||
receiver.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'decrypt', sessionId: call.sessionId });
|
||||
call.transformedReceivers.add(receiver);
|
||||
console.log(`Decrypt transform applied successfully to ${track.kind} track`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to apply E2EE to received track:", error);
|
||||
}
|
||||
} else {
|
||||
console.log(`Skipping decrypt transform for ${track.kind} track - session key not available or RTCRtpScriptTransform not supported`);
|
||||
console.log("Session key exists:", !!call.sessionKey);
|
||||
console.log("RTCRtpScriptTransform available:", !!window.RTCRtpScriptTransform);
|
||||
}
|
||||
|
||||
// Determine stream type based on track kind and signaling state
|
||||
@@ -897,7 +904,7 @@ export async function toggleVideo(userId: number): Promise<boolean> {
|
||||
|
||||
// Add video track to peer connection
|
||||
const videoTrack = videoStream.getVideoTracks()[0];
|
||||
const sender = call.peerConnection.addTrack(videoTrack, videoStream);
|
||||
call.peerConnection.addTrack(videoTrack, videoStream);
|
||||
|
||||
console.log("Video track added successfully");
|
||||
console.log("Current senders:", call.peerConnection.getSenders().map(s => s.track?.kind));
|
||||
@@ -907,19 +914,21 @@ export async function toggleVideo(userId: number): Promise<boolean> {
|
||||
direction: t.direction
|
||||
})));
|
||||
|
||||
// Apply E2EE transform if session key exists and not already transformed
|
||||
if (call.sessionKey && window.RTCRtpScriptTransform && !call.transformedSenders.has(sender)) {
|
||||
// Apply E2EE transform with header-preserving encryption for video
|
||||
if (call.sessionKey && window.RTCRtpScriptTransform) {
|
||||
try {
|
||||
const key = await importAesGcmKey(call.sessionKey);
|
||||
console.log("Applying E2EE to video sender");
|
||||
console.log("Applying E2EE to video sender with header preservation");
|
||||
const sender = call.peerConnection.getSenders().find(s => s.track === videoTrack);
|
||||
if (sender && !call.transformedSenders.has(sender)) {
|
||||
sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId });
|
||||
call.transformedSenders.add(sender);
|
||||
console.log("E2EE applied to video sender successfully");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to apply E2EE to video:", error);
|
||||
throw error; // Fail securely
|
||||
}
|
||||
} else {
|
||||
console.log("Skipping E2EE for video - session key not available yet or already transformed");
|
||||
}
|
||||
|
||||
// Notify local video stream handler
|
||||
@@ -988,9 +997,12 @@ export async function toggleScreenShare(userId: number): Promise<boolean> {
|
||||
if (!call.isScreenSharing) {
|
||||
// Enable screen sharing
|
||||
try {
|
||||
// @ts-ignore - getDisplayMedia might not be in all TypeScript versions
|
||||
const screenStream = await navigator.mediaDevices.getDisplayMedia({
|
||||
video: true,
|
||||
video: {
|
||||
width: { ideal: 1920, max: 3840 },
|
||||
height: { ideal: 1080, max: 2160 },
|
||||
frameRate: { ideal: 60, max: 60 }
|
||||
},
|
||||
audio: false
|
||||
});
|
||||
|
||||
@@ -1017,32 +1029,9 @@ export async function toggleScreenShare(userId: number): Promise<boolean> {
|
||||
toggleScreenShare(userId);
|
||||
});
|
||||
|
||||
const sender = call.peerConnection.addTrack(videoTrack, screenStream);
|
||||
|
||||
console.log("Screen share track added successfully");
|
||||
|
||||
// Apply E2EE transform if session key exists and not already transformed
|
||||
if (call.sessionKey && window.RTCRtpScriptTransform && !call.transformedSenders.has(sender)) {
|
||||
try {
|
||||
const key = await importAesGcmKey(call.sessionKey);
|
||||
console.log("Applying E2EE to screen share sender");
|
||||
// @ts-ignore
|
||||
sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId });
|
||||
call.transformedSenders.add(sender);
|
||||
} catch (error) {
|
||||
console.error("Failed to apply E2EE to screen share:", error);
|
||||
throw error; // Fail securely
|
||||
}
|
||||
} else {
|
||||
console.log("Skipping E2EE for screen share - session key not available yet or already transformed");
|
||||
}
|
||||
|
||||
// Notify local screen share handler
|
||||
if (onLocalScreenShare) {
|
||||
onLocalScreenShare(userId, screenStream);
|
||||
}
|
||||
|
||||
// Send signaling message to notify remote peer
|
||||
// Send signaling message FIRST to notify remote peer before adding track
|
||||
// This ensures the receiver knows it's screen share before the track arrives
|
||||
console.log("Sending screen share toggle BEFORE adding track");
|
||||
await sendSignalingMessage({
|
||||
type: "call_screen_share_toggle",
|
||||
fromUserId: 0,
|
||||
@@ -1050,6 +1039,56 @@ export async function toggleScreenShare(userId: number): Promise<boolean> {
|
||||
data: { enabled: true }
|
||||
});
|
||||
|
||||
// Small delay to ensure signaling message is processed before track arrives
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
call.peerConnection.addTrack(videoTrack, screenStream);
|
||||
|
||||
console.log("Screen share track added, immediately applying E2EE transform");
|
||||
|
||||
// CRITICAL: Apply E2EE transform IMMEDIATELY after track is added
|
||||
if (call.sessionKey && window.RTCRtpScriptTransform) {
|
||||
try {
|
||||
const key = await importAesGcmKey(call.sessionKey);
|
||||
console.log("Applying E2EE to screen share sender:");
|
||||
console.log("- sessionId:", call.sessionId);
|
||||
console.log("- sessionKey (first 8 bytes):", Array.from(new Uint8Array(call.sessionKey).slice(0, 8)));
|
||||
console.log("Available senders:", call.peerConnection.getSenders().map(s => ({
|
||||
track: s.track?.kind,
|
||||
id: s.track?.id
|
||||
})));
|
||||
console.log("Looking for screen share track:", videoTrack.id);
|
||||
|
||||
const sender = call.peerConnection.getSenders().find(s => s.track === videoTrack);
|
||||
console.log("Found screen share sender:", !!sender);
|
||||
|
||||
if (sender && !call.transformedSenders.has(sender)) {
|
||||
sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId });
|
||||
call.transformedSenders.add(sender);
|
||||
console.log("E2EE applied to screen share sender successfully");
|
||||
} else {
|
||||
console.log("Screen share sender not found or already transformed");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to apply E2EE to screen share:", error);
|
||||
throw error; // Fail securely
|
||||
}
|
||||
} else {
|
||||
console.log("Skipping E2EE for screen share - session key not available yet");
|
||||
console.log("Session key exists:", !!call.sessionKey);
|
||||
console.log("RTCRtpScriptTransform available:", !!window.RTCRtpScriptTransform);
|
||||
}
|
||||
|
||||
// Let browser handle screen share settings naturally
|
||||
// Avoid applying constraints that might cause glitches
|
||||
|
||||
// Notify local screen share handler
|
||||
if (onLocalScreenShare) {
|
||||
onLocalScreenShare(userId, screenStream);
|
||||
}
|
||||
|
||||
console.log("Screen share setup complete - using existing session key:", call.sessionId);
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error("Failed to enable screen sharing:", error);
|
||||
|
||||
@@ -26,6 +26,9 @@ export async function aesGcmDecrypt(key: CryptoKey, iv: Uint8Array | ArrayBuffer
|
||||
}
|
||||
|
||||
export async function importAesGcmKey(rawKey: Uint8Array | ArrayBuffer): Promise<CryptoKey> {
|
||||
const keyBuffer = rawKey instanceof Uint8Array ? rawKey.buffer as ArrayBuffer : rawKey;
|
||||
// Normalize to a contiguous ArrayBuffer slice to avoid offset/length issues
|
||||
const keyBuffer = rawKey instanceof Uint8Array
|
||||
? (rawKey.buffer as ArrayBuffer).slice(rawKey.byteOffset, rawKey.byteOffset + rawKey.byteLength)
|
||||
: (rawKey as ArrayBuffer);
|
||||
return crypto.subtle.importKey("raw", keyBuffer, { name: "AES-GCM" }, false, ["encrypt", "decrypt"]);
|
||||
}
|
||||
Reference in New Issue
Block a user