diff --git a/backend/routes/messaging.py b/backend/routes/messaging.py index e44eede..d76a28d 100644 --- a/backend/routes/messaging.py +++ b/backend/routes/messaging.py @@ -914,6 +914,9 @@ class MessaggingSocketManager: elif type == "call_signaling": # Forward WebRTC signaling between peers try: + current_user = get_current_user_inner() + if not current_user: + raise HTTPException(401) self.user_by_ws[websocket] = current_user.id payload = data.get("data") or {} @@ -934,6 +937,64 @@ class MessaggingSocketManager: await websocket.send_json({"type": "call_signaling", "data": {"status": "ok"}}) except HTTPException as e: await self.send_error(websocket, type, e) + elif type == "call_video_toggle": + # Forward video toggle state between peers + try: + current_user = get_current_user_inner() + if not current_user: + raise HTTPException(401) + self.user_by_ws[websocket] = current_user.id + + payload = data.get("data") or {} + to_user_id = int(payload.get("toUserId") or 0) + if not to_user_id: + raise HTTPException(status_code=400, detail="Missing toUserId") + + # Ensure sender is set by the server + payload["fromUserId"] = current_user.id + + await self.send_to_user(to_user_id, { + "type": "call_signaling", + "data": { + "type": "call_video_toggle", + "fromUserId": current_user.id, + "toUserId": to_user_id, + "data": {"enabled": payload.get("enabled", False)} + } + }) + + await websocket.send_json({"type": "call_video_toggle", "data": {"status": "ok"}}) + except HTTPException as e: + await self.send_error(websocket, type, e) + elif type == "call_screen_share_toggle": + # Forward screen share toggle state between peers + try: + current_user = get_current_user_inner() + if not current_user: + raise HTTPException(401) + self.user_by_ws[websocket] = current_user.id + + payload = data.get("data") or {} + to_user_id = int(payload.get("toUserId") or 0) + if not to_user_id: + raise HTTPException(status_code=400, detail="Missing toUserId") + + # Ensure sender is set by the server + payload["fromUserId"] = current_user.id + + await self.send_to_user(to_user_id, { + "type": "call_signaling", + "data": { + "type": "call_screen_share_toggle", + "fromUserId": current_user.id, + "toUserId": to_user_id, + "data": {"enabled": payload.get("enabled", False)} + } + }) + + await websocket.send_json({"type": "call_screen_share_toggle", "data": {"status": "ok"}}) + except HTTPException as e: + await self.send_error(websocket, type, e) else: await websocket.send_json({"type": type, "error": {"code": 400, "detail": "Invalid type"}}) diff --git a/frontend/src/core/calls/e2eeWorker.ts b/frontend/src/core/calls/e2eeWorker.ts index 3d453de..0feb9a5 100644 --- a/frontend/src/core/calls/e2eeWorker.ts +++ b/frontend/src/core/calls/e2eeWorker.ts @@ -1,96 +1,146 @@ /** * E2EE Worker for WebRTC Insertable Streams - * Conditionally encrypts or decrypts encoded audio frames using AES-GCM + * Encrypts/decrypts encoded audio and video frames using AES-GCM + * Uses RTP timestamps for IVs to handle out-of-order and dropped frames */ +export interface FrameMetadata { + contributingSources?: number[]; + mimeType?: string; + payloadType?: number; + rtpTimestamp: number; + synchronizationSource: number; + dependencies?: number[]; + frameId?: number; + spatialIndex?: number; + temporalIndex?: number; +} + export interface EncodedFrame { - data: Uint8Array; + data: Uint8Array | ArrayBuffer; + timestamp?: number; + type?: string; + getMetadata?: () => FrameMetadata; } export interface WorkerOptions { key: CryptoKey; mode: 'encrypt' | 'decrypt'; - sessionId?: string; // For replay protection + sessionId?: string; +} + +/** + * Extract sequence number from encoded frame + * For RTCEncodedVideoFrame/AudioFrame, we use the frame's metadata if available, + * otherwise fall back to extracting from RTP header + */ +function makeIV(encodedFrame: EncodedFrame): ArrayBuffer { + // Create IV using ONLY RTP metadata - this ensures sender and receiver use identical IVs + // Frame data can differ between sender/receiver due to encoding differences + const ivBuffer = new ArrayBuffer(12); + const view = new DataView(ivBuffer); + + if (encodedFrame.getMetadata) { + try { + const metadata = encodedFrame.getMetadata(); + if (metadata && typeof metadata.rtpTimestamp === 'number') { + // Use ONLY RTP timestamp + sync source - these are identical on both sides + view.setUint32(0, metadata.rtpTimestamp, false); // First 4 bytes + view.setUint32(4, metadata.synchronizationSource || 0, false); // Middle 4 bytes + view.setUint32(8, 0, false); // Last 4 bytes (padding for 12-byte IV) + + + return ivBuffer; + } + } catch (e) { + console.error("Failed to get metadata:", e); + } + } + + // Fallback: use timestamp only (no random to avoid desync) + view.setUint32(0, Date.now() & 0xFFFFFFFF, false); + view.setUint32(4, 0, false); + view.setUint32(8, 0, false); + return ivBuffer; } addEventListener("rtctransform", (event) => { const { transformer } = event; const { readable, writable } = transformer; - const { key, mode, sessionId } = transformer.options as WorkerOptions; + const { key, mode } = transformer.options as WorkerOptions; + + const isEncrypting = mode === 'encrypt'; + + console.log(`E2EE Worker started in ${mode.toUpperCase()} mode`); + + let frameCount = 0; - // Generate a random base IV once per transform session - const ivBase = crypto.getRandomValues(new Uint8Array(8)); // 8 random bytes - let frameCounter = 0; - let lastFrameTime = 0; - const FRAME_WINDOW_MS = 5000; // 5 second window for replay protection - async function transform(encodedFrame: EncodedFrame, controller: TransformStreamDefaultController) { try { - const currentTime = Date.now(); - - // Replay protection for decryption - if (mode === 'decrypt') { - // Simple time-based replay protection - if (currentTime - lastFrameTime > FRAME_WINDOW_MS && lastFrameTime > 0) { - console.warn("Potential replay attack detected - frame outside time window"); - controller.error(new Error("Replay attack detected")); - return; - } - lastFrameTime = currentTime; - } - - // Create IV: 8 random bytes + 4-byte frame counter - const iv = new Uint8Array(12); - iv.set(ivBase, 0); // Copy random base - const view = new DataView(iv.buffer); - view.setUint32(8, frameCounter++, false); // Big-endian frame counter - const data = new Uint8Array(encodedFrame.data); - // Add frame metadata for authentication - const frameMetadata = new TextEncoder().encode(JSON.stringify({ - frameNumber: frameCounter - 1, - timestamp: currentTime, - sessionId: sessionId || 'default' - })); + // Increment frame counter + frameCount++; - // Combine frame data with metadata - const combinedData = new Uint8Array(data.length + frameMetadata.length); - combinedData.set(frameMetadata, 0); - combinedData.set(data, frameMetadata.length); + // Create IV using RTP timestamp from metadata (synchronized between peers) + const iv = makeIV(encodedFrame); - const params: AesGcmParams = { name: 'AES-GCM', iv }; + // Ensure IV is properly typed + const ivArray = new Uint8Array(iv); + const params: AesGcmParams = { name: 'AES-GCM', iv: ivArray }; - let result: ArrayBuffer; - if (mode === 'encrypt') { - result = await crypto.subtle.encrypt(params, key, combinedData); + // COMPROMISE: Encrypt most of the frame while preserving minimal codec compatibility + // This prevents most visual leakage while maintaining decodability + let headerSize = 0; + let payloadData: Uint8Array; + + if (data.length > 20) { + // For video frames, preserve first 8 bytes for better codec compatibility + // This includes frame type, keyframe info, and basic header structure + headerSize = Math.min(8, Math.floor(data.length / 10)); + payloadData = data.slice(headerSize); + } else { + // For small frames (likely audio), encrypt everything + payloadData = data; + } + + // Encrypt the payload data + const payloadBuffer = new ArrayBuffer(payloadData.byteLength); + new Uint8Array(payloadBuffer).set(payloadData); + + let encryptedPayload: ArrayBuffer; + if (isEncrypting) { + encryptedPayload = await crypto.subtle.encrypt(params, key, payloadBuffer); } else { - result = await crypto.subtle.decrypt(params, key, combinedData); - - // Verify frame metadata on decryption - const decryptedData = new Uint8Array(result); - const metadataLength = frameMetadata.length; - const extractedMetadata = decryptedData.slice(0, metadataLength); - const extractedData = decryptedData.slice(metadataLength); - try { - const metadata = JSON.parse(new TextDecoder().decode(extractedMetadata)); - if (metadata.frameNumber !== frameCounter - 1) { - throw new Error("Frame sequence number mismatch"); - } - result = extractedData.buffer; - } catch (parseError) { - console.warn("Frame authentication failed:", parseError); - controller.error(new Error("Frame authentication failed")); - return; + encryptedPayload = await crypto.subtle.decrypt(params, key, payloadBuffer); + } catch (error) { + console.error(`E2EE ${mode} FAILED - dropping frame #${frameCount}, size: ${data.length}`, error); + return; // Drop the frame } } - encodedFrame.data = new Uint8Array(result); + // Reconstruct frame: minimal headers + encrypted payload + const encryptedArray = new Uint8Array(encryptedPayload); + const result = new Uint8Array(headerSize + encryptedArray.length); + + if (headerSize > 0) { + result.set(data.slice(0, headerSize), 0); // Copy minimal headers + result.set(encryptedArray, headerSize); // Add encrypted payload + } else { + result.set(encryptedArray, 0); + } + + + // CRITICAL: Video frames need ArrayBuffer, not Uint8Array + encodedFrame.data = result.buffer; controller.enqueue(encodedFrame); + } catch (e) { - console.error(`E2EE ${mode} failed:`, e); - controller.error(new Error(`E2EE ${mode} failed`)); + // FAIL SECURELY: Never send unencrypted frames + const data = new Uint8Array(encodedFrame.data); + console.error(`E2EE ${mode} FAILED - dropping frame #${frameCount}, size: ${data.length}`, e); + return; // Drop the frame completely } } diff --git a/frontend/src/core/calls/signaling.ts b/frontend/src/core/calls/signaling.ts index 831c6f1..b89d161 100644 --- a/frontend/src/core/calls/signaling.ts +++ b/frontend/src/core/calls/signaling.ts @@ -5,6 +5,8 @@ export interface CallState { receiveCall: (userId: number, username: string) => void; endCall: () => void; setCallSessionKeyHash: (sessionKeyHash: string) => void; + setRemoteVideoEnabled: (enabled: boolean) => void; + setRemoteScreenSharing: (enabled: boolean) => void; } export class CallSignalingHandler { @@ -52,6 +54,12 @@ export class CallSignalingHandler { case "call_session_key": this.handleCallSessionKey(data); break; + case "call_video_toggle": + this.handleVideoToggle(data); + break; + case "call_screen_share_toggle": + this.handleScreenShareToggle(data); + break; } } @@ -59,11 +67,11 @@ export class CallSignalingHandler { const { fromUserId, fromUsername } = data; const state = this.getState(); - // Show incoming call UI - state.receiveCall(fromUserId, fromUsername); - - // Handle incoming call in WebRTC service + // First, create the peer connection in WebRTC service await WebRTC.handleIncomingCall(fromUserId, fromUsername); + + // Then show incoming call UI + state.receiveCall(fromUserId, fromUsername); } private async handleCallAccept(data: any) { @@ -126,4 +134,36 @@ export class CallSignalingHandler { WebRTC.receiveWrappedSessionKey(message.fromUserId, data.wrappedSessionKey, sessionKeyHash); } } + + private handleVideoToggle(data: any) { + console.log("handleVideoToggle called with data:", data); + const state = this.getState(); + const { fromUserId, data: toggleData } = data; + + if (toggleData && typeof toggleData.enabled === "boolean" && fromUserId) { + console.log("Setting remote video enabled to:", toggleData.enabled); + // Update Zustand state (for UI) + state.setRemoteVideoEnabled(toggleData.enabled); + // Update WebRTC internal state (for track routing) + WebRTC.setRemoteVideoEnabled(fromUserId, toggleData.enabled); + } else { + console.warn("Invalid toggle data:", data); + } + } + + private handleScreenShareToggle(data: any) { + console.log("handleScreenShareToggle called with data:", data); + const state = this.getState(); + const { fromUserId, data: toggleData } = data; + + if (toggleData && typeof toggleData.enabled === "boolean" && fromUserId) { + console.log("Setting remote screen sharing to:", toggleData.enabled); + // Update Zustand state (for UI) + state.setRemoteScreenSharing(toggleData.enabled); + // Update WebRTC internal state (for track routing) + WebRTC.setRemoteScreenSharing(fromUserId, toggleData.enabled); + } else { + console.warn("Invalid toggle data:", data); + } + } } diff --git a/frontend/src/core/calls/webrtc.ts b/frontend/src/core/calls/webrtc.ts index 1424230..e18607c 100644 --- a/frontend/src/core/calls/webrtc.ts +++ b/frontend/src/core/calls/webrtc.ts @@ -1,33 +1,52 @@ import { getAuthHeaders } from "@/core/api/authApi"; import type { CallSignalingMessage, IceServersResponse } from "@/core/types"; import { request } from "@/core/websocket"; -import { wrapCallSessionKeyForRecipient, unwrapCallSessionKeyFromSender } from "./encryption"; +import { wrapCallSessionKeyForRecipient, unwrapCallSessionKeyFromSender, rotateCallSessionKey } from "./encryption"; import { fetchUserPublicKey } from "@/core/api/dmApi"; import { importAesGcmKey } from "@/utils/crypto/symmetric"; import E2EEWorker from "./e2eeWorker?worker"; -import { rotateCallSessionKey, createSharedSecretAndDeriveSessionKey } from "./encryption"; export interface WebRTCCall { peerConnection: RTCPeerConnection; localStream: MediaStream | null; remoteStream: MediaStream | null; + localVideoStream: MediaStream | null; + screenShareStream: MediaStream | null; isInitiator: boolean; remoteUserId: number; remoteUsername: string; isEnding?: boolean; isMuted?: boolean; + isLocalVideoEnabled: boolean; + isScreenSharing: boolean; + isRemoteScreenSharing: boolean; // Track remote screen share state from signaling + isRemoteVideoEnabled: boolean; // Track remote video state from signaling + isNegotiating?: boolean; // Insertable Streams E2EE sessionKey?: Uint8Array | null; sessionCryptoKey?: CryptoKey | null; sessionId: string; keyRotationTimer?: NodeJS.Timeout; lastKeyRotation?: number; + transformedSenders: Set; + transformedReceivers: Set; + // Track specific senders for proper routing when both video and screen share are active + videoSender?: RTCRtpSender | null; + screenShareSender?: RTCRtpSender | null; + // Track the number of video tracks received for each type + receivedVideoTrackCount: number; + receivedScreenShareTrackCount: number; } // Global state export let authToken: string | null = null; export let onCallStateChange: ((userId: number, state: string) => void) | null = null; export let onRemoteStream: ((userId: number, stream: MediaStream) => void) | null = null; +export let onLocalVideoStream: ((userId: number, stream: MediaStream | null) => void) | null = null; +export let onRemoteVideoStream: ((userId: number, stream: MediaStream | null) => void) | null = null; +export let onLocalScreenShare: ((userId: number, stream: MediaStream | null) => void) | null = null; +export let onRemoteScreenShare: ((userId: number, stream: MediaStream | null) => void) | null = null; +export let onScreenShareStateChange: ((userId: number, isSharing: boolean) => void) | null = null; const calls: Map = new Map(); export function setAuthToken(token: string) { @@ -42,6 +61,22 @@ export function setRemoteStreamHandler(handler: (userId: number, stream: MediaSt onRemoteStream = handler; } +export function setLocalVideoStreamHandler(handler: (userId: number, stream: MediaStream | null) => void) { + onLocalVideoStream = handler; +} + +export function setRemoteVideoStreamHandler(handler: (userId: number, stream: MediaStream | null) => void) { + onRemoteVideoStream = handler; +} + +export function setLocalScreenShareHandler(handler: (userId: number, stream: MediaStream | null) => void) { + onLocalScreenShare = handler; +} + +export function setRemoteScreenShareHandler(handler: (userId: number, stream: MediaStream | null) => void) { + onRemoteScreenShare = handler; +} + async function sendSignalingMessage(message: CallSignalingMessage) { if (!authToken) { throw new Error("No auth token available"); @@ -74,7 +109,6 @@ async function getIceServers(): Promise { if (response.ok) { const data = await response.json() as IceServersResponse; - console.log("Received ICE servers:", data.iceServers); return data.iceServers || []; } else { console.warn("Failed to fetch ICE servers:", response.status, response.statusText); @@ -99,13 +133,23 @@ async function createPeerConnection(userId: number): Promise peerConnection, localStream: null, remoteStream: null, + localVideoStream: null, + screenShareStream: null, isInitiator: false, remoteUserId: userId, remoteUsername: "", isMuted: false, + isLocalVideoEnabled: false, + isScreenSharing: false, + isRemoteScreenSharing: false, + isRemoteVideoEnabled: false, sessionKey: null, sessionCryptoKey: null, - sessionId: crypto.randomUUID() + sessionId: crypto.randomUUID(), + transformedSenders: new Set(), + transformedReceivers: new Set(), + receivedVideoTrackCount: 0, + receivedScreenShareTrackCount: 0 }; calls.set(userId, call); @@ -117,8 +161,6 @@ async function createPeerConnection(userId: number): Promise // Add ICE candidate event listener for debugging and sending peerConnection.addEventListener("icecandidate", async (event) => { if (event.candidate) { - console.log("Local ICE candidate:", event.candidate.candidate); - // Send ICE candidate to remote peer try { await sendSignalingMessage({ @@ -134,8 +176,6 @@ async function createPeerConnection(userId: number): Promise } catch (error) { console.error("Failed to send ICE candidate:", error); } - } else { - console.log("ICE gathering complete"); } }); @@ -144,17 +184,156 @@ async function createPeerConnection(userId: number): Promise }); peerConnection.addEventListener("signalingstatechange", () => { - // Signaling state changed + console.log("Signaling state changed:", peerConnection.signalingState); + }); + + // Handle renegotiation when tracks are added/removed + peerConnection.addEventListener("negotiationneeded", async () => { + try { + const call = calls.get(userId); + if (!call) { + return; + } + + // Prevent multiple simultaneous negotiations + if (call.isNegotiating) { + return; + } + + // Skip if we're in "stable" state and haven't finished the initial handshake + if (peerConnection.signalingState !== "stable") { + return; + } + + call.isNegotiating = true; + + const offer = await peerConnection.createOffer(); + await peerConnection.setLocalDescription(offer); + + console.log("Sending renegotiation offer to user", userId); + await sendSignalingMessage({ + type: "call_offer", + fromUserId: 0, + toUserId: userId, + data: offer + }); + + call.isNegotiating = false; + } catch (error) { + console.error("Failed to handle negotiation:", error); + const call = calls.get(userId); + if (call) { + call.isNegotiating = false; + } + } }); // Handle remote stream - peerConnection.addEventListener("track", (event) => { + peerConnection.addEventListener("track", async (event) => { + console.log("Received track:", event.track.kind, "from user", userId, "stream ID:", event.streams[0]?.id); + const [remoteStream] = event.streams; const call = calls.get(userId); - if (call) { - call.remoteStream = remoteStream; - if (onRemoteStream) { - onRemoteStream(userId, remoteStream); + if (call && remoteStream) { + const track = event.track; + + // Apply E2EE transform to all tracks - video now uses header-preserving encryption + if (call.sessionKey && window.RTCRtpScriptTransform) { + try { + const receiver = call.peerConnection.getReceivers().find(r => r.track === track); + if (receiver && !call.transformedReceivers.has(receiver)) { + const key = await importAesGcmKey(call.sessionKey); + console.log(`Applying decrypt transform to newly received ${track.kind} track:`); + console.log("- sessionId:", call.sessionId); + console.log("- sessionKey (first 8 bytes):", Array.from(new Uint8Array(call.sessionKey).slice(0, 8))); + // @ts-ignore + receiver.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'decrypt', sessionId: call.sessionId }); + call.transformedReceivers.add(receiver); + console.log(`Decrypt transform applied successfully to ${track.kind} track`); + } + } catch (error) { + console.error("Failed to apply E2EE to received track:", error); + } + } else { + console.log(`Skipping decrypt transform for ${track.kind} track - session key not available or RTCRtpScriptTransform not supported`); + console.log("Session key exists:", !!call.sessionKey); + console.log("RTCRtpScriptTransform available:", !!window.RTCRtpScriptTransform); + } + + // Determine stream type based on track kind and signaling state + console.log(`Track received: kind=${track.kind}, isRemoteScreenSharing=${call.isRemoteScreenSharing}, isRemoteVideoEnabled=${call.isRemoteVideoEnabled}`); + + if (track.kind === "video") { + const receiver = call.peerConnection.getReceivers().find(r => r.track === track); + const transceiver = receiver ? call.peerConnection.getTransceivers().find(t => t.receiver === receiver) : null; + + console.log("Video track transceiver mid:", transceiver?.mid); + console.log("Video sender mid:", call.videoSender ? call.peerConnection.getTransceivers().find(t => t.sender === call.videoSender)?.mid : "none"); + console.log("Screen share sender mid:", call.screenShareSender ? call.peerConnection.getTransceivers().find(t => t.sender === call.screenShareSender)?.mid : "none"); + + let isScreenShare = false; + let isVideo = false; + + if (call.isRemoteScreenSharing && call.isRemoteVideoEnabled) { + // Both active - route based on which one we haven't received yet + console.log("Both features active - routing based on received track counts"); + console.log("Received video tracks:", call.receivedVideoTrackCount); + console.log("Received screen share tracks:", call.receivedScreenShareTrackCount); + + // Simple logic: if we haven't received video yet, this is video + // if we haven't received screen share yet, this is screen share + if (call.receivedVideoTrackCount === 0) { + isVideo = true; + call.receivedVideoTrackCount++; + console.log("Routing as video (first video track)"); + } else if (call.receivedScreenShareTrackCount === 0) { + isScreenShare = true; + call.receivedScreenShareTrackCount++; + console.log("Routing as screen share (first screen share track)"); + } else { + // Both already received - this shouldn't happen, log warning + console.warn("Both tracks already received, but got another video track!"); + console.warn("This might be a track replacement, routing as screen share by default"); + isScreenShare = true; + } + } else if (call.isRemoteScreenSharing) { + console.log("Only screen share active"); + isScreenShare = true; + call.receivedScreenShareTrackCount++; + } else if (call.isRemoteVideoEnabled) { + console.log("Only video active"); + isVideo = true; + call.receivedVideoTrackCount++; + } else { + console.log("Neither video nor screen share active - this shouldn't happen!"); + } + + console.log("Routing decision: isScreenShare:", isScreenShare, "isVideo:", isVideo); + + if (isScreenShare) { + console.log("Detected screen share track, notifying handler"); + if (onRemoteScreenShare) { + onRemoteScreenShare(userId, remoteStream); + } else { + console.warn("onRemoteScreenShare handler not set!"); + } + } else if (isVideo) { + console.log("Detected video track, notifying handler"); + if (onRemoteVideoStream) { + onRemoteVideoStream(userId, remoteStream); + } else { + console.warn("onRemoteVideoStream handler not set!"); + } + } + } else if (track.kind === "audio") { + console.log("Detected audio track, notifying handler"); + // Handle remote audio (existing behavior) + call.remoteStream = remoteStream; + if (onRemoteStream) { + onRemoteStream(userId, remoteStream); + } else { + console.warn("onRemoteStream handler not set!"); + } } } }); @@ -168,10 +347,10 @@ async function createPeerConnection(userId: number): Promise onCallStateChange(userId, peerConnection.connectionState); } - // Clean up if connection failed or closed + // Clean up only on permanent failures + // Don't end on "disconnected" - ICE can recover from temporary disconnections if (peerConnection.connectionState === "failed" || - peerConnection.connectionState === "closed" || - peerConnection.connectionState === "disconnected") { + peerConnection.connectionState === "closed") { // Only send end call message if we're not already cleaning up const call = calls.get(userId); if (call && !call.isEnding) { @@ -270,28 +449,55 @@ export async function sendWrappedCallSessionKey(userId: number, sessionKey: Uint async function applyE2EETransforms(call: WebRTCCall): Promise { try { // @ts-ignore - if (!call.sessionKey || !window.RTCRtpScriptTransform) return; + if (!call.sessionKey || !window.RTCRtpScriptTransform) { + console.log("Skipping E2EE transforms - session key or RTCRtpScriptTransform not available"); + return; + } + + console.log("Applying E2EE transforms for call", call.remoteUserId); const key = await importAesGcmKey(call.sessionKey); call.sessionCryptoKey = key; - const receiver = call.peerConnection.getReceivers().find(r => r.track && r.track.kind === 'audio'); - if (receiver) { - // @ts-ignore - receiver.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'decrypt' }); + + // Apply to receivers that don't already have transforms + const receivers = call.peerConnection.getReceivers(); + for (const receiver of receivers) { + if (receiver.track && !call.transformedReceivers.has(receiver)) { + console.log(`Applying decrypt transform to ${receiver.track.kind} receiver`); + // @ts-ignore + receiver.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'decrypt', sessionId: call.sessionId }); + call.transformedReceivers.add(receiver); + } } - const sender = call.peerConnection.getSenders().find(s => s.track && s.track.kind === 'audio'); - if (sender) { - // @ts-ignore - sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'encrypt' }); + + // Apply to senders that don't already have transforms + const senders = call.peerConnection.getSenders(); + for (const sender of senders) { + if (sender.track && !call.transformedSenders.has(sender)) { + console.log(`Applying encrypt transform to ${sender.track.kind} sender`); + // @ts-ignore + sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'encrypt', sessionId: call.sessionId }); + call.transformedSenders.add(sender); + } } - } catch {} + } catch (error) { + console.error("Failed to apply E2EE transforms:", error); + } } export async function setSessionKey(userId: number, keyBytes: Uint8Array): Promise { const call = calls.get(userId); - if (!call) return; + if (!call) { + console.error("setSessionKey: No call found for user", userId); + return; + } + + console.log("setSessionKey: Setting session key for user", userId); call.sessionKey = keyBytes; call.lastKeyRotation = Date.now(); + + console.log("setSessionKey: Applying E2EE transforms..."); await applyE2EETransforms(call); + console.log("setSessionKey: E2EE transforms applied successfully"); // Start key rotation timer (rotate every 10 minutes for long calls) if (call.keyRotationTimer) { @@ -337,23 +543,29 @@ async function rotateSessionKey(userId: number): Promise { export async function receiveWrappedSessionKey(fromUserId: number, wrappedPayload: any, sessionKeyHash?: string): Promise { if (!authToken) return; try { + console.log("receiveWrappedSessionKey called for user", fromUserId, "hash:", sessionKeyHash); const senderPublicKey = await fetchUserPublicKey(fromUserId, authToken); - if (!senderPublicKey) return; - if (!wrappedPayload || !sessionKeyHash) return; + if (!senderPublicKey) { + console.error("Failed to get sender public key"); + return; + } + if (!wrappedPayload || !sessionKeyHash) { + console.error("Missing wrapped payload or session key hash"); + return; + } - // First unwrap the session key from the encrypted payload (for validation) - await unwrapCallSessionKeyFromSender(senderPublicKey, { + console.log("Unwrapping session key..."); + // Unwrap the session key from the encrypted payload + const unwrappedSessionKey = await unwrapCallSessionKeyFromSender(senderPublicKey, { salt: wrappedPayload.salt, iv2: wrappedPayload.iv2, wrapped: wrappedPayload.wrapped }); - // Then derive the actual session key from the shared secret - const call = calls.get(fromUserId); - const isInitiator = call?.isInitiator ?? false; - const derivedSessionKey = await createSharedSecretAndDeriveSessionKey(senderPublicKey, sessionKeyHash, isInitiator); - - await setSessionKey(fromUserId, derivedSessionKey.key); + console.log("Session key unwrapped successfully, setting it..."); + // Use the unwrapped session key directly (both sides should have the same key) + await setSessionKey(fromUserId, unwrappedSessionKey); + console.log("Session key set successfully for user", fromUserId); } catch (e) { console.error("Failed to unwrap session key:", e); } @@ -369,10 +581,12 @@ export async function acceptCall(userId: number): Promise { if (!call) return false; } - // Get user media and attach - const localStream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); - call.localStream = localStream; - localStream.getTracks().forEach(track => call!.peerConnection.addTrack(track, localStream)); + // Get user media and attach (only if not already attached) + if (!call.localStream) { + const localStream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + call.localStream = localStream; + localStream.getTracks().forEach(track => call!.peerConnection.addTrack(track, localStream)); + } // Notify initiator that callee accepted; initiator will generate offer await sendSignalingMessage({ @@ -440,6 +654,10 @@ export async function onRemoteAccepted(userId: number): Promise { } try { + // Small delay to ensure remote peer finishes processing the accept + // This prevents race conditions where our offer arrives before they're ready + await new Promise(resolve => setTimeout(resolve, 100)); + // Create offer const offer = await call.peerConnection.createOffer(); await call.peerConnection.setLocalDescription(offer); @@ -457,41 +675,93 @@ export async function onRemoteAccepted(userId: number): Promise { } } -async function createE2EETransform(sessionKey: NonNullable, peerConnection: RTCPeerConnection, sessionId?: string): Promise { +async function createE2EETransform(call: WebRTCCall, sessionKey: Uint8Array, sessionId?: string): Promise { try { - if (sessionKey && window.RTCRtpScriptTransform) { - const key = await importAesGcmKey(sessionKey); - const receiver = peerConnection.getReceivers().find(r => r.track && r.track.kind === 'audio'); - if (receiver) { + // @ts-ignore + if (!sessionKey || !window.RTCRtpScriptTransform) { + console.log("Skipping E2EE transform in createE2EETransform - not supported or no session key"); + return; + } + + console.log("Creating E2EE transform with sessionId:", sessionId); + const key = await importAesGcmKey(sessionKey); + + // Apply to receivers that don't already have transforms + const receivers = call.peerConnection.getReceivers(); + for (const receiver of receivers) { + if (receiver.track && !call.transformedReceivers.has(receiver)) { + console.log(`Applying decrypt transform to ${receiver.track.kind} in createE2EETransform`); + // @ts-ignore receiver.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'decrypt', sessionId }); + call.transformedReceivers.add(receiver); } - const sender = peerConnection.getSenders().find(s => s.track && s.track.kind === 'audio'); - if (sender) { + } + + // Apply to senders that don't already have transforms + const senders = call.peerConnection.getSenders(); + for (const sender of senders) { + if (sender.track && !call.transformedSenders.has(sender)) { + console.log(`Applying encrypt transform to ${sender.track.kind} in createE2EETransform`); + // @ts-ignore sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: 'encrypt', sessionId }); + call.transformedSenders.add(sender); } } } catch (error) { console.error("Failed to create E2EE transform:", error); + // Fail securely - throw to prevent call from continuing without E2EE throw error; } } export async function handleCallOffer(userId: number, offer: RTCSessionDescriptionInit): Promise { - const call = calls.get(userId); + let call = calls.get(userId); + + console.log("handleCallOffer called for user", userId, "offer type:", offer.type); + + // Handle race condition - offer might arrive before peer connection is created if (!call) { - throw new Error("No call found for offer"); + console.log("No call found for offer, creating peer connection (race condition handling)"); + await createPeerConnection(userId); + call = calls.get(userId); + if (!call) { + throw new Error("Failed to create call for offer"); + } } try { + // Ensure we have local media before answering + if (!call.localStream) { + console.log("Getting local media for answer"); + try { + const localStream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + call.localStream = localStream; + localStream.getTracks().forEach(track => call!.peerConnection.addTrack(track, localStream)); + } catch (mediaError) { + console.error("Failed to get local media:", mediaError); + // Continue anyway - we can still receive media + } + } + + console.log("Setting remote description with", offer.sdp?.split('\n').filter(l => l.includes('m=')).join(', ')); + // Set remote description await call.peerConnection.setRemoteDescription(offer); + console.log("Creating answer..."); // Create answer const answer = await call.peerConnection.createAnswer(); await call.peerConnection.setLocalDescription(answer); - // Attach transforms on callee side if session key set and insertable streams supported - await createE2EETransform(call.sessionKey!, call.peerConnection, call.sessionId); + console.log("Answer created with", answer.sdp?.split('\n').filter(l => l.includes('m=')).join(', ')); + + // Attach transforms on callee side if session key is available + // If not available yet, setSessionKey will apply them when it arrives + if (call.sessionKey) { + await createE2EETransform(call, call.sessionKey, call.sessionId); + } else { + console.log("Session key not yet available in handleCallOffer - will apply transforms when key arrives"); + } // Send answer to remote peer await sendSignalingMessage({ @@ -500,6 +770,8 @@ export async function handleCallOffer(userId: number, offer: RTCSessionDescripti toUserId: userId, data: answer }); + + console.log("Answer sent successfully"); } catch (error) { console.error("Failed to handle offer:", error); throw error; @@ -513,9 +785,47 @@ export async function handleCallAnswer(userId: number, answer: RTCSessionDescrip } try { + console.log("handleCallAnswer - setting remote description"); + console.log("Answer SDP media lines:", answer.sdp?.split('\n').filter(l => l.includes('m=')).join(', ')); + await call.peerConnection.setRemoteDescription(answer); - // After signaling completes, attach transforms on initiator side if supported - await createE2EETransform(call.sessionKey!, call.peerConnection, call.sessionId); + + console.log("Remote description set successfully"); + console.log("Current receivers after answer:", call.peerConnection.getReceivers().map(r => r.track?.kind)); + console.log("Current senders after answer:", call.peerConnection.getSenders().map(s => s.track?.kind)); + + // Reset negotiating flag + call.isNegotiating = false; + + // Attach transforms on initiator side if session key is available + // If not available yet, setSessionKey will apply them when it arrives + if (call.sessionKey) { + await createE2EETransform(call, call.sessionKey, call.sessionId); + } else { + console.log("Session key not yet available in handleCallAnswer - will apply transforms when key arrives"); + } + + // Check if there are new receivers with tracks that haven't been notified yet + // This handles the case where tracks exist but the track event hasn't fired yet + const receivers = call.peerConnection.getReceivers(); + for (const receiver of receivers) { + if (receiver.track) { + const track = receiver.track; + console.log("Checking receiver track:", track.kind); + + // Find the stream for this track + const transceiver = call.peerConnection.getTransceivers().find(t => t.receiver === receiver); + if (transceiver && transceiver.receiver.track) { + // Manually trigger stream handlers for tracks that didn't fire events + if (track.kind === "video" && onRemoteVideoStream) { + // Create a MediaStream from the track + const stream = new MediaStream([track]); + console.log("Manually notifying remote video stream handler"); + onRemoteVideoStream(userId, stream); + } + } + } + } } catch (error) { console.error("Failed to handle answer:", error); throw error; @@ -523,9 +833,10 @@ export async function handleCallAnswer(userId: number, answer: RTCSessionDescrip } export async function handleIceCandidate(userId: number, candidate: RTCIceCandidateInit): Promise { - const call = calls.get(userId); + let call = calls.get(userId); if (!call) { - console.warn("No call found for ICE candidate from user", userId); + console.warn("No call found for ICE candidate from user", userId, "- might arrive before connection setup"); + // Don't create peer connection here - ICE candidates will be gathered again after connection is established return; } @@ -616,6 +927,300 @@ export function getCall(userId: number): WebRTCCall | undefined { return calls.get(userId); } +export async function toggleVideo(userId: number): Promise { + const call = calls.get(userId); + if (!call) { + return false; + } + + if (!call.isLocalVideoEnabled) { + // Enable video + try { + const videoStream = await navigator.mediaDevices.getUserMedia({ + video: true, + audio: false + }); + + call.localVideoStream = videoStream; + call.isLocalVideoEnabled = true; + + // Add video track to peer connection + const videoTrack = videoStream.getVideoTracks()[0]; + const sender = call.peerConnection.addTrack(videoTrack, videoStream); + call.videoSender = sender; + + console.log("Video track added successfully"); + console.log("Current senders:", call.peerConnection.getSenders().map(s => s.track?.kind)); + console.log("Current transceivers:", call.peerConnection.getTransceivers().map(t => ({ + sender: t.sender.track?.kind, + receiver: t.receiver.track?.kind, + direction: t.direction, + mid: t.mid + }))); + + // Apply E2EE transform with header-preserving encryption for video + if (call.sessionKey && window.RTCRtpScriptTransform) { + try { + const key = await importAesGcmKey(call.sessionKey); + console.log("Applying E2EE to video sender with header preservation"); + const sender = call.peerConnection.getSenders().find(s => s.track === videoTrack); + if (sender && !call.transformedSenders.has(sender)) { + sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId }); + call.transformedSenders.add(sender); + console.log("E2EE applied to video sender successfully"); + } + } catch (error) { + console.error("Failed to apply E2EE to video:", error); + throw error; // Fail securely + } + } + + // Notify local video stream handler + if (onLocalVideoStream) { + console.log("Calling onLocalVideoStream handler with stream:", videoStream); + onLocalVideoStream(userId, videoStream); + } else { + console.warn("onLocalVideoStream handler is not set!"); + } + + // Send signaling message to notify remote peer + console.log("Sending call_video_toggle with enabled: true"); + await sendSignalingMessage({ + type: "call_video_toggle", + fromUserId: 0, + toUserId: userId, + data: { enabled: true } + }); + + console.log("Video enabled successfully"); + return true; + } catch (error) { + console.error("Failed to enable video:", error); + return false; + } + } else { + // Disable video + if (call.localVideoStream) { + call.localVideoStream.getTracks().forEach(track => { + track.stop(); + // Remove track from peer connection + const senders = call.peerConnection.getSenders(); + const videoSender = senders.find(s => s.track === track); + if (videoSender) { + call.peerConnection.removeTrack(videoSender); + call.transformedSenders.delete(videoSender); + // Clear sender reference + if (call.videoSender === videoSender) { + call.videoSender = null; + } + } + }); + call.localVideoStream = null; + } + + call.isLocalVideoEnabled = false; + + // Notify local video stream handler + if (onLocalVideoStream) { + onLocalVideoStream(userId, null); + } + + // Send signaling message to notify remote peer + await sendSignalingMessage({ + type: "call_video_toggle", + fromUserId: 0, + toUserId: userId, + data: { enabled: false } + }); + + return false; + } +} + +export async function toggleScreenShare(userId: number): Promise { + const call = calls.get(userId); + if (!call) { + return false; + } + + if (!call.isScreenSharing) { + // Enable screen sharing + try { + const screenStream = await navigator.mediaDevices.getDisplayMedia({ + video: { + width: { ideal: 1920, max: 3840 }, + height: { ideal: 1080, max: 2160 }, + frameRate: { ideal: 60, max: 60 } + }, + audio: false + }); + + // Set a special ID to identify screen share streams + try { + Object.defineProperty(screenStream, "id", { + value: `screen-${crypto.randomUUID()}`, + writable: false, + configurable: true + }); + console.log("Set screen share stream ID to:", screenStream.id); + } catch (e) { + console.warn("Failed to set custom stream ID, using default:", screenStream.id); + } + + call.screenShareStream = screenStream; + call.isScreenSharing = true; + + // Add screen share track to peer connection + const videoTrack = screenStream.getVideoTracks()[0]; + + // Handle when user stops sharing via browser UI + videoTrack.addEventListener("ended", async () => { + console.log("Screen share track ended by browser controls"); + + // Clean up screen share state + if (call.screenShareStream) { + call.screenShareStream.getTracks().forEach(t => t.stop()); + call.screenShareStream = null; + } + call.isScreenSharing = false; + + // Remove screen share track from peer connection + const senders = call.peerConnection.getSenders(); + const screenSender = senders.find(sender => + sender.track && sender.track.kind === 'video' && + sender.track.readyState === 'ended' && + call.transformedSenders.has(sender) + ); + + if (screenSender) { + await call.peerConnection.removeTrack(screenSender); + call.transformedSenders.delete(screenSender); + } + + // Notify local screen share handler + if (onLocalScreenShare) { + onLocalScreenShare(userId, null); + } + + // Notify state change handler + if (onScreenShareStateChange) { + onScreenShareStateChange(userId, false); + } + + // Send signaling message to remote peer + await sendSignalingMessage({ + type: "call_screen_share_toggle", + fromUserId: 0, // Will be set by server + toUserId: userId, + data: { enabled: false } + }); + }); + + // Send signaling message FIRST to notify remote peer before adding track + // This ensures the receiver knows it's screen share before the track arrives + console.log("Sending screen share toggle BEFORE adding track"); + await sendSignalingMessage({ + type: "call_screen_share_toggle", + fromUserId: 0, + toUserId: userId, + data: { enabled: true } + }); + + // Small delay to ensure signaling message is processed before track arrives + await new Promise(resolve => setTimeout(resolve, 100)); + + const sender = call.peerConnection.addTrack(videoTrack, screenStream); + call.screenShareSender = sender; + + console.log("Screen share track added, immediately applying E2EE transform"); + + // CRITICAL: Apply E2EE transform IMMEDIATELY after track is added + if (call.sessionKey && window.RTCRtpScriptTransform) { + try { + const key = await importAesGcmKey(call.sessionKey); + console.log("Applying E2EE to screen share sender:"); + console.log("- sessionId:", call.sessionId); + console.log("- sessionKey (first 8 bytes):", Array.from(new Uint8Array(call.sessionKey).slice(0, 8))); + console.log("Available senders:", call.peerConnection.getSenders().map(s => ({ + track: s.track?.kind, + id: s.track?.id + }))); + console.log("Looking for screen share track:", videoTrack.id); + + const sender = call.peerConnection.getSenders().find(s => s.track === videoTrack); + console.log("Found screen share sender:", !!sender); + + if (sender && !call.transformedSenders.has(sender)) { + sender.transform = new RTCRtpScriptTransform(new E2EEWorker(), { key, mode: "encrypt", sessionId: call.sessionId }); + call.transformedSenders.add(sender); + console.log("E2EE applied to screen share sender successfully"); + } else { + console.log("Screen share sender not found or already transformed"); + } + } catch (error) { + console.error("Failed to apply E2EE to screen share:", error); + throw error; // Fail securely + } + } else { + console.log("Skipping E2EE for screen share - session key not available yet"); + console.log("Session key exists:", !!call.sessionKey); + console.log("RTCRtpScriptTransform available:", !!window.RTCRtpScriptTransform); + } + + // Let browser handle screen share settings naturally + // Avoid applying constraints that might cause glitches + + // Notify local screen share handler + if (onLocalScreenShare) { + onLocalScreenShare(userId, screenStream); + } + + console.log("Screen share setup complete - using existing session key:", call.sessionId); + + return true; + } catch (error) { + console.error("Failed to enable screen sharing:", error); + return false; + } + } else { + // Disable screen sharing + if (call.screenShareStream) { + call.screenShareStream.getTracks().forEach(track => { + track.stop(); + // Remove track from peer connection + const senders = call.peerConnection.getSenders(); + const screenSender = senders.find(s => s.track === track); + if (screenSender) { + call.peerConnection.removeTrack(screenSender); + call.transformedSenders.delete(screenSender); + // Clear sender reference + if (call.screenShareSender === screenSender) { + call.screenShareSender = null; + } + } + }); + call.screenShareStream = null; + } + + call.isScreenSharing = false; + + // Notify local screen share handler + if (onLocalScreenShare) { + onLocalScreenShare(userId, null); + } + + // Send signaling message to notify remote peer + await sendSignalingMessage({ + type: "call_screen_share_toggle", + fromUserId: 0, + toUserId: userId, + data: { enabled: false } + }); + + return false; + } +} + export function cleanupCall(userId: number): void { const call = calls.get(userId); if (call) { @@ -634,10 +1239,56 @@ export function cleanupCall(userId: number): void { call.localStream.getTracks().forEach(track => track.stop()); } + // Stop local video stream + if (call.localVideoStream) { + call.localVideoStream.getTracks().forEach(track => track.stop()); + } + + // Stop screen share stream + if (call.screenShareStream) { + call.screenShareStream.getTracks().forEach(track => track.stop()); + } + calls.delete(userId); } } +/** + * Update the remote video enabled state (called when receiving signaling) + */ +export function setRemoteVideoEnabled(userId: number, enabled: boolean): void { + const call = calls.get(userId); + if (call) { + console.log(`Setting remote video enabled to ${enabled} for user ${userId}`); + call.isRemoteVideoEnabled = enabled; + // Reset counter when feature is disabled + if (!enabled) { + call.receivedVideoTrackCount = 0; + console.log("Reset video track counter"); + } + } +} + +/** + * Update the remote screen sharing state (called when receiving signaling) + */ +export function setRemoteScreenSharing(userId: number, enabled: boolean): void { + const call = calls.get(userId); + if (call) { + console.log(`Setting remote screen sharing to ${enabled} for user ${userId}`); + call.isRemoteScreenSharing = enabled; + // Reset counter when feature is disabled + if (!enabled) { + call.receivedScreenShareTrackCount = 0; + console.log("Reset screen share track counter"); + } + } +} + +export function setScreenShareStateChangeHandler(handler: ((userId: number, isSharing: boolean) => void) | null): void { + onScreenShareStateChange = handler; +} + export function cleanup(): void { // Clean up all calls for (const userId of calls.keys()) { diff --git a/frontend/src/core/types.d.ts b/frontend/src/core/types.d.ts index 7e34bfb..d57fc3b 100644 --- a/frontend/src/core/types.d.ts +++ b/frontend/src/core/types.d.ts @@ -442,7 +442,7 @@ export interface CallInvite extends CallSignalingData { timestamp: string; } -export type CallSignalingDataType = "call_offer" | "call_answer" | "call_ice_candidate" | "call_end" | "call_invite" | "call_accept" | "call_reject" | "call_session_key" | "call_signaling"; +export type CallSignalingDataType = "call_offer" | "call_answer" | "call_ice_candidate" | "call_end" | "call_invite" | "call_accept" | "call_reject" | "call_session_key" | "call_signaling" | "call_video_toggle" | "call_screen_share_toggle"; export interface CallSignalingMessage extends WebSocketMessage { type: CallSignalingDataType; @@ -450,4 +450,18 @@ export interface CallSignalingMessage extends WebSocketMessage { toUserId: number; sessionKeyHash?: string; data: any; +} + +export interface CallVideoToggleMessage extends CallSignalingMessage { + type: "call_video_toggle"; + data: { + enabled: boolean; + }; +} + +export interface CallScreenShareToggleMessage extends CallSignalingMessage { + type: "call_screen_share_toggle"; + data: { + enabled: boolean; + }; } \ No newline at end of file diff --git a/frontend/src/pages/chat/css/_callWindow.scss b/frontend/src/pages/chat/css/_callWindow.scss index 465faaa..b54e2ab 100644 --- a/frontend/src/pages/chat/css/_callWindow.scss +++ b/frontend/src/pages/chat/css/_callWindow.scss @@ -1,40 +1,175 @@ @use "../../../css/material" as *; +@use "sass:color"; .call-window { $transition: cubic-bezier(0.4, 0, 0.2, 1); position: fixed; z-index: 1000; - width: 300px; - background-color: rgba($color-dark-surface, 0.85); - backdrop-filter: blur(20px); - -webkit-backdrop-filter: blur(20px); - border: 1px solid rgba($color-dark-outline, 0.3); - border-radius: 12px; - box-shadow: 0 8px 32px rgba(0, 0, 0, 0.3); - cursor: grab; - transition: opacity 0.3s $transition, transform 0.3s $transition; - transform-origin: center; + display: flex; + flex-direction: column; user-select: none; - - &.visible { - opacity: 1; - transform: scale(1); - } - - &.hidden { - opacity: 0; - transform: scale(0.8) translateY(-20px); - pointer-events: none; - } - + + // Base transition for all properties + transition: all 0.4s $transition; + + // Disable all transitions while dragging for immediate feedback &.dragging { - cursor: grabbing; + transition: none !important; + } + + // Maximized mode (fullscreen) + &.maximized { + top: 0; + left: 0; + right: 0; + bottom: 0; + width: 100vw; + height: 100vh; + background-color: rgba($color-dark-surface, 0.98); + backdrop-filter: blur(40px); + -webkit-backdrop-filter: blur(40px); + border: none; + border-radius: 0; + cursor: default; + + &.visible { + opacity: 1; + transform: translateY(0); + } + + &.hidden { + opacity: 0; + transform: translateY(-100%); // Slide up when ending in fullscreen + pointer-events: none; + } + + .call-content { + padding: 32px; + gap: 24px; + + .video-tiles-grid { + gap: 24px; + min-height: 400px; + } + } + + .call-header { + .window-controls { + .window-control-btn { + color: $color-dark-on-surface-variant; + + &:hover { + color: $color-dark-on-surface; + background: rgba($color-dark-on-surface, 0.08); + } + } + } + } + } + + // Minimized mode (PiP) + &.minimized { + width: 400px; + height: 300px; + background-color: rgba($color-dark-surface, 0.95); + backdrop-filter: blur(20px); + -webkit-backdrop-filter: blur(20px); + border: 2px solid rgba($color-dark-outline, 0.4); + border-radius: 16px; + box-shadow: 0 12px 40px rgba(0, 0, 0, 0.5); + cursor: grab; + + &.visible { + opacity: 1; + transform: scale(1); + } + + &.hidden { + opacity: 0 !important; + transform: scale(0.7) !important; // Scale down and fade when ending in PiP + pointer-events: none; + transition: opacity 0.4s cubic-bezier(0.4, 0, 0.2, 1), transform 0.4s cubic-bezier(0.4, 0, 0.2, 1) !important; + } + + &.dragging { + cursor: grabbing; + box-shadow: 0 20px 60px rgba(0, 0, 0, 0.7); + } + + .call-header { + padding: 12px; + min-height: auto; + + .call-header-info { + .username { + font-size: 14px; + } + + .status { + font-size: 12px; + } + } + + .window-controls { + .window-control-btn { + color: $color-dark-primary; + + &:hover { + color: color.adjust($color-dark-primary, $lightness: 10%); + background: rgba($color-dark-primary, 0.1); + } + } + } + } + + .call-content { + padding: 8px; + gap: 8px; + + .video-tiles-grid { + gap: 8px; + min-height: 120px; + } + + .video-tile { + .tile-label { + font-size: 10px; + padding: 2px 6px; + } + + .video-placeholder { + .placeholder-avatar { + width: 40px; + height: 40px; + } + + .placeholder-username { + font-size: 12px; + } + } + } + + .screen-share-tile { + .screen-share-video { + object-fit: contain; + } + } + } + + .call-controls { + padding: 8px; + gap: 8px; + + mdui-button-icon { + --mdui-comp-icon-button-size: 36px; + } + } } // Dynamic gradients based on call state &.gradient-calling { - border-color: rgba(255, 193, 7, 0.4); + border-color: rgba(255, 193, 7, 0.5); &::before { content: ''; @@ -44,16 +179,17 @@ right: 0; bottom: 0; background: linear-gradient(135deg, - rgba(255, 193, 7, 0.15) 0%, - rgba(255, 152, 0, 0.15) 50%, - rgba(255, 193, 7, 0.15) 100%); - border-radius: 12px; + rgba(255, 193, 7, 0.12) 0%, + rgba(255, 152, 0, 0.12) 50%, + rgba(255, 193, 7, 0.12) 100%); + border-radius: inherit; animation: pulse-gradient 2s ease-in-out infinite; + pointer-events: none; } } &.gradient-connecting { - border-color: rgba(33, 150, 243, 0.4); + border-color: rgba(33, 150, 243, 0.5); &::before { content: ''; @@ -63,16 +199,17 @@ right: 0; bottom: 0; background: linear-gradient(135deg, - rgba(33, 150, 243, 0.15) 0%, - rgba(63, 81, 181, 0.15) 50%, - rgba(33, 150, 243, 0.15) 100%); - border-radius: 12px; + rgba(33, 150, 243, 0.12) 0%, + rgba(63, 81, 181, 0.12) 50%, + rgba(33, 150, 243, 0.12) 100%); + border-radius: inherit; animation: connecting-gradient 1.5s ease-in-out infinite; + pointer-events: none; } } &.gradient-active { - border-color: rgba(76, 175, 80, 0.4); + border-color: rgba(76, 175, 80, 0.5); &::before { content: ''; @@ -82,11 +219,12 @@ right: 0; bottom: 0; background: linear-gradient(135deg, - rgba(76, 175, 80, 0.15) 0%, - rgba(56, 142, 60, 0.15) 50%, - rgba(76, 175, 80, 0.15) 100%); - border-radius: 12px; + rgba(76, 175, 80, 0.12) 0%, + rgba(56, 142, 60, 0.12) 50%, + rgba(76, 175, 80, 0.12) 100%); + border-radius: inherit; animation: active-gradient 3s ease-in-out infinite; + pointer-events: none; } } @@ -96,223 +234,293 @@ } .call-header { - padding: 16px; + padding: 20px; border-bottom: 1px solid $color-dark-outline-variant; position: relative; z-index: 1; + display: flex; + align-items: center; + justify-content: space-between; + flex-shrink: 0; .window-controls { + display: flex; + gap: 8px; position: absolute; - top: 8px; - right: 8px; - z-index: 2; + left: 16px; + top: 50%; + transform: translateY(-50%); - .minimize-btn { + .window-control-btn { + transition: all 0.2s ease; + } + } + + .call-header-info { + flex: 1; + display: flex; + flex-direction: column; + align-items: center; + gap: 6px; + text-align: center; + + .username { + margin: 0; + font-size: 20px; + font-weight: 600; + color: $color-dark-on-surface; + } + + .status { + margin: 0; + font-size: 14px; color: $color-dark-on-surface-variant; - transition: color 0.2s ease; + font-weight: 500; + } - &:hover { - color: $color-dark-on-surface; + .encryption-emojis { + display: flex; + justify-content: center; + gap: 10px; + margin-top: 8px; + + .encryption-emoji { + font-size: 24px; + display: inline-block; + animation: emoji-pulse 2s ease-in-out infinite; + + &:nth-child(1) { animation-delay: 0s; } + &:nth-child(2) { animation-delay: 0.2s; } + &:nth-child(3) { animation-delay: 0.4s; } + &:nth-child(4) { animation-delay: 0.6s; } + } + } + } +} + +.call-content { + padding: 24px; + display: flex; + flex-direction: row; + gap: 20px; + position: relative; + z-index: 1; + flex: 1; + overflow: hidden; + + // Reduce padding when screen share is active to maximize space + &.with-screen-share { + padding: 12px; + gap: 16px; + } + + // Default layout (no screen share): tiles in grid + .screen-share-area { + display: none; + } + + .video-tiles-sidebar { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 20px; + flex: 1; + transition: all 0.4s cubic-bezier(0.4, 0, 0.2, 1); + } + + // Layout with screen share: screen share on left, video tiles on right + &.with-screen-share { + .screen-share-area { + display: flex; + align-items: center; + justify-content: center; + flex: 4; // Give 4x more space than sidebar + min-width: 0; + max-width: 80vw; // Limit screen share to 80% of viewport width + transition: all 0.4s cubic-bezier(0.4, 0, 0.2, 1); + position: relative; + // Ensure no padding/margin that could create blank space + padding: 0; + margin: 0; + } + + .video-tiles-sidebar { + display: flex; + flex-direction: column; + grid-template-columns: none; + flex: 1; // Give 1x space (4:1 ratio = 80:20) + min-width: 250px; // Ensure minimum 300px width + max-width: 20%; // Cap at 20% of container + flex-shrink: 0; + gap: 16px; + overflow-y: auto; + overflow-x: hidden; + padding: 5px; + + // Custom scrollbar + &::-webkit-scrollbar { + width: 6px; + } + + &::-webkit-scrollbar-track { + background: rgba($color-dark-surface-variant, 0.3); + border-radius: 3px; + } + + &::-webkit-scrollbar-thumb { + background: rgba($color-dark-primary, 0.5); + border-radius: 3px; + + &:hover { + background: rgba($color-dark-primary, 0.7); + } + } + + .video-tile { + min-height: 168px; } } } - .user-info-centered { + .video-tile { + position: relative; + background: rgba($color-dark-surface-variant, 0.5); + border-radius: 16px; + overflow: hidden; display: flex; - flex-direction: column; align-items: center; - gap: 12px; - text-align: center; + justify-content: center; + border: 1px solid rgba($color-dark-outline, 0.3); + transition: all 0.3s ease; - .avatar { - width: 64px; - height: 64px; - border-radius: 50%; - object-fit: cover; - box-shadow: 0 0 20px rgba(33, 150, 243, 0.3); - transition: box-shadow 0.3s ease; - position: relative; - z-index: 1; - - // Colored shadow based on call state - .gradient-calling & { - box-shadow: 0 0 20px rgba(255, 193, 7, 0.4); - } - - .gradient-connecting & { - box-shadow: 0 0 20px rgba(33, 150, 243, 0.4); - } - - .gradient-active & { - box-shadow: 0 0 20px rgba(76, 175, 80, 0.4); - } + &:hover { + border-color: rgba($color-dark-outline, 0.5); + transform: scale(1.02); } - .user-details { - .username { - margin: 0; + .video-element { + width: 100%; + height: 100%; + object-fit: cover; + } + + .video-placeholder { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 16px; + width: 100%; + height: 100%; + background: rgba($color-dark-surface-variant, 0.6); + + .placeholder-avatar { + width: 100px; + height: 100px; + border-radius: 50%; + object-fit: cover; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.4); + } + + .placeholder-username { font-size: 18px; font-weight: 600; color: $color-dark-on-surface; } + } - .status { - margin: 4px 0 0 0; - font-size: 14px; - color: $color-dark-on-surface-variant; - font-weight: 500; + .tile-label { + position: absolute; + bottom: 12px; + left: 12px; + padding: 6px 12px; + background: rgba(0, 0, 0, 0.75); + color: white; + font-size: 13px; + font-weight: 600; + border-radius: 8px; + backdrop-filter: blur(10px); + -webkit-backdrop-filter: blur(10px); + } + + &.local-video { + .video-element { + transform: scaleX(-1); // Mirror local video } + } + } - .encryption-emojis { - display: flex; - justify-content: center; - gap: 8px; - margin-top: 8px; + .screen-share-tile { + position: relative; + border: 2px solid rgba($color-dark-primary, 0.6); + border-radius: 12px; + overflow: hidden; + background: rgba(0, 0, 0, 0.95); + display: inline-block; + transition: all 0.4s cubic-bezier(0.4, 0, 0.2, 1); + // Constrain to available space but size to content + max-width: 100%; + max-height: 100%; - .encryption-emoji { - font-size: 20px; - display: inline-block; - animation: emoji-pulse 2s ease-in-out infinite; - - &:nth-child(1) { animation-delay: 0s; } - &:nth-child(2) { animation-delay: 0.2s; } - &:nth-child(3) { animation-delay: 0.4s; } - &:nth-child(4) { animation-delay: 0.6s; } - } - } + .screen-share-video { + width: auto; + height: auto; + max-width: 100%; + max-height: 100%; + display: block; + object-fit: contain; + } + + .tile-label { + position: absolute; + bottom: 12px; + left: 50%; + transform: translateX(-50%); + font-size: 14px; + padding: 8px 16px; + background: rgba($color-dark-primary, 0.9); + color: $color-dark-on-primary; + border-radius: 8px; + backdrop-filter: blur(10px); + -webkit-backdrop-filter: blur(10px); + pointer-events: none; + z-index: 1; } } } .call-controls { - padding: 16px; + padding: 20px; display: flex; justify-content: center; - gap: 12px; -} - -// Minimized call bar -.minimized-call-bar { - position: fixed; - bottom: 0; - left: 0; - right: 0; - height: 60px; - background: rgba($color-dark-surface, 0.85); - backdrop-filter: blur(20px); - -webkit-backdrop-filter: blur(20px); - border-top: 1px solid rgba($color-dark-outline, 0.3); - display: flex; - align-items: center; - justify-content: space-between; - padding: 0 16px; - z-index: 999; - cursor: pointer; - transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); + gap: 16px; + border-top: 1px solid $color-dark-outline-variant; position: relative; - overflow: hidden; + z-index: 1; + flex-shrink: 0; - &::before { - content: ''; - position: absolute; - top: 0; - left: -100%; - width: 100%; - height: 100%; - background: linear-gradient(90deg, - transparent 0%, - rgba(255, 255, 255, 0.1) 50%, - transparent 100%); - animation: shimmer 2s ease-in-out infinite; - } + mdui-button-icon { + transition: all 0.2s ease; - .call-info { - display: flex; - align-items: center; - gap: 12px; + &[icon="call_end"] { + background: rgba(244, 67, 54, 0.2); + color: rgb(244, 67, 54); - .avatar { - width: 40px; - height: 40px; - border-radius: 50%; - object-fit: cover; - } - - .user-details { - display: flex; - flex-direction: column; - gap: 2px; - - .username { - font-size: 14px; - font-weight: 600; - color: $color-dark-on-surface; - } - - .status { - font-size: 12px; - color: $color-dark-on-surface-variant; + &:hover { + background: rgba(244, 67, 54, 0.35); + transform: scale(1.1); } } - } - .call-actions { - display: flex; - gap: 8px; - } - - // Dynamic gradients for minimized bar - &.gradient-calling { - border-color: rgba(255, 193, 7, 0.4); - - &::before { - content: ''; - position: absolute; - top: 0; - left: -100%; - width: 100%; - height: 100%; - background: linear-gradient(90deg, - transparent 0%, - rgba(255, 193, 7, 0.1) 50%, - transparent 100%); - animation: shimmer-calling 2s ease-in-out infinite; - } - } - - &.gradient-connecting { - border-color: rgba(33, 150, 243, 0.4); - - &::before { - content: ''; - position: absolute; - top: 0; - left: -100%; - width: 100%; - height: 100%; - background: linear-gradient(90deg, - transparent 0%, - rgba(33, 150, 243, 0.1) 50%, - transparent 100%); - animation: shimmer-connecting 1.5s ease-in-out infinite; - } - } - - &.gradient-active { - border-color: rgba(76, 175, 80, 0.4); - - &::before { - content: ''; - position: absolute; - top: 0; - left: -100%; - width: 100%; - height: 100%; - background: linear-gradient(90deg, - transparent 0%, - rgba(76, 175, 80, 0.1) 50%, - transparent 100%); - animation: shimmer-active 3s ease-in-out infinite; + &[icon="videocam"], + &[icon="videocam_off"], + &[icon="screen_share"], + &[icon="stop_screen_share"], + &[icon="mic"], + &[icon="mic_off"] { + &:hover { + background: rgba($color-dark-primary, 0.15); + transform: scale(1.1); + } } } } @@ -330,68 +538,28 @@ // Animations @keyframes pulse-gradient { 0%, 100% { - opacity: 0.3; - transform: scale(1); + opacity: 0.4; } 50% { - opacity: 0.6; - transform: scale(1.02); + opacity: 0.7; } } @keyframes connecting-gradient { 0%, 100% { - opacity: 0.2; - transform: translateX(0); + opacity: 0.3; } 50% { - opacity: 0.4; - transform: translateX(10px); + opacity: 0.6; } } @keyframes active-gradient { 0%, 100% { - opacity: 0.15; + opacity: 0.2; } 50% { - opacity: 0.25; - } -} - -@keyframes shimmer { - 0% { - left: -100%; - } - 100% { - left: 100%; - } -} - -@keyframes shimmer-calling { - 0% { - left: -100%; - } - 100% { - left: 100%; - } -} - -@keyframes shimmer-connecting { - 0% { - left: -100%; - } - 100% { - left: 100%; - } -} - -@keyframes shimmer-active { - 0% { - left: -100%; - } - 100% { - left: 100%; + opacity: 0.4; } } @@ -401,7 +569,7 @@ opacity: 0.8; } 50% { - transform: scale(1.2); + transform: scale(1.15); opacity: 1; } -} \ No newline at end of file +} diff --git a/frontend/src/pages/chat/hooks/useAudioCall.ts b/frontend/src/pages/chat/hooks/useAudioCall.ts deleted file mode 100644 index 8a83532..0000000 --- a/frontend/src/pages/chat/hooks/useAudioCall.ts +++ /dev/null @@ -1,233 +0,0 @@ -import { useAppState } from "@/pages/chat/state"; -import * as WebRTC from "@/core/calls/webrtc"; -import { CallSignalingHandler } from "@/core/calls/signaling"; -import { setCallSignalingHandler } from "@/core/websocket"; -import { generateCallSessionKey, generateCallEmojis, createCallSessionKeyFromHash } from "@/core/calls/encryption"; -import { createRef, useEffect } from "react"; - -// Global audio ref shared across all instances -let globalRemoteAudioRef = createRef(); - -export default function useAudioCall() { - const { chat, startCall, endCall, setCallStatus, toggleMute, setCallEncryption, setCallSessionKeyHash, user } = useAppState(); - const remoteAudioRef = globalRemoteAudioRef; - - useEffect(() => { - if (user.authToken) { - WebRTC.setAuthToken(user.authToken); - } - - // Initialize call signaling handler - const signalingHandler = new CallSignalingHandler(() => ({ - receiveCall: (userId: number, username: string) => { - // Use the receiveCall function from state - const state = useAppState.getState(); - state.receiveCall(userId, username); - }, - endCall, - setCallSessionKeyHash - })); - setCallSignalingHandler(signalingHandler); - - // Set up call state change handler - WebRTC.setCallStateChangeHandler((userId: number, state: string) => { - const call = chat.call; - if (call.remoteUserId === userId) { - switch (state) { - case "connecting": - setCallStatus("connecting"); - break; - case "connected": - setCallStatus("active"); - break; - case "disconnected": - case "failed": - case "closed": - endCall(); - break; - } - } - }); - - // Set up remote stream handler - WebRTC.setRemoteStreamHandler((_userId: number, stream: MediaStream) => { - if (!remoteAudioRef.current) { - return; - } - const el = remoteAudioRef.current; - try { - el.srcObject = stream; - el.muted = false; - el.volume = 1.0; - el.autoplay = true; - - // Handle audio events - el.addEventListener("error", () => { - console.warn("[AUDIO] element error", (el.error?.message) || el.error); - }); - - el.play().catch(() => { - // Try to play after user interaction if autoplay is blocked - const playAfterInteraction = () => { - el.play().catch(() => {}); - document.removeEventListener('click', playAfterInteraction); - document.removeEventListener('touchstart', playAfterInteraction); - }; - document.addEventListener('click', playAfterInteraction); - document.addEventListener('touchstart', playAfterInteraction); - }); - } catch (e) { - console.warn("failed to attach remote stream:", e); - } - }); - - return () => { - WebRTC.cleanup(); - setCallSignalingHandler(null); - }; - }, [user.authToken, chat.call.remoteUserId, setCallStatus, endCall, startCall]); - - // Watch for session key hash changes and generate emojis - useEffect(() => { - if (chat.call.sessionKeyHash && chat.call.encryptionEmojis.length === 0) { - const emojis = generateCallEmojis(chat.call.sessionKeyHash); - setCallEncryption(chat.call.sessionKeyHash, emojis); - } - }, [chat.call.sessionKeyHash, chat.call.encryptionEmojis.length, setCallEncryption]); - - async function requestAudioPermissions(): Promise { - try { - const stream = await navigator.mediaDevices.getUserMedia({ - audio: true, - video: false - }); - - // Stop the stream immediately as we just needed permission - stream.getTracks().forEach(track => track.stop()); - return true; - } catch (error) { - console.error("Failed to get audio permissions:", error); - return false; - } - }; - - async function initiateCall(userId: number, username: string) { - const hasPermission = await requestAudioPermissions(); - - if (!hasPermission) { - console.log("Audio permission denied"); - return; - } - - - let sessionKey; - try { - // Generate call session key and emojis - sessionKey = await generateCallSessionKey(); - const emojis = generateCallEmojis(sessionKey.hash); - - // Start the call in state - startCall(userId, username); - setCallStatus("calling"); - setCallEncryption(sessionKey.hash, emojis); - } catch (error) { - console.error("Failed to generate call encryption:", error); - endCall(); - return; - } - - // Initiate WebRTC call - const success = await WebRTC.initiateCall(userId, username); - - if (success && sessionKey) { - // Send session key hash to the receiver for visual verification - await WebRTC.sendCallSessionKey(userId, sessionKey.hash); - // Also wrap and send the actual session key for E2EE media - await WebRTC.sendWrappedCallSessionKey(userId, sessionKey.key, sessionKey.hash); - } else { - endCall(); - } - }; - - async function acceptCall() { - if (!chat.call.remoteUserId) { - return; - } - - setCallStatus("connecting"); - const success = await WebRTC.acceptCall(chat.call.remoteUserId); - - if (!success) { - endCall(); - } - }; - - async function rejectCall() { - if (!chat.call.remoteUserId) { - return; - } - - await WebRTC.rejectCall(chat.call.remoteUserId); - endCall(); - }; - - async function handleEndCall() { - if (chat.call.remoteUserId) { - await WebRTC.endCall(chat.call.remoteUserId); - } - endCall(); - }; - - function handleToggleMute() { - if (chat.call.remoteUserId) { - const isMuted = WebRTC.toggleMute(chat.call.remoteUserId); - // Update mute state in store - if (isMuted !== chat.call.isMuted) { - toggleMute(); - } - } - }; - - async function handleIncomingCall(userId: number, username: string) { - // Don't generate session key here - wait for it from the caller - await WebRTC.handleIncomingCall(userId, username); - }; - - async function handleCallOffer(userId: number, offer: RTCSessionDescriptionInit) { - await WebRTC.handleCallOffer(userId, offer); - }; - - async function handleCallAnswer(userId: number, answer: RTCSessionDescriptionInit) { - await WebRTC.handleCallAnswer(userId, answer); - }; - - async function handleIceCandidate(userId: number, candidate: RTCIceCandidateInit) { - await WebRTC.handleIceCandidate(userId, candidate); - }; - - async function handleCallSessionKey(sessionKeyHash: string) { - try { - // Create session key from the hash provided by the caller - const sessionKey = await createCallSessionKeyFromHash(sessionKeyHash); - const emojis = generateCallEmojis(sessionKey.hash); - setCallEncryption(sessionKey.hash, emojis); - } catch (error) { - console.error("Failed to create call session key from hash:", error); - } - }; - - return { - call: chat.call, - initiateCall, - acceptCall, - rejectCall, - endCall: handleEndCall, - toggleMute: handleToggleMute, - handleIncomingCall, - handleCallOffer, - handleCallAnswer, - handleIceCandidate, - handleCallSessionKey, - remoteAudioRef - }; -} diff --git a/frontend/src/pages/chat/hooks/useCall.ts b/frontend/src/pages/chat/hooks/useCall.ts new file mode 100644 index 0000000..0b6757e --- /dev/null +++ b/frontend/src/pages/chat/hooks/useCall.ts @@ -0,0 +1,392 @@ +import { useAppState } from "@/pages/chat/state"; +import * as WebRTC from "@/core/calls/webrtc"; +import { CallSignalingHandler } from "@/core/calls/signaling"; +import { setCallSignalingHandler } from "@/core/websocket"; +import { generateCallSessionKey, generateCallEmojis } from "@/core/calls/encryption"; +import { createRef, useEffect } from "react"; + +// Global refs shared across all instances +let globalRemoteAudioRef = createRef(); +let globalLocalVideoRef = createRef(); +let globalRemoteVideoRef = createRef(); +let globalLocalScreenShareRef = createRef(); +let globalRemoteScreenShareRef = createRef(); + +export default function useCall() { + const { + chat, + startCall, + endCall, + setCallStatus, + toggleMute, + toggleVideo, + toggleScreenShare, + setCallEncryption, + setCallSessionKeyHash, + setRemoteVideoEnabled, + setRemoteScreenSharing, + user + } = useAppState(); + + const remoteAudioRef = globalRemoteAudioRef; + const localVideoRef = globalLocalVideoRef; + const remoteVideoRef = globalRemoteVideoRef; + const localScreenShareRef = globalLocalScreenShareRef; + const remoteScreenShareRef = globalRemoteScreenShareRef; + + useEffect(() => { + if (user.authToken) { + WebRTC.setAuthToken(user.authToken); + } + + // Initialize call signaling handler + const signalingHandler = new CallSignalingHandler(() => ({ + receiveCall: (userId: number, username: string) => { + // Use the receiveCall function from state + const state = useAppState.getState(); + state.receiveCall(userId, username); + }, + endCall, + setCallSessionKeyHash, + setRemoteVideoEnabled, + setRemoteScreenSharing + })); + setCallSignalingHandler(signalingHandler); + + // Set up call state change handler + WebRTC.setCallStateChangeHandler((userId: number, state: string) => { + const call = chat.call; + if (call.remoteUserId === userId) { + switch (state) { + case "connecting": + setCallStatus("connecting"); + break; + case "connected": + setCallStatus("active"); + break; + case "disconnected": + case "failed": + case "closed": + endCall(); + break; + } + } + }); + + // Set up remote audio stream handler + WebRTC.setRemoteStreamHandler((_userId: number, stream: MediaStream) => { + if (!remoteAudioRef.current) { + return; + } + const el = remoteAudioRef.current; + try { + el.srcObject = stream; + el.muted = false; + el.volume = 1.0; + el.autoplay = true; + + // Handle audio events + el.addEventListener("error", () => { + console.warn("[AUDIO] element error", (el.error?.message) || el.error); + }); + + el.play().catch(() => { + // Try to play after user interaction if autoplay is blocked + const playAfterInteraction = () => { + el.play().catch(() => {}); + document.removeEventListener("click", playAfterInteraction); + document.removeEventListener("touchstart", playAfterInteraction); + }; + document.addEventListener("click", playAfterInteraction); + document.addEventListener("touchstart", playAfterInteraction); + }); + } catch (e) { + console.warn("failed to attach remote stream:", e); + } + }); + + // Set up local video stream handler + WebRTC.setLocalVideoStreamHandler((_userId: number, stream: MediaStream | null) => { + console.log("Local video stream handler called, stream:", stream, "ref exists:", !!localVideoRef.current); + if (!localVideoRef.current) { + console.warn("Local video ref not available yet"); + return; + } + const el = localVideoRef.current; + try { + el.srcObject = stream; + el.muted = true; // Always mute local video to avoid feedback + el.autoplay = true; + if (stream) { + console.log("Playing local video stream"); + el.play().catch((err) => { + console.error("Failed to play local video:", err); + }); + } + } catch (e) { + console.warn("failed to attach local video stream:", e); + } + }); + + // Set up remote video stream handler + WebRTC.setRemoteVideoStreamHandler((_userId: number, stream: MediaStream | null) => { + console.log("Remote video stream handler called, stream:", stream, "ref exists:", !!remoteVideoRef.current); + if (!remoteVideoRef.current) { + console.warn("Remote video ref not available yet"); + return; + } + const el = remoteVideoRef.current; + try { + el.srcObject = stream; + el.muted = false; + el.autoplay = true; + if (stream) { + console.log("Playing remote video stream"); + el.play().catch((err) => { + console.error("Failed to play remote video:", err); + const playAfterInteraction = () => { + el.play().catch(() => {}); + document.removeEventListener("click", playAfterInteraction); + document.removeEventListener("touchstart", playAfterInteraction); + }; + document.addEventListener("click", playAfterInteraction); + document.addEventListener("touchstart", playAfterInteraction); + }); + } + } catch (e) { + console.warn("failed to attach remote video stream:", e); + } + }); + + // Set up local screen share handler + WebRTC.setLocalScreenShareHandler((_userId: number, stream: MediaStream | null) => { + console.log("Local screen share handler called, stream:", stream, "ref exists:", !!localScreenShareRef.current); + if (!localScreenShareRef.current) { + console.warn("Local screen share ref not available yet"); + return; + } + const el = localScreenShareRef.current; + try { + el.srcObject = stream; + el.muted = true; + el.autoplay = true; + if (stream) { + console.log("Playing local screen share"); + el.play().catch((err) => { + console.error("Failed to play local screen share:", err); + }); + } + } catch (e) { + console.warn("failed to attach local screen share stream:", e); + } + }); + + // Set up remote screen share handler + WebRTC.setRemoteScreenShareHandler((_userId: number, stream: MediaStream | null) => { + console.log("Remote screen share handler called, stream:", stream, "ref exists:", !!remoteScreenShareRef.current); + if (!remoteScreenShareRef.current) { + console.warn("Remote screen share ref not available yet"); + return; + } + const el = remoteScreenShareRef.current; + try { + el.srcObject = stream; + el.muted = false; + el.autoplay = true; + if (stream) { + console.log("Playing remote screen share"); + el.play().catch((err) => { + console.error("Failed to play remote screen share:", err); + const playAfterInteraction = () => { + el.play().catch(() => {}); + document.removeEventListener("click", playAfterInteraction); + document.removeEventListener("touchstart", playAfterInteraction); + }; + document.addEventListener("click", playAfterInteraction); + document.addEventListener("touchstart", playAfterInteraction); + }); + } + } catch (e) { + console.warn("failed to attach remote screen share stream:", e); + } + }); + + return () => { + WebRTC.cleanup(); + setCallSignalingHandler(null); + }; + }, [user.authToken, chat.call.remoteUserId, setCallStatus, endCall, startCall, setRemoteVideoEnabled, setRemoteScreenSharing]); + + // Watch for session key hash changes and generate emojis + useEffect(() => { + if (chat.call.sessionKeyHash && chat.call.encryptionEmojis.length === 0) { + const emojis = generateCallEmojis(chat.call.sessionKeyHash); + setCallEncryption(chat.call.sessionKeyHash, emojis); + } + }, [chat.call.sessionKeyHash, chat.call.encryptionEmojis.length, setCallEncryption]); + + async function requestAudioPermissions(): Promise { + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: true, + video: false + }); + + // Stop the stream immediately as we just needed permission + stream.getTracks().forEach(track => track.stop()); + return true; + } catch (error) { + console.error("Failed to get audio permissions:", error); + return false; + } + } + + async function initiateCall(userId: number, username: string) { + const hasPermission = await requestAudioPermissions(); + + if (!hasPermission) { + console.log("Audio permission denied"); + return; + } + + let sessionKey; + try { + // Generate call session key and emojis + sessionKey = await generateCallSessionKey(); + const emojis = generateCallEmojis(sessionKey.hash); + + // Start the call in state + startCall(userId, username); + setCallStatus("calling"); + setCallEncryption(sessionKey.hash, emojis); + } catch (error) { + console.error("Failed to generate call encryption:", error); + endCall(); + return; + } + + // Initiate WebRTC call + const success = await WebRTC.initiateCall(userId, username); + + if (success && sessionKey) { + // Set the session key for ourselves (initiator) + await WebRTC.setSessionKey(userId, sessionKey.key); + + // Send session key hash to the receiver for visual verification + await WebRTC.sendCallSessionKey(userId, sessionKey.hash); + // Also wrap and send the actual session key for E2EE media + await WebRTC.sendWrappedCallSessionKey(userId, sessionKey.key, sessionKey.hash); + } else { + endCall(); + } + } + + async function acceptCall() { + if (!chat.call.remoteUserId) { + return; + } + + setCallStatus("connecting"); + const success = await WebRTC.acceptCall(chat.call.remoteUserId); + + if (!success) { + endCall(); + } + } + + async function rejectCall() { + if (!chat.call.remoteUserId) { + return; + } + + await WebRTC.rejectCall(chat.call.remoteUserId); + endCall(); + } + + async function handleEndCall() { + if (chat.call.remoteUserId) { + await WebRTC.endCall(chat.call.remoteUserId); + } + endCall(); + } + + function handleToggleMute() { + if (chat.call.remoteUserId) { + const isMuted = WebRTC.toggleMute(chat.call.remoteUserId); + // Update mute state in store + if (isMuted !== chat.call.isMuted) { + toggleMute(); + } + } + } + + async function handleToggleVideo() { + if (chat.call.remoteUserId) { + const isEnabled = await WebRTC.toggleVideo(chat.call.remoteUserId); + // Update video state in store + if (isEnabled !== chat.call.isVideoEnabled) { + toggleVideo(); + } + } + } + + async function handleToggleScreenShare() { + if (chat.call.remoteUserId) { + const isEnabled = await WebRTC.toggleScreenShare(chat.call.remoteUserId); + // Update screen share state in store + if (isEnabled !== chat.call.isSharingScreen) { + toggleScreenShare(); + } + } + } + + async function handleIncomingCall(userId: number, username: string) { + // Don't generate session key here - wait for it from the caller + await WebRTC.handleIncomingCall(userId, username); + } + + async function handleCallOffer(userId: number, offer: RTCSessionDescriptionInit) { + await WebRTC.handleCallOffer(userId, offer); + } + + async function handleCallAnswer(userId: number, answer: RTCSessionDescriptionInit) { + await WebRTC.handleCallAnswer(userId, answer); + } + + async function handleIceCandidate(userId: number, candidate: RTCIceCandidateInit) { + await WebRTC.handleIceCandidate(userId, candidate); + } + + async function handleCallSessionKey(sessionKeyHash: string) { + try { + // Just generate and display the emojis from the hash + // The actual session key will arrive via the wrapped key mechanism + const emojis = generateCallEmojis(sessionKeyHash); + setCallEncryption(sessionKeyHash, emojis); + } catch (error) { + console.error("Failed to generate call emojis from hash:", error); + } + } + + return { + call: chat.call, + initiateCall, + acceptCall, + rejectCall, + endCall: handleEndCall, + toggleMute: handleToggleMute, + toggleVideo: handleToggleVideo, + toggleScreenShare: handleToggleScreenShare, + handleIncomingCall, + handleCallOffer, + handleCallAnswer, + handleIceCandidate, + handleCallSessionKey, + remoteAudioRef, + localVideoRef, + remoteVideoRef, + localScreenShareRef, + remoteScreenShareRef + }; +} + diff --git a/frontend/src/pages/chat/state.ts b/frontend/src/pages/chat/state.ts index 1c947ac..cf16791 100644 --- a/frontend/src/pages/chat/state.ts +++ b/frontend/src/pages/chat/state.ts @@ -31,6 +31,10 @@ interface CallState { isMinimized: boolean; sessionKeyHash: string | null; encryptionEmojis: string[]; + isVideoEnabled: boolean; + isRemoteVideoEnabled: boolean; + isSharingScreen: boolean; + isRemoteScreenSharing: boolean; } interface ChatState { @@ -79,6 +83,11 @@ interface AppState { receiveCall: (userId: number, username: string) => void; setCallEncryption: (sessionKeyHash: string, encryptionEmojis: string[]) => void; setCallSessionKeyHash: (sessionKeyHash: string) => void; + toggleVideo: () => void; + toggleScreenShare: () => void; + setRemoteVideoEnabled: (enabled: boolean) => void; + setRemoteScreenSharing: (enabled: boolean) => void; + toggleCallMinimized: () => void; // User state user: UserState; @@ -116,7 +125,11 @@ export const useAppState = create((set, get) => ({ isInitiator: false, isMinimized: false, sessionKeyHash: null, - encryptionEmojis: [] + encryptionEmojis: [], + isVideoEnabled: false, + isRemoteVideoEnabled: false, + isSharingScreen: false, + isRemoteScreenSharing: false } }, addMessage: (message: Message) => set((state) => { @@ -412,7 +425,11 @@ export const useAppState = create((set, get) => ({ isInitiator: true, isMinimized: false, sessionKeyHash: null, - encryptionEmojis: [] + encryptionEmojis: [], + isVideoEnabled: false, + isRemoteVideoEnabled: false, + isSharingScreen: false, + isRemoteScreenSharing: false } } })), @@ -430,7 +447,11 @@ export const useAppState = create((set, get) => ({ isInitiator: false, isMinimized: false, sessionKeyHash: null, - encryptionEmojis: [] + encryptionEmojis: [], + isVideoEnabled: false, + isRemoteVideoEnabled: false, + isSharingScreen: false, + isRemoteScreenSharing: false } } })), @@ -479,7 +500,11 @@ export const useAppState = create((set, get) => ({ isInitiator: false, isMinimized: false, sessionKeyHash: null, - encryptionEmojis: [] + encryptionEmojis: [], + isVideoEnabled: false, + isRemoteVideoEnabled: false, + isSharingScreen: false, + isRemoteScreenSharing: false } } })), @@ -503,5 +528,54 @@ export const useAppState = create((set, get) => ({ sessionKeyHash } } + })), + + toggleVideo: () => set((state) => ({ + chat: { + ...state.chat, + call: { + ...state.chat.call, + isVideoEnabled: !state.chat.call.isVideoEnabled + } + } + })), + + toggleScreenShare: () => set((state) => ({ + chat: { + ...state.chat, + call: { + ...state.chat.call, + isSharingScreen: !state.chat.call.isSharingScreen + } + } + })), + + setRemoteVideoEnabled: (enabled: boolean) => set((state) => ({ + chat: { + ...state.chat, + call: { + ...state.chat.call, + isRemoteVideoEnabled: enabled + } + } + })), + + setRemoteScreenSharing: (enabled: boolean) => set((state) => ({ + chat: { + ...state.chat, + call: { + ...state.chat.call, + isRemoteScreenSharing: enabled + } + } + })), + toggleCallMinimized: () => set((state) => ({ + chat: { + ...state.chat, + call: { + ...state.chat.call, + isMinimized: !state.chat.call.isMinimized + } + } })) })); \ No newline at end of file diff --git a/frontend/src/pages/chat/ui/right/MessagePanelRenderer.tsx b/frontend/src/pages/chat/ui/right/MessagePanelRenderer.tsx index 48a0b68..b441f5b 100644 --- a/frontend/src/pages/chat/ui/right/MessagePanelRenderer.tsx +++ b/frontend/src/pages/chat/ui/right/MessagePanelRenderer.tsx @@ -8,7 +8,7 @@ import type { Message, WebSocketMessage } from "@/core/types"; import defaultAvatar from "@/images/default-avatar.png"; import AnimatedOpacity from "@/core/components/animations/AnimatedOpacity"; import type { DMPanel } from "./panels/DMPanel"; -import useAudioCall from "@/pages/chat/hooks/useAudioCall"; +import useCall from "@/pages/chat/hooks/useCall"; interface MessagePanelRendererProps { panel: MessagePanel | null; @@ -27,7 +27,7 @@ export function MessagePanelRenderer({ panel }: MessagePanelRendererProps) { const [editMessage, setEditMessage] = useState(null); const [editVisible, setEditVisible] = useState(Boolean(editMessage)); const [pendingAction, setPendingAction] = useState(null); - const { initiateCall } = useAudioCall(); + const { initiateCall } = useCall(); // Drag & drop diff --git a/frontend/src/pages/chat/ui/right/calls/CallWindow.tsx b/frontend/src/pages/chat/ui/right/calls/CallWindow.tsx index 41c059d..794db42 100644 --- a/frontend/src/pages/chat/ui/right/calls/CallWindow.tsx +++ b/frontend/src/pages/chat/ui/right/calls/CallWindow.tsx @@ -1,21 +1,36 @@ import { useState, useEffect } from "react"; -import { useAppState } from "@/pages/chat/state"; -import useAudioCall from "@/pages/chat/hooks/useAudioCall"; +import { useAppState, type CallStatus } from "@/pages/chat/state"; +import useCall from "@/pages/chat/hooks/useCall"; import defaultAvatar from "@/images/default-avatar.png"; +import { createPortal } from "react-dom"; +import { id } from "@/utils/utils"; export function CallWindow() { - const { chat, toggleMute, toggleCallMinimize } = useAppState(); + const { chat, toggleCallMinimize, user } = useAppState(); const { call } = chat; - const { acceptCall, rejectCall, remoteAudioRef, endCall } = useAudioCall(); - const [position, setPosition] = useState({ x: 100, y: 100 }); + const { + acceptCall, + rejectCall, + remoteAudioRef, + endCall, + toggleMute, + toggleVideo, + toggleScreenShare, + localVideoRef, + remoteVideoRef, + localScreenShareRef, + remoteScreenShareRef + } = useCall(); + const [pipPosition, setPipPosition] = useState({ x: window.innerWidth - 420, y: window.innerHeight - 320 }); const [isDragging, setIsDragging] = useState(false); - const [dragStart, setDragStart] = useState({ x: 0, y: 0 }); + const [dragOffset, setDragOffset] = useState({ x: 0, y: 0 }); const [callDuration, setCallDuration] = useState(0); const [isVisible, setIsVisible] = useState(false); const [shouldRender, setShouldRender] = useState(false); + const [wasMinimized, setWasMinimized] = useState(false); const [callData, setCallData] = useState<{ remoteUsername: string | null; - status: "calling" | "connecting" | "active" | "ended"; + status: CallStatus; isInitiator: boolean; isMuted: boolean; } | null>(null); @@ -53,40 +68,73 @@ export function CallWindow() { } }, [call.isActive, call.remoteUsername, call.status, call.isInitiator, call.isMuted]); + // Track minimized state for exit animation + useEffect(() => { + if (call.isActive) { + setWasMinimized(call.isMinimized); + } + }, [call.isActive, call.isMinimized]); + // Handle visibility animation with entrance and exit delays useEffect(() => { if (call.isActive) { setShouldRender(true); - if (!call.isMinimized) { - // Call is active and not minimized - show window with entrance animation - if (!isVisible) { - // Only animate in if not already visible (prevents animation on rapid calls) - requestAnimationFrame(() => { - setIsVisible(true); - }); - } - } else { - // Call is active but minimized - hide window but keep rendered - setIsVisible(false); - } + // Small delay to ensure DOM is ready, then trigger animation + requestAnimationFrame(() => { + requestAnimationFrame(() => { + setIsVisible(true); + }); + }); } else { if (shouldRender) { // Call ended - start exit animation + console.log(`Call ended - starting exit animation. Was minimized: ${wasMinimized}`); setIsVisible(false); // After animation completes, stop rendering const timer = setTimeout(() => { + console.log("Exit animation complete, removing window"); setShouldRender(false); setCallData(null); - }, 300); // Match the CSS transition duration + setWasMinimized(false); + }, 400); // Match the CSS transition duration return () => clearTimeout(timer); } else { // Call not active and not rendered - ensure clean state setShouldRender(false); setIsVisible(false); setCallData(null); + setWasMinimized(false); } } - }, [call.isActive, call.isMinimized, shouldRender, isVisible]); + }, [call.isActive, shouldRender, wasMinimized]); + + // Handle dragging for PiP mode + useEffect(() => { + const handleMouseMove = (e: MouseEvent) => { + if (isDragging && call.isMinimized) { + setPipPosition({ + x: e.clientX - dragOffset.x, + y: e.clientY - dragOffset.y + }); + } + }; + + const handleMouseUp = () => { + if (isDragging) { + setIsDragging(false); + } + }; + + if (isDragging) { + window.addEventListener("mousemove", handleMouseMove); + window.addEventListener("mouseup", handleMouseUp); + } + + return () => { + window.removeEventListener("mousemove", handleMouseMove); + window.removeEventListener("mouseup", handleMouseUp); + }; + }, [isDragging, call.isMinimized, dragOffset]); // Cleanup effect to reset state when component unmounts useEffect(() => { @@ -131,80 +179,153 @@ export function CallWindow() { return ( - <> -