FromChat/web
1
0
Fork 0
mirror of https://github.com/fromchat-messenger/web.git synced 2026-09-22 19:15:08 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix track distinguishing

Browse Source
This commit is contained in:
denis0001-dev
2025-10-11 22:01:14 +03:00
Unverified
parent 174ab8f1fa
commit 8d864f39ce
4 changed files with 152 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/core/calls/e2eeWorker.ts
+9 -3
View File
@@ -132,9 +132,9 @@ addEventListener("rtctransform", (event) => {
let payloadData: Uint8Array;
if (data.length > 20) {
// For video frames, preserve only the first 2 bytes (minimal codec sync)
// This is the absolute minimum needed for frame identification
headerSize = 2;
// For video frames, preserve first 8 bytes for better codec compatibility
// This includes frame type, keyframe info, and basic header structure
headerSize = Math.min(8, Math.floor(data.length / 10));
payloadData = data.slice(headerSize);
} else {
// For small frames (likely audio), encrypt everything
@@ -173,6 +173,12 @@ addEventListener("rtctransform", (event) => {
console.log(`E2EE ${mode} frame #${frameCount}: ${data.length} -> ${result.length} bytes (header: ${headerSize})`);
}
// For video frames, check if we need to force keyframes more frequently
// This helps prevent "stuck at first frame" issues with encrypted video
if (data.length > 10000 && frameCount > 0 && frameCount % 30 === 0) {
console.log(`Large video frame #${frameCount} - suggesting keyframe for stability`);
}
// CRITICAL: Video frames need ArrayBuffer, not Uint8Array
encodedFrame.data = result.buffer;
controller.enqueue(encodedFrame);
frontend/src/core/calls/webrtc.ts
+130 -13
View File
@@ -30,6 +30,12 @@ export interface WebRTCCall {
lastKeyRotation?: number;
transformedSenders: Set<RTCRtpSender>;
transformedReceivers: Set<RTCRtpReceiver>;
// Track specific senders for proper routing when both video and screen share are active
videoSender?: RTCRtpSender | null;
screenShareSender?: RTCRtpSender | null;
// Track the number of video tracks received for each type
receivedVideoTrackCount: number;
receivedScreenShareTrackCount: number;
}
// Global state
@@ -40,6 +46,7 @@ export let onLocalVideoStream: ((userId: number, stream: MediaStream | null) =>
export let onRemoteVideoStream: ((userId: number, stream: MediaStream | null) => void) | null = null;
export let onLocalScreenShare: ((userId: number, stream: MediaStream | null) => void) | null = null;
export let onRemoteScreenShare: ((userId: number, stream: MediaStream | null) => void) | null = null;
export let onScreenShareStateChange: ((userId: number, isSharing: boolean) => void) | null = null;
const calls: Map<number, WebRTCCall> = new Map();
export function setAuthToken(token: string) {
@@ -141,7 +148,9 @@ async function createPeerConnection(userId: number): Promise<RTCPeerConnection>
sessionCryptoKey: null,
sessionId: crypto.randomUUID(),
transformedSenders: new Set(),
transformedReceivers: new Set()
transformedReceivers: new Set(),
receivedVideoTrackCount: 0,
receivedScreenShareTrackCount: 0
};
calls.set(userId, call);
@@ -265,18 +274,61 @@ async function createPeerConnection(userId: number): Promise<RTCPeerConnection>
console.log(`Track received: kind=${track.kind}, isRemoteScreenSharing=${call.isRemoteScreenSharing}, isRemoteVideoEnabled=${call.isRemoteVideoEnabled}`);
if (track.kind === "video") {
// Prioritize screen share over regular video
// If remote is screen sharing, this video track is the screen share
if (call.isRemoteScreenSharing) {
console.log("Detected screen share track (based on signaling), notifying handler");
const receiver = call.peerConnection.getReceivers().find(r => r.track === track);
const transceiver = receiver ? call.peerConnection.getTransceivers().find(t => t.receiver === receiver) : null;
console.log("Video track transceiver mid:", transceiver?.mid);
console.log("Video sender mid:", call.videoSender ? call.peerConnection.getTransceivers().find(t => t.sender === call.videoSender)?.mid : "none");
console.log("Screen share sender mid:", call.screenShareSender ? call.peerConnection.getTransceivers().find(t => t.sender === call.screenShareSender)?.mid : "none");
let isScreenShare = false;
let isVideo = false;
if (call.isRemoteScreenSharing && call.isRemoteVideoEnabled) {
// Both active - route based on which one we haven't received yet
console.log("Both features active - routing based on received track counts");
console.log("Received video tracks:", call.receivedVideoTrackCount);
console.log("Received screen share tracks:", call.receivedScreenShareTrackCount);
// Simple logic: if we haven't received video yet, this is video
// if we haven't received screen share yet, this is screen share
if (call.receivedVideoTrackCount === 0) {
isVideo = true;
call.receivedVideoTrackCount++;
console.log("Routing as video (first video track)");
} else if (call.receivedScreenShareTrackCount === 0) {
isScreenShare = true;
call.receivedScreenShareTrackCount++;
console.log("Routing as screen share (first screen share track)");
} else {
// Both already received - this shouldn't happen, log warning
console.warn("Both tracks already received, but got another video track!");
console.warn("This might be a track replacement, routing as screen share by default");
isScreenShare = true;
}
} else if (call.isRemoteScreenSharing) {
console.log("Only screen share active");
isScreenShare = true;
call.receivedScreenShareTrackCount++;
} else if (call.isRemoteVideoEnabled) {
console.log("Only video active");
isVideo = true;
call.receivedVideoTrackCount++;
} else {
console.log("Neither video nor screen share active - this shouldn't happen!");
}
console.log("Routing decision: isScreenShare:", isScreenShare, "isVideo:", isVideo);
if (isScreenShare) {
console.log("Detected screen share track, notifying handler");
if (onRemoteScreenShare) {
onRemoteScreenShare(userId, remoteStream);
} else {
console.warn("onRemoteScreenShare handler not set!");
}
} else {
console.log("Detected video track (based on signaling), notifying handler");
// Handle remote video
} else if (isVideo) {
console.log("Detected video track, notifying handler");
if (onRemoteVideoStream) {
onRemoteVideoStream(userId, remoteStream);
} else {
@@ -904,14 +956,16 @@ export async function toggleVideo(userId: number): Promise<boolean> {
// Add video track to peer connection
const videoTrack = videoStream.getVideoTracks()[0];
call.peerConnection.addTrack(videoTrack, videoStream);
const sender = call.peerConnection.addTrack(videoTrack, videoStream);
call.videoSender = sender;
console.log("Video track added successfully");
console.log("Current senders:", call.peerConnection.getSenders().map(s => s.track?.kind));
console.log("Current transceivers:", call.peerConnection.getTransceivers().map(t => ({
sender: t.sender.track?.kind,
receiver: t.receiver.track?.kind,
direction: t.direction
direction: t.direction,
mid: t.mid
})));
// Apply E2EE transform with header-preserving encryption for video
@@ -964,6 +1018,11 @@ export async function toggleVideo(userId: number): Promise<boolean> {
const videoSender = senders.find(s => s.track === track);
if (videoSender) {
call.peerConnection.removeTrack(videoSender);
call.transformedSenders.delete(videoSender);
// Clear sender reference
if (call.videoSender === videoSender) {
call.videoSender = null;
}
}
});
call.localVideoStream = null;
@@ -1025,8 +1084,46 @@ export async function toggleScreenShare(userId: number): Promise<boolean> {
const videoTrack = screenStream.getVideoTracks()[0];
// Handle when user stops sharing via browser UI
videoTrack.addEventListener("ended", () => {
toggleScreenShare(userId);
videoTrack.addEventListener("ended", async () => {
console.log("Screen share track ended by browser controls");
// Clean up screen share state
if (call.screenShareStream) {
call.screenShareStream.getTracks().forEach(t => t.stop());
call.screenShareStream = null;
}
call.isScreenSharing = false;
// Remove screen share track from peer connection
const senders = call.peerConnection.getSenders();
const screenSender = senders.find(sender =>
sender.track && sender.track.kind === 'video' &&
sender.track.readyState === 'ended' &&
call.transformedSenders.has(sender)
);
if (screenSender) {
await call.peerConnection.removeTrack(screenSender);
call.transformedSenders.delete(screenSender);
}
// Notify local screen share handler
if (onLocalScreenShare) {
onLocalScreenShare(userId, null);
}
// Notify state change handler
if (onScreenShareStateChange) {
onScreenShareStateChange(userId, false);
}
// Send signaling message to remote peer
await sendSignalingMessage({
type: "call_screen_share_toggle",
fromUserId: 0, // Will be set by server
toUserId: userId,
data: { enabled: false }
});
});
// Send signaling message FIRST to notify remote peer before adding track
@@ -1042,7 +1139,8 @@ export async function toggleScreenShare(userId: number): Promise<boolean> {
// Small delay to ensure signaling message is processed before track arrives
await new Promise(resolve => setTimeout(resolve, 100));
call.peerConnection.addTrack(videoTrack, screenStream);
const sender = call.peerConnection.addTrack(videoTrack, screenStream);
call.screenShareSender = sender;
console.log("Screen share track added, immediately applying E2EE transform");
@@ -1104,6 +1202,11 @@ export async function toggleScreenShare(userId: number): Promise<boolean> {
const screenSender = senders.find(s => s.track === track);
if (screenSender) {
call.peerConnection.removeTrack(screenSender);
call.transformedSenders.delete(screenSender);
// Clear sender reference
if (call.screenShareSender === screenSender) {
call.screenShareSender = null;
}
}
});
call.screenShareStream = null;
@@ -1168,6 +1271,11 @@ export function setRemoteVideoEnabled(userId: number, enabled: boolean): void {
if (call) {
console.log(`Setting remote video enabled to ${enabled} for user ${userId}`);
call.isRemoteVideoEnabled = enabled;
// Reset counter when feature is disabled
if (!enabled) {
call.receivedVideoTrackCount = 0;
console.log("Reset video track counter");
}
}
}
@@ -1179,9 +1287,18 @@ export function setRemoteScreenSharing(userId: number, enabled: boolean): void {
if (call) {
console.log(`Setting remote screen sharing to ${enabled} for user ${userId}`);
call.isRemoteScreenSharing = enabled;
// Reset counter when feature is disabled
if (!enabled) {
call.receivedScreenShareTrackCount = 0;
console.log("Reset screen share track counter");
}
}
}
export function setScreenShareStateChangeHandler(handler: ((userId: number, isSharing: boolean) => void) | null): void {
onScreenShareStateChange = handler;
}
export function cleanup(): void {
// Clean up all calls
for (const userId of calls.keys()) {
frontend/src/pages/chat/css/_callWindow.scss
+11 -4
View File
@@ -154,10 +154,14 @@
}
}
.screen-share-tile {
min-height: 150px;
.screen-share-tile {
min-height: 150px;
.screen-share-video {
object-fit: contain;
}
}
}
.call-controls {
padding: 8px;
@@ -391,10 +395,13 @@
min-height: 480px;
grid-column: 1 / -1;
border: 2px solid rgba($color-dark-primary, 0.6);
overflow: hidden;
.video-element {
.screen-share-video {
width: 100%;
height: 100%;
object-fit: contain;
background: rgba(0, 0, 0, 0.9);
background: rgba(0, 0, 0, 0.95);
}
.tile-label {
frontend/src/pages/chat/ui/right/calls/CallWindow.tsx
+2 -2
View File
@@ -247,7 +247,7 @@ export function CallWindow() {
style={{ display: call.isSharingScreen ? "flex" : "none" }}>
<video
ref={localScreenShareRef}
className="video-element"
className="video-element screen-share-video"
autoPlay
playsInline
muted />
@@ -259,7 +259,7 @@ export function CallWindow() {
style={{ display: call.isRemoteScreenSharing ? "flex" : "none" }}>
<video
ref={remoteScreenShareRef}
className="video-element"
className="video-element screen-share-video"
autoPlay
playsInline />
<div className="tile-label">{remoteUsername}&apos;s Screen</div>