Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions client/src/components/RemoteAudio.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import { render } from '@testing-library/react';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import RemoteAudio from './RemoteAudio';

// Each PeerAudio builds its own AudioContext graph: source → gain → destination.
// Record which streams got a graph and at what gain.
const graphs: Array<{ stream: MediaStream; gain: { value: number } }> = [];

class FakeAudioContext {
state = 'running';
destination = {};
createMediaStreamSource(stream: MediaStream) {
const graph = { stream, gain: { value: 1 } };
graphs.push(graph);
return { connect: vi.fn(), disconnect: vi.fn(), graph };
}
createGain() {
const graph = graphs[graphs.length - 1];
return { gain: graph.gain, connect: vi.fn(), disconnect: vi.fn() };
}
resume() { return Promise.resolve(); }
close() { return Promise.resolve(); }
}

function fakeStream(kinds: Array<'audio' | 'video'>): MediaStream {
const tracks = kinds.map((kind) => ({ kind }));
return {
getTracks: () => tracks,
getAudioTracks: () => tracks.filter((t) => t.kind === 'audio'),
getVideoTracks: () => tracks.filter((t) => t.kind === 'video'),
} as unknown as MediaStream;
}

const originalAudioContext = window.AudioContext;
beforeEach(() => {
graphs.length = 0;
window.AudioContext = FakeAudioContext as unknown as typeof AudioContext;
});
afterEach(() => { window.AudioContext = originalAudioContext; });

describe('RemoteAudio', () => {
it('plays a shared screen\'s audio alongside the peer\'s mic', () => {
const mic = fakeStream(['audio', 'video']);
const screen = fakeStream(['video', 'audio']);

render(<RemoteAudio streams={{ 'peer-1': mic }} screenStreams={{ 'peer-1': screen }} />);

expect(graphs.map((g) => g.stream)).toEqual([mic, screen]);
});

it('skips a screen share that carries no audio', () => {
render(<RemoteAudio streams={{}} screenStreams={{ 'peer-1': fakeStream(['video']) }} />);

expect(graphs).toHaveLength(0);
});

it('plays shared audio at the master volume, not the peer\'s mic override', () => {
const screen = fakeStream(['video', 'audio']);

render(
<RemoteAudio
streams={{ 'peer-1': fakeStream(['audio']) }}
screenStreams={{ 'peer-1': screen }}
masterVolume={0.5}
peerVolumes={{ 'peer-1': 0 }}
/>,
);

const [micGraph, screenGraph] = graphs;
expect(micGraph.gain.value).toBe(0);
expect(screenGraph.stream).toBe(screen);
expect(screenGraph.gain.value).toBe(0.5);
});
});
14 changes: 13 additions & 1 deletion client/src/components/RemoteAudio.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,20 @@ function PeerAudio({ socketId, stream, volume = 1 }: PeerAudioProps) {
// masterVolume: global speaker slider (0–1).
// peerVolumes: per-peer overrides keyed by socketId (0–1, default 1).
// Final per-peer gain = clamp(masterVolume × peerVolume, 0, 1).
// screenStreams: remote screen shares keyed by socketId. A share's audio (tab or
// system audio) gets its own graph at masterVolume only: the per-peer override is
// for that person's voice, and turning down a talker shouldn't silence the video
// they're presenting. Video-only shares are skipped — a stream with no audio
// track can't feed createMediaStreamSource.
interface RemoteAudioProps {
streams: Record<string, MediaStream>;
screenStreams?: Record<string, MediaStream>;
masterVolume?: number;
peerVolumes?: Record<string, number>;
}

export default function RemoteAudio({ streams, masterVolume = 1, peerVolumes = {} }: RemoteAudioProps) {
export default function RemoteAudio({ streams, screenStreams = {}, masterVolume = 1, peerVolumes = {} }: RemoteAudioProps) {
const master = Math.max(0, Math.min(1, masterVolume));
return (
<>
{Object.entries(streams).map(([socketId, stream]) => {
Expand All @@ -74,6 +81,11 @@ export default function RemoteAudio({ streams, masterVolume = 1, peerVolumes = {
<PeerAudio key={socketId} socketId={socketId} stream={stream} volume={finalVol} />
);
})}
{Object.entries(screenStreams)
.filter(([, stream]) => stream.getAudioTracks().length > 0)
.map(([socketId, stream]) => (
<PeerAudio key={`screen:${socketId}`} socketId={`${socketId}:screen`} stream={stream} volume={master} />
))}
</>
);
}
64 changes: 64 additions & 0 deletions client/src/hooks/useMediasoup.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,70 @@ describe('useMediasoup', () => {
expect(H.request).toHaveBeenCalledWith('sfu-close-producer', expect.objectContaining({ producerId: expect.any(String) }));
});

it('asks the picker for audio and shares video-only when none is picked', async () => {
vi.mocked(navigator.mediaDevices.getDisplayMedia).mockResolvedValue(
new FakeMediaStream([fakeTrack('video', 'screen-1')]) as unknown as MediaStream,
);
const { result } = mount();
await waitForSetup();

await act(async () => { await result.current.shareScreen(); });

const [constraints] = vi.mocked(navigator.mediaDevices.getDisplayMedia).mock.calls[0];
expect(constraints?.audio).toBeTruthy();
expect(result.current.isScreenSharing).toBe(true);
expect(H.produced.map((p: any) => p.appData?.mediaTag)).not.toContain('screen-audio');
});

it('shares tab audio as a second screen producer and stops both together', async () => {
const screenVideo = fakeTrack('video', 'screen-1');
const screenAudio = fakeTrack('audio', 'tab-audio-1');
vi.mocked(navigator.mediaDevices.getDisplayMedia).mockResolvedValue(
new FakeMediaStream([screenVideo, screenAudio]) as unknown as MediaStream,
);
const { result } = mount();
await waitForSetup();

await act(async () => { await result.current.shareScreen(); });

const audioProduce = H.produced.find((p: any) => p.appData?.mediaTag === 'screen-audio');
expect(audioProduce?.track).toBe(screenAudio);
expect(audioProduce?.appData?.source).toBe('screen');
expect(result.current.localScreenStream?.getAudioTracks()).toEqual([screenAudio]);

act(() => { result.current.stopScreenShare(); });

const closed = H.request.mock.calls
.filter(([event]: [string]) => event === 'sfu-close-producer')
.map(([, data]: [string, { producerId: string }]) => data.producerId);
expect(closed).toHaveLength(2);
expect(closed.some((id: string) => id.startsWith('producer-screen-audio-'))).toBe(true);
expect(screenVideo.stop).toHaveBeenCalled();
expect(screenAudio.stop).toHaveBeenCalled();
expect(result.current.isScreenSharing).toBe(false);
});

it('keeps the screen share going when its audio producer fails', async () => {
const screenAudio = fakeTrack('audio', 'tab-audio-1');
vi.mocked(navigator.mediaDevices.getDisplayMedia).mockResolvedValue(
new FakeMediaStream([fakeTrack('video', 'screen-1'), screenAudio]) as unknown as MediaStream,
);
const { result } = mount();
await waitForSetup();
const sendTransport = H.sendTransports[0];
const produce = sendTransport.produce.getMockImplementation();
sendTransport.produce.mockImplementation(async (opts: any) => {
if (opts.appData?.mediaTag === 'screen-audio') throw new Error('too many producers');
return produce(opts);
});

await act(async () => { await result.current.shareScreen(); });

expect(result.current.isScreenSharing).toBe(true);
expect(screenAudio.stop).toHaveBeenCalled();
expect(result.current.localScreenStream?.getAudioTracks()).toHaveLength(0);
});

it('reflects peer pause/resume, hand-raise, and active-speaker events', async () => {
const { result } = mount();
await waitForSetup();
Expand Down
67 changes: 60 additions & 7 deletions client/src/hooks/useMediasoup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,36 @@ const CAM_VIDEO_ENCODINGS = [
];
const CAM_VIDEO_CODEC_OPTIONS = { videoGoogleStartBitrate: 1000 };

// Screen-share capture. Asking for audio makes Chrome/Edge show "Also share tab
// audio" (and, with `systemAudio`, "system audio" for whole-screen shares); the
// stream carries an audio track only when the presenter ticks it. Shared audio
// is music or a video soundtrack, not a voice, so the mic's voice processing is
// off, and the presenter keeps hearing their own tab. `systemAudio`, `cursor`
// and `suppressLocalAudioPlayback` are Chromium hints missing from lib.dom.
type DisplayCaptureOptions = DisplayMediaStreamOptions & {
video: MediaTrackConstraints & { cursor?: 'always' | 'motion' | 'never' };
audio: MediaTrackConstraints & { suppressLocalAudioPlayback?: boolean };
systemAudio?: 'include' | 'exclude';
};
const DISPLAY_CAPTURE_OPTIONS: DisplayCaptureOptions = {
video: { cursor: 'always' },
audio: {
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
suppressLocalAudioPlayback: false,
},
systemAudio: 'include',
};
// Stereo music-grade Opus. DTX stays off: it treats quiet passages as silence
// and drops them, which voice tolerates and a soundtrack does not.
const SCREEN_AUDIO_CODEC_OPTIONS = {
opusStereo: true,
opusFec: true,
opusDtx: false,
opusMaxAverageBitrate: 128_000,
};

// First-connection recovery: a failed setup (incl. the very first one, which the
// 2026-07-05 incident stranded media-less) is retried with capped exponential
// backoff while the socket is connected. A socket drop pauses the ladder — the
Expand Down Expand Up @@ -133,6 +163,7 @@ export function useMediasoup(roomId: string, devices: MediaDevicesOptions = {})
const producerInfoRef = useRef(new Map<string, ProducerInfo>());
const pendingProducersRef = useRef(new Map<string, SfuProducerDescriptor>());
const screenProducerRef = useRef<Producer | null>(null);
const screenAudioProducerRef = useRef<Producer | null>(null);
const localScreenStreamRef = useRef<MediaStream | null>(null);
const handRaisedRef = useRef(false);
const audioCtxRef = useRef<AudioContext | null>(null);
Expand Down Expand Up @@ -280,10 +311,13 @@ export function useMediasoup(roomId: string, devices: MediaDevicesOptions = {})
const stopScreenShare = useCallback(() => {
const producer = screenProducerRef.current;
if (!producer) return;
const producerId = producer.id;
try { producer.close(); } catch { /* gone */ }
const audioProducer = screenAudioProducerRef.current;
screenProducerRef.current = null;
request('sfu-close-producer', { producerId }).catch(() => {});
screenAudioProducerRef.current = null;
for (const p of audioProducer ? [producer, audioProducer] : [producer]) {
try { p.close(); } catch { /* gone */ }
request('sfu-close-producer', { producerId: p.id }).catch(() => {});
}
const stream = localScreenStreamRef.current;
if (stream) stream.getTracks().forEach((t) => t.stop());
localScreenStreamRef.current = null;
Expand All @@ -297,14 +331,31 @@ export function useMediasoup(roomId: string, devices: MediaDevicesOptions = {})
const sendTransport = sendTransportRef.current;
if (!sendTransport) return;
try {
const stream = await navigator.mediaDevices.getDisplayMedia({
video: { cursor: 'always' } as MediaTrackConstraints,
audio: false,
});
const stream = await navigator.mediaDevices.getDisplayMedia(DISPLAY_CAPTURE_OPTIONS);
const track = stream.getVideoTracks()[0];
if (!track) { stream.getTracks().forEach((t) => t.stop()); return; }
const producer = await sendTransport.produce({ track, appData: { source: 'screen' } });
screenProducerRef.current = producer;
// Shared audio is a second `source: 'screen'` producer, so receivers file
// it with the share rather than the mic, and the server already keeps it
// out of active-speaker and host-mute. The share is already live, so a
// failed audio produce degrades to video-only instead of cancelling it.
const audioTrack = stream.getAudioTracks()[0];
if (audioTrack) {
try {
const audioProducer = await sendTransport.produce({
track: audioTrack,
codecOptions: SCREEN_AUDIO_CODEC_OPTIONS,
appData: { source: 'screen', mediaTag: 'screen-audio' },
});
screenAudioProducerRef.current = audioProducer;
audioProducer.on('transportclose', stopScreenShare);
} catch (err: unknown) {
stream.removeTrack(audioTrack);
audioTrack.stop();
if (import.meta.env.DEV) console.warn('[sfu] screen audio failed:', errorMessage(err));
}
}
localScreenStreamRef.current = stream;
setLocalScreenStream(stream);
setLocalScreenSurface(track.getSettings().displaySurface ?? null);
Expand Down Expand Up @@ -968,7 +1019,9 @@ export function useMediasoup(roomId: string, devices: MediaDevicesOptions = {})
pendingProducers.clear();

try { screenProducerRef.current?.close(); } catch { /* gone */ }
try { screenAudioProducerRef.current?.close(); } catch { /* gone */ }
screenProducerRef.current = null;
screenAudioProducerRef.current = null;
localScreenStreamRef.current?.getTracks().forEach((t) => t.stop());
localScreenStreamRef.current = null;

Expand Down
12 changes: 10 additions & 2 deletions client/src/pages/RoomPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -831,9 +831,12 @@ export default function RoomPage() {
</Box>
) : (
<>
{/* Shared tab/system audio plays through <RemoteAudio> so it
survives layout switches and unpinned shares; the stage
<video> stays muted to avoid playing it twice. */}
<VideoTile
stream={pinnedShare.stream}
muted={pinnedShare.isLocal}
muted
name={pinnedShare.isLocal ? 'Your screen' : `${pinnedShare.name}'s screen`}
videoOn
audioOn
Expand Down Expand Up @@ -1190,7 +1193,12 @@ export default function RoomPage() {
>
{/* Remote audio: dedicated hidden <audio> per peer, mounted once outside
the tile layout so audio survives layout switches and late tracks. */}
<RemoteAudio streams={remoteStreams} masterVolume={outputVolume} peerVolumes={peerVolumes} />
<RemoteAudio
streams={remoteStreams}
screenStreams={remoteScreens}
masterVolume={outputVolume}
peerVolumes={peerVolumes}
/>

{/* Dev-only WebRTC stats overlay (no-op in production builds). */}
<RtcStatsOverlay stats={rtcStats} />
Expand Down
Loading