diff --git a/client/src/components/RemoteAudio.test.tsx b/client/src/components/RemoteAudio.test.tsx new file mode 100644 index 0000000..a308019 --- /dev/null +++ b/client/src/components/RemoteAudio.test.tsx @@ -0,0 +1,74 @@ +import { render } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import RemoteAudio from './RemoteAudio'; + +// Each PeerAudio builds its own AudioContext graph: source → gain → destination. +// Record which streams got a graph and at what gain. +const graphs: Array<{ stream: MediaStream; gain: { value: number } }> = []; + +class FakeAudioContext { + state = 'running'; + destination = {}; + createMediaStreamSource(stream: MediaStream) { + const graph = { stream, gain: { value: 1 } }; + graphs.push(graph); + return { connect: vi.fn(), disconnect: vi.fn(), graph }; + } + createGain() { + const graph = graphs[graphs.length - 1]; + return { gain: graph.gain, connect: vi.fn(), disconnect: vi.fn() }; + } + resume() { return Promise.resolve(); } + close() { return Promise.resolve(); } +} + +function fakeStream(kinds: Array<'audio' | 'video'>): MediaStream { + const tracks = kinds.map((kind) => ({ kind })); + return { + getTracks: () => tracks, + getAudioTracks: () => tracks.filter((t) => t.kind === 'audio'), + getVideoTracks: () => tracks.filter((t) => t.kind === 'video'), + } as unknown as MediaStream; +} + +const originalAudioContext = window.AudioContext; +beforeEach(() => { + graphs.length = 0; + window.AudioContext = FakeAudioContext as unknown as typeof AudioContext; +}); +afterEach(() => { window.AudioContext = originalAudioContext; }); + +describe('RemoteAudio', () => { + it('plays a shared screen\'s audio alongside the peer\'s mic', () => { + const mic = fakeStream(['audio', 'video']); + const screen = fakeStream(['video', 'audio']); + + render(); + + expect(graphs.map((g) => g.stream)).toEqual([mic, screen]); + }); + + it('skips a screen share that carries no audio', () => { + render(); + + expect(graphs).toHaveLength(0); + }); + + it('plays shared audio at the master volume, not the peer\'s mic override', () => { + const screen = fakeStream(['video', 'audio']); + + render( + , + ); + + const [micGraph, screenGraph] = graphs; + expect(micGraph.gain.value).toBe(0); + expect(screenGraph.stream).toBe(screen); + expect(screenGraph.gain.value).toBe(0.5); + }); +}); diff --git a/client/src/components/RemoteAudio.tsx b/client/src/components/RemoteAudio.tsx index d67152e..b96b686 100644 --- a/client/src/components/RemoteAudio.tsx +++ b/client/src/components/RemoteAudio.tsx @@ -58,13 +58,20 @@ function PeerAudio({ socketId, stream, volume = 1 }: PeerAudioProps) { // masterVolume: global speaker slider (0–1). // peerVolumes: per-peer overrides keyed by socketId (0–1, default 1). // Final per-peer gain = clamp(masterVolume × peerVolume, 0, 1). +// screenStreams: remote screen shares keyed by socketId. A share's audio (tab or +// system audio) gets its own graph at masterVolume only: the per-peer override is +// for that person's voice, and turning down a talker shouldn't silence the video +// they're presenting. Video-only shares are skipped — a stream with no audio +// track can't feed createMediaStreamSource. interface RemoteAudioProps { streams: Record; + screenStreams?: Record; masterVolume?: number; peerVolumes?: Record; } -export default function RemoteAudio({ streams, masterVolume = 1, peerVolumes = {} }: RemoteAudioProps) { +export default function RemoteAudio({ streams, screenStreams = {}, masterVolume = 1, peerVolumes = {} }: RemoteAudioProps) { + const master = Math.max(0, Math.min(1, masterVolume)); return ( <> {Object.entries(streams).map(([socketId, stream]) => { @@ -74,6 +81,11 @@ export default function RemoteAudio({ streams, masterVolume = 1, peerVolumes = { ); })} + {Object.entries(screenStreams) + .filter(([, stream]) => stream.getAudioTracks().length > 0) + .map(([socketId, stream]) => ( + + ))} ); } diff --git a/client/src/hooks/useMediasoup.test.tsx b/client/src/hooks/useMediasoup.test.tsx index 6aaf699..bf71905 100644 --- a/client/src/hooks/useMediasoup.test.tsx +++ b/client/src/hooks/useMediasoup.test.tsx @@ -385,6 +385,70 @@ describe('useMediasoup', () => { expect(H.request).toHaveBeenCalledWith('sfu-close-producer', expect.objectContaining({ producerId: expect.any(String) })); }); + it('asks the picker for audio and shares video-only when none is picked', async () => { + vi.mocked(navigator.mediaDevices.getDisplayMedia).mockResolvedValue( + new FakeMediaStream([fakeTrack('video', 'screen-1')]) as unknown as MediaStream, + ); + const { result } = mount(); + await waitForSetup(); + + await act(async () => { await result.current.shareScreen(); }); + + const [constraints] = vi.mocked(navigator.mediaDevices.getDisplayMedia).mock.calls[0]; + expect(constraints?.audio).toBeTruthy(); + expect(result.current.isScreenSharing).toBe(true); + expect(H.produced.map((p: any) => p.appData?.mediaTag)).not.toContain('screen-audio'); + }); + + it('shares tab audio as a second screen producer and stops both together', async () => { + const screenVideo = fakeTrack('video', 'screen-1'); + const screenAudio = fakeTrack('audio', 'tab-audio-1'); + vi.mocked(navigator.mediaDevices.getDisplayMedia).mockResolvedValue( + new FakeMediaStream([screenVideo, screenAudio]) as unknown as MediaStream, + ); + const { result } = mount(); + await waitForSetup(); + + await act(async () => { await result.current.shareScreen(); }); + + const audioProduce = H.produced.find((p: any) => p.appData?.mediaTag === 'screen-audio'); + expect(audioProduce?.track).toBe(screenAudio); + expect(audioProduce?.appData?.source).toBe('screen'); + expect(result.current.localScreenStream?.getAudioTracks()).toEqual([screenAudio]); + + act(() => { result.current.stopScreenShare(); }); + + const closed = H.request.mock.calls + .filter(([event]: [string]) => event === 'sfu-close-producer') + .map(([, data]: [string, { producerId: string }]) => data.producerId); + expect(closed).toHaveLength(2); + expect(closed.some((id: string) => id.startsWith('producer-screen-audio-'))).toBe(true); + expect(screenVideo.stop).toHaveBeenCalled(); + expect(screenAudio.stop).toHaveBeenCalled(); + expect(result.current.isScreenSharing).toBe(false); + }); + + it('keeps the screen share going when its audio producer fails', async () => { + const screenAudio = fakeTrack('audio', 'tab-audio-1'); + vi.mocked(navigator.mediaDevices.getDisplayMedia).mockResolvedValue( + new FakeMediaStream([fakeTrack('video', 'screen-1'), screenAudio]) as unknown as MediaStream, + ); + const { result } = mount(); + await waitForSetup(); + const sendTransport = H.sendTransports[0]; + const produce = sendTransport.produce.getMockImplementation(); + sendTransport.produce.mockImplementation(async (opts: any) => { + if (opts.appData?.mediaTag === 'screen-audio') throw new Error('too many producers'); + return produce(opts); + }); + + await act(async () => { await result.current.shareScreen(); }); + + expect(result.current.isScreenSharing).toBe(true); + expect(screenAudio.stop).toHaveBeenCalled(); + expect(result.current.localScreenStream?.getAudioTracks()).toHaveLength(0); + }); + it('reflects peer pause/resume, hand-raise, and active-speaker events', async () => { const { result } = mount(); await waitForSetup(); diff --git a/client/src/hooks/useMediasoup.ts b/client/src/hooks/useMediasoup.ts index 70d0722..c5e2a86 100644 --- a/client/src/hooks/useMediasoup.ts +++ b/client/src/hooks/useMediasoup.ts @@ -32,6 +32,36 @@ const CAM_VIDEO_ENCODINGS = [ ]; const CAM_VIDEO_CODEC_OPTIONS = { videoGoogleStartBitrate: 1000 }; +// Screen-share capture. Asking for audio makes Chrome/Edge show "Also share tab +// audio" (and, with `systemAudio`, "system audio" for whole-screen shares); the +// stream carries an audio track only when the presenter ticks it. Shared audio +// is music or a video soundtrack, not a voice, so the mic's voice processing is +// off, and the presenter keeps hearing their own tab. `systemAudio`, `cursor` +// and `suppressLocalAudioPlayback` are Chromium hints missing from lib.dom. +type DisplayCaptureOptions = DisplayMediaStreamOptions & { + video: MediaTrackConstraints & { cursor?: 'always' | 'motion' | 'never' }; + audio: MediaTrackConstraints & { suppressLocalAudioPlayback?: boolean }; + systemAudio?: 'include' | 'exclude'; +}; +const DISPLAY_CAPTURE_OPTIONS: DisplayCaptureOptions = { + video: { cursor: 'always' }, + audio: { + echoCancellation: false, + noiseSuppression: false, + autoGainControl: false, + suppressLocalAudioPlayback: false, + }, + systemAudio: 'include', +}; +// Stereo music-grade Opus. DTX stays off: it treats quiet passages as silence +// and drops them, which voice tolerates and a soundtrack does not. +const SCREEN_AUDIO_CODEC_OPTIONS = { + opusStereo: true, + opusFec: true, + opusDtx: false, + opusMaxAverageBitrate: 128_000, +}; + // First-connection recovery: a failed setup (incl. the very first one, which the // 2026-07-05 incident stranded media-less) is retried with capped exponential // backoff while the socket is connected. A socket drop pauses the ladder — the @@ -133,6 +163,7 @@ export function useMediasoup(roomId: string, devices: MediaDevicesOptions = {}) const producerInfoRef = useRef(new Map()); const pendingProducersRef = useRef(new Map()); const screenProducerRef = useRef(null); + const screenAudioProducerRef = useRef(null); const localScreenStreamRef = useRef(null); const handRaisedRef = useRef(false); const audioCtxRef = useRef(null); @@ -280,10 +311,13 @@ export function useMediasoup(roomId: string, devices: MediaDevicesOptions = {}) const stopScreenShare = useCallback(() => { const producer = screenProducerRef.current; if (!producer) return; - const producerId = producer.id; - try { producer.close(); } catch { /* gone */ } + const audioProducer = screenAudioProducerRef.current; screenProducerRef.current = null; - request('sfu-close-producer', { producerId }).catch(() => {}); + screenAudioProducerRef.current = null; + for (const p of audioProducer ? [producer, audioProducer] : [producer]) { + try { p.close(); } catch { /* gone */ } + request('sfu-close-producer', { producerId: p.id }).catch(() => {}); + } const stream = localScreenStreamRef.current; if (stream) stream.getTracks().forEach((t) => t.stop()); localScreenStreamRef.current = null; @@ -297,14 +331,31 @@ export function useMediasoup(roomId: string, devices: MediaDevicesOptions = {}) const sendTransport = sendTransportRef.current; if (!sendTransport) return; try { - const stream = await navigator.mediaDevices.getDisplayMedia({ - video: { cursor: 'always' } as MediaTrackConstraints, - audio: false, - }); + const stream = await navigator.mediaDevices.getDisplayMedia(DISPLAY_CAPTURE_OPTIONS); const track = stream.getVideoTracks()[0]; if (!track) { stream.getTracks().forEach((t) => t.stop()); return; } const producer = await sendTransport.produce({ track, appData: { source: 'screen' } }); screenProducerRef.current = producer; + // Shared audio is a second `source: 'screen'` producer, so receivers file + // it with the share rather than the mic, and the server already keeps it + // out of active-speaker and host-mute. The share is already live, so a + // failed audio produce degrades to video-only instead of cancelling it. + const audioTrack = stream.getAudioTracks()[0]; + if (audioTrack) { + try { + const audioProducer = await sendTransport.produce({ + track: audioTrack, + codecOptions: SCREEN_AUDIO_CODEC_OPTIONS, + appData: { source: 'screen', mediaTag: 'screen-audio' }, + }); + screenAudioProducerRef.current = audioProducer; + audioProducer.on('transportclose', stopScreenShare); + } catch (err: unknown) { + stream.removeTrack(audioTrack); + audioTrack.stop(); + if (import.meta.env.DEV) console.warn('[sfu] screen audio failed:', errorMessage(err)); + } + } localScreenStreamRef.current = stream; setLocalScreenStream(stream); setLocalScreenSurface(track.getSettings().displaySurface ?? null); @@ -968,7 +1019,9 @@ export function useMediasoup(roomId: string, devices: MediaDevicesOptions = {}) pendingProducers.clear(); try { screenProducerRef.current?.close(); } catch { /* gone */ } + try { screenAudioProducerRef.current?.close(); } catch { /* gone */ } screenProducerRef.current = null; + screenAudioProducerRef.current = null; localScreenStreamRef.current?.getTracks().forEach((t) => t.stop()); localScreenStreamRef.current = null; diff --git a/client/src/pages/RoomPage.tsx b/client/src/pages/RoomPage.tsx index 432f0c8..d75a5c4 100644 --- a/client/src/pages/RoomPage.tsx +++ b/client/src/pages/RoomPage.tsx @@ -831,9 +831,12 @@ export default function RoomPage() { ) : ( <> + {/* Shared tab/system audio plays through so it + survives layout switches and unpinned shares; the stage +