From cef6fe3f5d8968b1f8893ba808fb3ed62f933c0e Mon Sep 17 00:00:00 2001 From: Tim Mendoza Date: Tue, 3 May 2022 15:12:51 -0600 Subject: [PATCH 1/8] Add useTracks hook --- src/hooks/useTracks/useTracks.test.ts | 58 +++++++++++++++++++++++++++ src/hooks/useTracks/useTracks.ts | 29 ++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 src/hooks/useTracks/useTracks.test.ts create mode 100644 src/hooks/useTracks/useTracks.ts diff --git a/src/hooks/useTracks/useTracks.test.ts b/src/hooks/useTracks/useTracks.test.ts new file mode 100644 index 000000000..6b5c863b0 --- /dev/null +++ b/src/hooks/useTracks/useTracks.test.ts @@ -0,0 +1,58 @@ +import { renderHook, act } from '@testing-library/react-hooks'; +import EventEmitter from 'events'; +import useTracks from './useTracks'; + +describe('the useTracks hook', () => { + let mockParticipant: any; + + beforeEach(() => { + mockParticipant = new EventEmitter(); + mockParticipant.tracks = new Map([ + [0, { track: 'track1' }], + [1, { track: null }], + [2, { track: 'track2' }], + ]); + }); + + it('should return an array of mockParticipant.tracks by default, filtering out null tracks', () => { + const { result } = renderHook(() => useTracks(mockParticipant)); + expect(result.current).toEqual(['track1', 'track2']); + }); + + it('should respond to "trackSubscribed" events', async () => { + const { result } = renderHook(() => useTracks(mockParticipant)); + act(() => { + mockParticipant.emit('trackSubscribed', 'newMockTrack'); + }); + expect(result.current).toEqual(['track1', 'track2', 'newMockTrack']); + }); + + it('should respond to "trackUnsubscribed" events', async () => { + const { result } = renderHook(() => useTracks(mockParticipant)); + act(() => { + mockParticipant.emit('trackUnsubscribed', 'track1'); + }); + expect(result.current).toEqual(['track2']); + }); + + it('should return a new set of tracks if the participant changes', () => { + const { result, rerender } = renderHook(({ participant }) => useTracks(participant), { + initialProps: { participant: mockParticipant }, + }); + expect(result.current).toEqual(['track1', 'track2']); + mockParticipant = new EventEmitter(); + mockParticipant.tracks = new Map([ + [0, { track: 'track3' }], + [1, { track: 'track4' }], + ]); + rerender({ participant: mockParticipant }); + expect(result.current).toEqual(['track3', 'track4']); + }); + + it('should clean up listeners on unmount', () => { + const { unmount } = renderHook(() => useTracks(mockParticipant)); + unmount(); + expect(mockParticipant.listenerCount('trackSubscribed')).toBe(0); + expect(mockParticipant.listenerCount('trackUnsubscribed')).toBe(0); + }); +}); diff --git a/src/hooks/useTracks/useTracks.ts b/src/hooks/useTracks/useTracks.ts new file mode 100644 index 000000000..b5eda6985 --- /dev/null +++ b/src/hooks/useTracks/useTracks.ts @@ -0,0 +1,29 @@ +import { RemoteParticipant, RemoteTrack } from 'twilio-video'; +import { useEffect, useState } from 'react'; + +export default function useTracks(participant: RemoteParticipant | undefined) { + const [tracks, setTracks] = useState([]); + + useEffect(() => { + if (participant) { + const subscribedTracks = Array.from(participant.tracks.values()) + .filter(trackPublication => trackPublication.track !== null) + .map(trackPublication => trackPublication.track!); + + setTracks(subscribedTracks); + + const handleTrackSubscribed = (track: RemoteTrack) => setTracks(prevTracks => [...prevTracks, track]); + const handleTrackUnsubscribed = (track: RemoteTrack) => + setTracks(prevTracks => prevTracks.filter(t => t !== track)); + + participant.on('trackSubscribed', handleTrackSubscribed); + participant.on('trackUnsubscribed', handleTrackUnsubscribed); + return () => { + participant.off('trackSubscribed', handleTrackSubscribed); + participant.off('trackUnsubscribed', handleTrackUnsubscribed); + }; + } + }, [participant]); + + return tracks; +} From 3cd6dd4e3b061f229ec52d7a0ec79c8df5e12552 Mon Sep 17 00:00:00 2001 From: Tim Mendoza Date: Tue, 3 May 2022 15:34:14 -0600 Subject: [PATCH 2/8] Remove media-transcriber from UI --- src/components/ParticipantList/ParticipantList.tsx | 2 +- src/hooks/useMainParticipant/useMainParticipant.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/ParticipantList/ParticipantList.tsx b/src/components/ParticipantList/ParticipantList.tsx index b511971fa..510122653 100644 --- a/src/components/ParticipantList/ParticipantList.tsx +++ b/src/components/ParticipantList/ParticipantList.tsx @@ -45,7 +45,7 @@ export default function ParticipantList() { const classes = useStyles(); const { room } = useVideoContext(); const localParticipant = room!.localParticipant; - const participants = useParticipants(); + const participants = useParticipants().filter(p => p.identity !== 'media-transcriber'); const [selectedParticipant, setSelectedParticipant] = useSelectedParticipant(); const screenShareParticipant = useScreenShareParticipant(); const mainParticipant = useMainParticipant(); diff --git a/src/hooks/useMainParticipant/useMainParticipant.tsx b/src/hooks/useMainParticipant/useMainParticipant.tsx index 533b2d998..b28da2570 100644 --- a/src/hooks/useMainParticipant/useMainParticipant.tsx +++ b/src/hooks/useMainParticipant/useMainParticipant.tsx @@ -8,7 +8,7 @@ export default function useMainParticipant() { const [selectedParticipant] = useSelectedParticipant(); const screenShareParticipant = useScreenShareParticipant(); const dominantSpeaker = useDominantSpeaker(); - const participants = useParticipants(); + const participants = useParticipants().filter(p => p.identity !== 'media-transcriber'); const { room } = useVideoContext(); const localParticipant = room?.localParticipant; const remoteScreenShareParticipant = screenShareParticipant !== localParticipant ? screenShareParticipant : null; From b7d2dc70d3216d27e9cbb27d842ebc6fc8b6f6a6 Mon Sep 17 00:00:00 2001 From: Tim Mendoza Date: Tue, 3 May 2022 15:36:37 -0600 Subject: [PATCH 3/8] Add menu button to toggle captions --- src/components/MenuBar/Menu/Menu.tsx | 10 +++++++++- src/state/index.tsx | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/components/MenuBar/Menu/Menu.tsx b/src/components/MenuBar/Menu/Menu.tsx index c2a094b63..1e6b3db1d 100644 --- a/src/components/MenuBar/Menu/Menu.tsx +++ b/src/components/MenuBar/Menu/Menu.tsx @@ -1,6 +1,7 @@ import React, { useState, useRef } from 'react'; import AboutDialog from '../../AboutDialog/AboutDialog'; import BackgroundIcon from '../../../icons/BackgroundIcon'; +import ClosedCaptionsIcon from '@material-ui/icons/ClosedCaption'; import DeviceSelectionDialog from '../../DeviceSelectionDialog/DeviceSelectionDialog'; import ExpandMoreIcon from '@material-ui/icons/ExpandMore'; import InfoIconOutlined from '../../../icons/InfoIconOutlined'; @@ -34,7 +35,7 @@ export default function Menu(props: { buttonClassName?: string }) { const [menuOpen, setMenuOpen] = useState(false); const [settingsOpen, setSettingsOpen] = useState(false); - const { isFetching, updateRecordingRules, roomType } = useAppState(); + const { isFetching, updateRecordingRules, roomType, displayCaptions, setDisplayCaptions } = useAppState(); const { setIsChatWindowOpen } = useChatContext(); const isRecording = useIsRecording(); const { room, setIsBackgroundSelectionOpen } = useVideoContext(); @@ -133,6 +134,13 @@ export default function Menu(props: { buttonClassName?: string }) { Room Monitor + setDisplayCaptions(prevDisplayCaptions => !prevDisplayCaptions)}> + + + + {displayCaptions ? 'Hide Captions' : 'Show Captions'} + + setAboutOpen(true)}> diff --git a/src/state/index.tsx b/src/state/index.tsx index 6d928c869..64e675a17 100644 --- a/src/state/index.tsx +++ b/src/state/index.tsx @@ -22,6 +22,8 @@ export interface StateContextType { dispatchSetting: React.Dispatch; roomType?: RoomType; updateRecordingRules(room_sid: string, rules: RecordingRules): Promise; + displayCaptions: boolean; + setDisplayCaptions: React.Dispatch>; } export const StateContext = createContext(null!); @@ -41,6 +43,7 @@ export default function AppStateProvider(props: React.PropsWithChildren<{}>) { const [activeSinkId, setActiveSinkId] = useActiveSinkId(); const [settings, dispatchSetting] = useReducer(settingsReducer, initialSettings); const [roomType, setRoomType] = useState(); + const [displayCaptions, setDisplayCaptions] = useState(false); let contextValue = { error, @@ -51,6 +54,8 @@ export default function AppStateProvider(props: React.PropsWithChildren<{}>) { settings, dispatchSetting, roomType, + displayCaptions, + setDisplayCaptions, } as StateContextType; if (process.env.REACT_APP_SET_AUTH === 'firebase') { From d91038e4df298b44301f07f648c5645b11bb2f72 Mon Sep 17 00:00:00 2001 From: Tim Mendoza Date: Tue, 3 May 2022 15:36:49 -0600 Subject: [PATCH 4/8] Add caption renderer component --- .../CaptionRenderer/CaptionRenderer.tsx | 117 ++++++++++++++++++ .../CaptionRenderer/CaptionTypes.ts | 37 ++++++ src/components/Room/Room.tsx | 2 + 3 files changed, 156 insertions(+) create mode 100644 src/components/CaptionRenderer/CaptionRenderer.tsx create mode 100644 src/components/CaptionRenderer/CaptionTypes.ts diff --git a/src/components/CaptionRenderer/CaptionRenderer.tsx b/src/components/CaptionRenderer/CaptionRenderer.tsx new file mode 100644 index 000000000..47c04189b --- /dev/null +++ b/src/components/CaptionRenderer/CaptionRenderer.tsx @@ -0,0 +1,117 @@ +import React, { useCallback, useEffect, useState } from 'react'; +import { makeStyles } from '@material-ui/core/styles'; +import { TwilioCaptionResult } from './CaptionTypes'; +import { Typography } from '@material-ui/core'; +import useParticipants from '../../hooks/useParticipants/useParticipants'; +import useTracks from '../../hooks/useTracks/useTracks'; +import { useAppState } from '../../state'; + +interface Caption { + identity: string; + id: string; + timestamp: number; + transcript: string; +} + +const useStyles = makeStyles({ + captionContainer: { + position: 'fixed', + left: '15%', + right: '15%', + top: 'calc(100% - 300px)', + zIndex: 100, + }, + caption: { + color: 'white', + background: 'rgba(0, 0, 0, 0.8)', + padding: '0.2em', + display: 'inline-block', + }, +}); + +export function CaptionRenderer() { + const classes = useStyles(); + const [captions, setCaptions] = useState([]); + const participants = useParticipants(); + const transcriberParticipant = participants.find(p => p.identity === 'media-transcriber'); + const transcriberTracks = useTracks(transcriberParticipant); + const transcriberDataTrack = transcriberTracks.find(track => track.kind === 'data'); + const { displayCaptions } = useAppState(); + + const registerResult = useCallback((result: TwilioCaptionResult) => { + if (result.transcriptionResponse.TranscriptEvent.Transcript.Results.length) { + const transcript = result.transcriptionResponse.TranscriptEvent.Transcript.Results[0].Alternatives[0].Transcript; + const id = result.transcriptionResponse.TranscriptEvent.Transcript.Results[0].ResultId; + const timestamp = Date.now(); + const identity = result.participantIdentity; + + setCaptions(prevCaptions => { + // Make a copy of the caption array, keeping only the 4 most recent captions + const arrayCopy = prevCaptions.slice(-4); + + const existingID = arrayCopy.find(item => item.id === id); + if (existingID) { + const existingIdIndex = arrayCopy.indexOf(existingID); + arrayCopy[existingIdIndex] = { transcript, id, timestamp, identity }; + } + // else if (/* the last transcript has the same identity */){ + // /* append to that transcription instead of creating a new one */ + // } + else { + arrayCopy.push({ transcript, id, timestamp, identity }); + } + + return arrayCopy; + }); + } + }, []); + + useEffect(() => { + if (transcriberDataTrack) { + const handleMessage = (message: string) => { + try { + registerResult(JSON.parse(message)); + } catch (e) { + console.log('received unexpected dataTrack message: ', message); + } + }; + transcriberDataTrack.on('message', handleMessage); + + return () => { + transcriberDataTrack.on('message', handleMessage); + }; + } + }, [transcriberDataTrack, registerResult]); + + // Every second, we go through the captions, and remove any that are older than ten seconds + useEffect(() => { + const intervalId = setInterval(() => { + setCaptions(prevCaptions => { + const now = Date.now(); + const filteredCaptions = prevCaptions.filter(caption => caption.timestamp > now - 10000); + if (filteredCaptions.length !== prevCaptions.length) { + return filteredCaptions; + } else { + return prevCaptions; + } + }); + }, 1000); + return () => { + clearInterval(intervalId); + }; + }, []); + + if (!displayCaptions) return null; + + return ( +
+ {captions.map(caption => ( +
+ + {caption.identity}: {caption.transcript} + +
+ ))} +
+ ); +} diff --git a/src/components/CaptionRenderer/CaptionTypes.ts b/src/components/CaptionRenderer/CaptionTypes.ts new file mode 100644 index 000000000..f956a2631 --- /dev/null +++ b/src/components/CaptionRenderer/CaptionTypes.ts @@ -0,0 +1,37 @@ +export interface TwilioCaptionResult { + transcriptionResponse: TranscriptionResponse; + participantIdentity: string; +} + +export interface TranscriptionResponse { + TranscriptEvent: TranscriptEvent; +} + +export interface TranscriptEvent { + Transcript: Transcript; +} + +export interface Transcript { + Results: Result[]; +} + +export interface Result { + Alternatives: Alternative[]; + EndTime: number; + IsPartial: boolean; + ResultId: string; + StartTime: number; +} + +export interface Alternative { + Items: Item[]; + Transcript: string; +} + +export interface Item { + Content: string; + EndTime: number; + StartTime: number; + Type: string; + VocabularyFilterMatch: boolean; +} diff --git a/src/components/Room/Room.tsx b/src/components/Room/Room.tsx index 542591236..cef8e0ef6 100644 --- a/src/components/Room/Room.tsx +++ b/src/components/Room/Room.tsx @@ -1,6 +1,7 @@ import React from 'react'; import clsx from 'clsx'; import { makeStyles, Theme } from '@material-ui/core'; +import { CaptionRenderer } from '../CaptionRenderer/CaptionRenderer'; import ChatWindow from '../ChatWindow/ChatWindow'; import ParticipantList from '../ParticipantList/ParticipantList'; import MainParticipant from '../MainParticipant/MainParticipant'; @@ -42,6 +43,7 @@ export default function Room() { + ); } From 9f827312764acc57b9a16714f115d7080162d08a Mon Sep 17 00:00:00 2001 From: Tim Mendoza Date: Tue, 3 May 2022 16:17:18 -0600 Subject: [PATCH 5/8] Remove comment --- src/components/CaptionRenderer/CaptionRenderer.tsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/components/CaptionRenderer/CaptionRenderer.tsx b/src/components/CaptionRenderer/CaptionRenderer.tsx index 47c04189b..5fca1aa51 100644 --- a/src/components/CaptionRenderer/CaptionRenderer.tsx +++ b/src/components/CaptionRenderer/CaptionRenderer.tsx @@ -53,11 +53,7 @@ export function CaptionRenderer() { if (existingID) { const existingIdIndex = arrayCopy.indexOf(existingID); arrayCopy[existingIdIndex] = { transcript, id, timestamp, identity }; - } - // else if (/* the last transcript has the same identity */){ - // /* append to that transcription instead of creating a new one */ - // } - else { + } else { arrayCopy.push({ transcript, id, timestamp, identity }); } From 3a8d4fb728a319457da19fb43600be10e17847d6 Mon Sep 17 00:00:00 2001 From: Tim Mendoza Date: Fri, 6 May 2022 16:59:15 -0600 Subject: [PATCH 6/8] rename useTracks to useParticipantTracks --- src/components/CaptionRenderer/CaptionRenderer.tsx | 4 ++-- .../useParticipantTracks.test.ts} | 4 ++-- .../useParticipantTracks.ts} | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/hooks/{useTracks/useTracks.test.ts => useParticipantTracks/useParticipantTracks.test.ts} (95%) rename src/hooks/{useTracks/useTracks.ts => useParticipantTracks/useParticipantTracks.ts} (91%) diff --git a/src/components/CaptionRenderer/CaptionRenderer.tsx b/src/components/CaptionRenderer/CaptionRenderer.tsx index 5fca1aa51..e02568b49 100644 --- a/src/components/CaptionRenderer/CaptionRenderer.tsx +++ b/src/components/CaptionRenderer/CaptionRenderer.tsx @@ -3,7 +3,7 @@ import { makeStyles } from '@material-ui/core/styles'; import { TwilioCaptionResult } from './CaptionTypes'; import { Typography } from '@material-ui/core'; import useParticipants from '../../hooks/useParticipants/useParticipants'; -import useTracks from '../../hooks/useTracks/useTracks'; +import useParticipantTracks from '../../hooks/useParticipantTracks/useParticipantTracks'; import { useAppState } from '../../state'; interface Caption { @@ -34,7 +34,7 @@ export function CaptionRenderer() { const [captions, setCaptions] = useState([]); const participants = useParticipants(); const transcriberParticipant = participants.find(p => p.identity === 'media-transcriber'); - const transcriberTracks = useTracks(transcriberParticipant); + const transcriberTracks = useParticipantTracks(transcriberParticipant); const transcriberDataTrack = transcriberTracks.find(track => track.kind === 'data'); const { displayCaptions } = useAppState(); diff --git a/src/hooks/useTracks/useTracks.test.ts b/src/hooks/useParticipantTracks/useParticipantTracks.test.ts similarity index 95% rename from src/hooks/useTracks/useTracks.test.ts rename to src/hooks/useParticipantTracks/useParticipantTracks.test.ts index 6b5c863b0..b0edd1c7e 100644 --- a/src/hooks/useTracks/useTracks.test.ts +++ b/src/hooks/useParticipantTracks/useParticipantTracks.test.ts @@ -1,8 +1,8 @@ import { renderHook, act } from '@testing-library/react-hooks'; import EventEmitter from 'events'; -import useTracks from './useTracks'; +import useTracks from './useParticipantTracks'; -describe('the useTracks hook', () => { +describe('the useParticipantTracks hook', () => { let mockParticipant: any; beforeEach(() => { diff --git a/src/hooks/useTracks/useTracks.ts b/src/hooks/useParticipantTracks/useParticipantTracks.ts similarity index 91% rename from src/hooks/useTracks/useTracks.ts rename to src/hooks/useParticipantTracks/useParticipantTracks.ts index b5eda6985..9c9503a84 100644 --- a/src/hooks/useTracks/useTracks.ts +++ b/src/hooks/useParticipantTracks/useParticipantTracks.ts @@ -1,7 +1,7 @@ import { RemoteParticipant, RemoteTrack } from 'twilio-video'; import { useEffect, useState } from 'react'; -export default function useTracks(participant: RemoteParticipant | undefined) { +export default function useParticipantTracks(participant: RemoteParticipant | undefined) { const [tracks, setTracks] = useState([]); useEffect(() => { From 6a92bab2b0452ed03e91508f3a1825a1a9255529 Mon Sep 17 00:00:00 2001 From: Tim Mendoza Date: Wed, 22 Jun 2022 21:32:33 -0600 Subject: [PATCH 7/8] Update caption renderer to handle new format and error tracks --- .../CaptionRenderer/CaptionRenderer.tsx | 48 ++++++++++++------- .../CaptionRenderer/CaptionTypes.ts | 2 +- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/components/CaptionRenderer/CaptionRenderer.tsx b/src/components/CaptionRenderer/CaptionRenderer.tsx index e02568b49..e90239ee7 100644 --- a/src/components/CaptionRenderer/CaptionRenderer.tsx +++ b/src/components/CaptionRenderer/CaptionRenderer.tsx @@ -1,5 +1,6 @@ import React, { useCallback, useEffect, useState } from 'react'; import { makeStyles } from '@material-ui/core/styles'; +import Snackbar from '../Snackbar/Snackbar'; import { TwilioCaptionResult } from './CaptionTypes'; import { Typography } from '@material-ui/core'; import useParticipants from '../../hooks/useParticipants/useParticipants'; @@ -35,29 +36,34 @@ export function CaptionRenderer() { const participants = useParticipants(); const transcriberParticipant = participants.find(p => p.identity === 'media-transcriber'); const transcriberTracks = useParticipantTracks(transcriberParticipant); - const transcriberDataTrack = transcriberTracks.find(track => track.kind === 'data'); + const transcriberDataTrack = transcriberTracks.find( + track => track.kind === 'data' && track.name !== 'transcriber-error' + ); + const transcriberError = transcriberTracks.find(track => track.kind === 'data' && track.name === 'transcriber-error'); const { displayCaptions } = useAppState(); - const registerResult = useCallback((result: TwilioCaptionResult) => { - if (result.transcriptionResponse.TranscriptEvent.Transcript.Results.length) { - const transcript = result.transcriptionResponse.TranscriptEvent.Transcript.Results[0].Alternatives[0].Transcript; - const id = result.transcriptionResponse.TranscriptEvent.Transcript.Results[0].ResultId; - const timestamp = Date.now(); - const identity = result.participantIdentity; + const registerResult = useCallback((captionResult: TwilioCaptionResult) => { + if (captionResult.transcriptionResponse.TranscriptEvent.Transcript.Results.length) { + captionResult.transcriptionResponse.TranscriptEvent.Transcript.Results.forEach(result => { + const transcript = result.Alternatives[0].Transcript; + const id = result.ResultId; + const timestamp = Date.now(); + const identity = result.Identity; - setCaptions(prevCaptions => { - // Make a copy of the caption array, keeping only the 4 most recent captions - const arrayCopy = prevCaptions.slice(-4); + setCaptions(prevCaptions => { + // Make a copy of the caption array, keeping only the 4 most recent captions + const arrayCopy = prevCaptions.slice(-4); - const existingID = arrayCopy.find(item => item.id === id); - if (existingID) { - const existingIdIndex = arrayCopy.indexOf(existingID); - arrayCopy[existingIdIndex] = { transcript, id, timestamp, identity }; - } else { - arrayCopy.push({ transcript, id, timestamp, identity }); - } + const existingID = arrayCopy.find(item => item.id === id); + if (existingID) { + const existingIdIndex = arrayCopy.indexOf(existingID); + arrayCopy[existingIdIndex] = { transcript, id, timestamp, identity }; + } else { + arrayCopy.push({ transcript, id, timestamp, identity }); + } - return arrayCopy; + return arrayCopy; + }); }); } }, []); @@ -101,6 +107,12 @@ export function CaptionRenderer() { return (
+ {captions.map(caption => (
diff --git a/src/components/CaptionRenderer/CaptionTypes.ts b/src/components/CaptionRenderer/CaptionTypes.ts index f956a2631..f36b051ac 100644 --- a/src/components/CaptionRenderer/CaptionTypes.ts +++ b/src/components/CaptionRenderer/CaptionTypes.ts @@ -1,6 +1,5 @@ export interface TwilioCaptionResult { transcriptionResponse: TranscriptionResponse; - participantIdentity: string; } export interface TranscriptionResponse { @@ -21,6 +20,7 @@ export interface Result { IsPartial: boolean; ResultId: string; StartTime: number; + Identity: string; } export interface Alternative { From 3d89af7afeb909b3b2710348118e5a632a6713ef Mon Sep 17 00:00:00 2001 From: Tim Mendoza Date: Thu, 8 Sep 2022 17:09:25 -0600 Subject: [PATCH 8/8] Add chrome fix for AudioLevelIndicator --- .../AudioLevelIndicator.tsx | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/components/AudioLevelIndicator/AudioLevelIndicator.tsx b/src/components/AudioLevelIndicator/AudioLevelIndicator.tsx index cd1cd637f..2927b0a0b 100644 --- a/src/components/AudioLevelIndicator/AudioLevelIndicator.tsx +++ b/src/components/AudioLevelIndicator/AudioLevelIndicator.tsx @@ -9,10 +9,9 @@ const getUniqueClipId = () => clipId++; // @ts-ignore const AudioContext = window.AudioContext || window.webkitAudioContext; -let audioContext: AudioContext; export function initializeAnalyser(stream: MediaStream) { - audioContext = audioContext || new AudioContext(); + const audioContext = new AudioContext(); // Create a new audioContext for each audio indicator const audioSource = audioContext.createMediaStreamSource(stream); const analyser = audioContext.createAnalyser(); @@ -20,9 +19,20 @@ export function initializeAnalyser(stream: MediaStream) { analyser.fftSize = 256; audioSource.connect(analyser); + + // Here we provide a way for the audioContext to be closed. + // Closing the audioContext allows the unused audioSource to be garbage collected. + stream.addEventListener('cleanup', () => { + if (audioContext.state !== 'closed') { + audioContext.close(); + } + }); + return analyser; } +const isIOS = /iPhone|iPad/.test(navigator.userAgent); + function AudioLevelIndicator({ audioTrack, color = 'white' }: { audioTrack?: AudioTrack; color?: string }) { const SVGRectRef = useRef(null); const [analyser, setAnalyser] = useState(); @@ -33,19 +43,27 @@ function AudioLevelIndicator({ audioTrack, color = 'white' }: { audioTrack?: Aud if (audioTrack && mediaStreamTrack && isTrackEnabled) { // Here we create a new MediaStream from a clone of the mediaStreamTrack. // A clone is created to allow multiple instances of this component for a single - // AudioTrack on iOS Safari. - let newMediaStream = new MediaStream([mediaStreamTrack.clone()]); + // AudioTrack on iOS Safari. We only clone the mediaStreamTrack on iOS. + let newMediaStream = new MediaStream([isIOS ? mediaStreamTrack.clone() : mediaStreamTrack]); // Here we listen for the 'stopped' event on the audioTrack. When the audioTrack is stopped, // we stop the cloned track that is stored in 'newMediaStream'. It is important that we stop // all tracks when they are not in use. Browsers like Firefox don't let you create a new stream // from a new audio device while the active audio device still has active tracks. - const stopAllMediaStreamTracks = () => newMediaStream.getTracks().forEach(track => track.stop()); + const stopAllMediaStreamTracks = () => { + if (isIOS) { + // If we are on iOS, then we want to stop the MediaStreamTrack that we have previously cloned. + // If we are not on iOS, then we do not stop the MediaStreamTrack since it is the original and still in use. + newMediaStream.getTracks().forEach(track => track.stop()); + } + newMediaStream.dispatchEvent(new Event('cleanup')); // Stop the audioContext + }; audioTrack.on('stopped', stopAllMediaStreamTracks); const reinitializeAnalyser = () => { stopAllMediaStreamTracks(); - newMediaStream = new MediaStream([mediaStreamTrack.clone()]); + // We only clone the mediaStreamTrack on iOS. + newMediaStream = new MediaStream([isIOS ? mediaStreamTrack.clone() : mediaStreamTrack]); setAnalyser(initializeAnalyser(newMediaStream)); };