From d3f069e76389c30347938ea41e2856239c30f322 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 6 Nov 2024 04:36:48 -0500 Subject: [PATCH] Keep tiles in a stable order (#2670) * Keep tiles in a stable order This introduces a new layer of abstraction on top of MediaViewModel: TileViewModel, which gives us a place to store data relating to tiles rather than their media, and also generally makes it easier to reason about tiles as they move about the call layout. I have created a class called TileStore to keep track of these tiles. This allows us to swap out the media shown on a tile as the spotlight speaker changes, and avoid moving tiles around unless they really need to jump between the visible/invisible regions of the layout. * Don't throttle spotlight updates Since we now assume that the spotlight and grid will be in sync (i.e. an active speaker in one will behave as an active speaker in the other), we don't want the spotlight to ever lag behind due to throttling. If this causes usability issues we should maybe look into making LiveKit's 'speaking' indicators less erratic first. * Make layout shifts due to a change in speaker less surprising Although we try now to avoid layout shifts due to the spotlight speaker changing wherever possible, a spotlight speaker coming from off screen can still trigger one. Let's shift the layout a bit more gracefully in this case. * Improve the tile ordering tests * Maximize the spotlight tile in portrait layout * Tell tiles whether they're actually visible in a more timely manner * Fix test * Fix speaking indicators logic * Improve readability of marbles * Fix test case --------- Co-authored-by: Hugh Nimmo-Smith --- src/grid/CallLayout.ts | 19 +- src/grid/Grid.tsx | 50 ++- src/grid/GridLayout.tsx | 35 +- src/grid/OneOnOneLayout.tsx | 21 +- src/grid/SpotlightExpandedLayout.tsx | 22 +- src/grid/SpotlightLandscapeLayout.tsx | 33 +- src/grid/SpotlightPortraitLayout.tsx | 38 +-- src/room/InCallView.tsx | 24 +- src/state/CallViewModel.test.ts | 454 +++++++++++++++++++------- src/state/CallViewModel.ts | 230 +++++++++---- src/state/GridLikeLayout.ts | 43 +++ src/state/OneOnOneLayout.ts | 32 ++ src/state/PipLayout.ts | 30 ++ src/state/SpotlightExpandedLayout.ts | 36 ++ src/state/TileStore.ts | 259 +++++++++++++++ src/state/TileViewModel.ts | 43 +++ src/tile/GridTile.test.tsx | 11 +- src/tile/GridTile.tsx | 23 +- src/tile/SpotlightTile.test.tsx | 5 +- src/tile/SpotlightTile.tsx | 51 +-- src/utils/iter.test.ts | 22 ++ src/utils/iter.ts | 36 ++ 22 files changed, 1178 insertions(+), 339 deletions(-) create mode 100644 src/state/GridLikeLayout.ts create mode 100644 src/state/OneOnOneLayout.ts create mode 100644 src/state/PipLayout.ts create mode 100644 src/state/SpotlightExpandedLayout.ts create mode 100644 src/state/TileStore.ts create mode 100644 src/state/TileViewModel.ts create mode 100644 src/utils/iter.test.ts create mode 100644 src/utils/iter.ts diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index 5e73b38b6..895af23f4 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -8,8 +8,8 @@ Please see LICENSE in the repository root for full details. import { BehaviorSubject, Observable } from "rxjs"; import { ComponentType } from "react"; -import { MediaViewModel, UserMediaViewModel } from "../state/MediaViewModel"; import { LayoutProps } from "./Grid"; +import { TileViewModel } from "../state/TileViewModel"; export interface Bounds { width: number; @@ -42,19 +42,6 @@ export interface CallLayoutInputs { pipAlignment: BehaviorSubject; } -export interface GridTileModel { - type: "grid"; - vm: UserMediaViewModel; -} - -export interface SpotlightTileModel { - type: "spotlight"; - vms: MediaViewModel[]; - maximised: boolean; -} - -export type TileModel = GridTileModel | SpotlightTileModel; - export interface CallLayoutOutputs { /** * Whether the scrolling layer of the layout should appear on top. @@ -63,11 +50,11 @@ export interface CallLayoutOutputs { /** * The visually fixed (non-scrolling) layer of the layout. */ - fixed: ComponentType>; + fixed: ComponentType>; /** * The layer of the layout that can overflow and be scrolled. */ - scrolling: ComponentType>; + scrolling: ComponentType>; } /** diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index 51d258e3b..2983357b1 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -24,6 +24,7 @@ import { createContext, forwardRef, memo, + useCallback, useContext, useEffect, useMemo, @@ -33,6 +34,8 @@ import { import useMeasure from "react-use-measure"; import classNames from "classnames"; import { logger } from "matrix-js-sdk/src/logger"; +import { useObservableEagerState } from "observable-hooks"; +import { fromEvent, map, startWith } from "rxjs"; import styles from "./Grid.module.css"; import { useMergedRefs } from "../useMergedRefs"; @@ -51,6 +54,7 @@ interface Tile { id: string; model: Model; onDrag: DragCallback | undefined; + setVisible: (visible: boolean) => void; } type PlacedTile = Tile & Rect; @@ -84,6 +88,7 @@ interface SlotProps extends Omit, "onDrag"> { id: string; model: Model; onDrag?: DragCallback; + onVisibilityChange?: (visible: boolean) => void; style?: CSSProperties; className?: string; } @@ -131,6 +136,11 @@ export function useUpdateLayout(): void { ); } +const windowHeightObservable = fromEvent(window, "resize").pipe( + startWith(null), + map(() => window.innerHeight), +); + export interface LayoutProps { ref: LegacyRef; model: LayoutModel; @@ -232,6 +242,7 @@ export function Grid< const [gridRoot, gridRef2] = useState(null); const gridRef = useMergedRefs(gridRef1, gridRef2); + const windowHeight = useObservableEagerState(windowHeightObservable); const [layoutRoot, setLayoutRoot] = useState(null); const [generation, setGeneration] = useState(null); const tiles = useInitial(() => new Map>()); @@ -239,12 +250,34 @@ export function Grid< const Slot: FC> = useMemo( () => - function Slot({ id, model, onDrag, style, className, ...props }) { + function Slot({ + id, + model, + onDrag, + onVisibilityChange, + style, + className, + ...props + }) { const ref = useRef(null); + const prevVisible = useRef(null); + const setVisible = useCallback( + (visible: boolean) => { + if ( + onVisibilityChange !== undefined && + visible !== prevVisible.current + ) { + onVisibilityChange(visible); + prevVisible.current = visible; + } + }, + [onVisibilityChange], + ); + useEffect(() => { - tiles.set(id, { id, model, onDrag }); + tiles.set(id, { id, model, onDrag, setVisible }); return (): void => void tiles.delete(id); - }, [id, model, onDrag]); + }, [id, model, onDrag, setVisible]); return (
Math.min(gridBounds.bottom, windowHeight) - gridBounds.top, + [gridBounds, windowHeight], + ); + + useEffect(() => { + for (const tile of placedTiles) + tile.setVisible(tile.y + tile.height <= visibleHeight); + }, [placedTiles, visibleHeight]); + // Drag state is stored in a ref rather than component state, because we use // react-spring's imperative API during gestures to improve responsiveness const dragState = useRef(null); diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index ec6937e47..3dc3bef1d 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -12,12 +12,7 @@ import { useObservableEagerState } from "observable-hooks"; import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; import styles from "./GridLayout.module.css"; import { useInitial } from "../useInitial"; -import { - CallLayout, - GridTileModel, - TileModel, - arrangeTiles, -} from "./CallLayout"; +import { CallLayout, arrangeTiles } from "./CallLayout"; import { DragCallback, useUpdateLayout } from "./Grid"; interface GridCSSProperties extends CSSProperties { @@ -49,15 +44,6 @@ export const makeGridLayout: CallLayout = ({ ), ), ); - const tileModel: TileModel | undefined = useMemo( - () => - model.spotlight && { - type: "spotlight", - vms: model.spotlight, - maximised: false, - }, - [model.spotlight], - ); const onDragSpotlight: DragCallback = useCallback( ({ xRatio, yRatio }) => @@ -70,11 +56,11 @@ export const makeGridLayout: CallLayout = ({ return (
- {tileModel && ( + {model.spotlight && ( = ({ [width, minHeight, model.grid.length], ); - const tileModels: GridTileModel[] = useMemo( - () => model.grid.map((vm) => ({ type: "grid", vm })), - [model.grid], - ); - return (
= ({ } as GridCSSProperties } > - {tileModels.map((m) => ( - + {model.grid.map((m) => ( + ))}
); diff --git a/src/grid/OneOnOneLayout.tsx b/src/grid/OneOnOneLayout.tsx index 71db635d9..03ff5b32e 100644 --- a/src/grid/OneOnOneLayout.tsx +++ b/src/grid/OneOnOneLayout.tsx @@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel"; -import { CallLayout, GridTileModel, arrangeTiles } from "./CallLayout"; +import { CallLayout, arrangeTiles } from "./CallLayout"; import styles from "./OneOnOneLayout.module.css"; import { DragCallback, useUpdateLayout } from "./Grid"; @@ -38,15 +38,6 @@ export const makeOneOnOneLayout: CallLayout = ({ [width, height], ); - const remoteTileModel: GridTileModel = useMemo( - () => ({ type: "grid", vm: model.remote }), - [model.remote], - ); - const localTileModel: GridTileModel = useMemo( - () => ({ type: "grid", vm: model.local }), - [model.local], - ); - const onDragLocalTile: DragCallback = useCallback( ({ xRatio, yRatio }) => pipAlignment.next({ @@ -59,16 +50,18 @@ export const makeOneOnOneLayout: CallLayout = ({ return (
diff --git a/src/grid/SpotlightExpandedLayout.tsx b/src/grid/SpotlightExpandedLayout.tsx index 34464bcc7..084950360 100644 --- a/src/grid/SpotlightExpandedLayout.tsx +++ b/src/grid/SpotlightExpandedLayout.tsx @@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { forwardRef, useCallback, useMemo } from "react"; +import { forwardRef, useCallback } from "react"; import { useObservableEagerState } from "observable-hooks"; import { SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel"; -import { CallLayout, GridTileModel, SpotlightTileModel } from "./CallLayout"; +import { CallLayout } from "./CallLayout"; import { DragCallback, useUpdateLayout } from "./Grid"; import styles from "./SpotlightExpandedLayout.module.css"; @@ -27,17 +27,13 @@ export const makeSpotlightExpandedLayout: CallLayout< ref, ) { useUpdateLayout(); - const spotlightTileModel: SpotlightTileModel = useMemo( - () => ({ type: "spotlight", vms: model.spotlight, maximised: true }), - [model.spotlight], - ); return (
); @@ -50,11 +46,6 @@ export const makeSpotlightExpandedLayout: CallLayout< useUpdateLayout(); const pipAlignmentValue = useObservableEagerState(pipAlignment); - const pipTileModel: GridTileModel | undefined = useMemo( - () => model.pip && { type: "grid", vm: model.pip }, - [model.pip], - ); - const onDragPip: DragCallback = useCallback( ({ xRatio, yRatio }) => pipAlignment.next({ @@ -66,12 +57,13 @@ export const makeSpotlightExpandedLayout: CallLayout< return (
- {pipTileModel && ( + {model.pip && ( diff --git a/src/grid/SpotlightLandscapeLayout.tsx b/src/grid/SpotlightLandscapeLayout.tsx index 4132535a9..b9e6b2891 100644 --- a/src/grid/SpotlightLandscapeLayout.tsx +++ b/src/grid/SpotlightLandscapeLayout.tsx @@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { forwardRef, useMemo } from "react"; +import { forwardRef } from "react"; import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; -import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; +import { CallLayout } from "./CallLayout"; import { SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel"; import styles from "./SpotlightLandscapeLayout.module.css"; import { useUpdateLayout } from "./Grid"; @@ -30,19 +30,15 @@ export const makeSpotlightLandscapeLayout: CallLayout< ) { useUpdateLayout(); useObservableEagerState(minBounds); - const tileModel: TileModel = useMemo( - () => ({ - type: "spotlight", - vms: model.spotlight, - maximised: false, - }), - [model.spotlight], - ); return (
- +
@@ -55,25 +51,24 @@ export const makeSpotlightLandscapeLayout: CallLayout< ) { useUpdateLayout(); useObservableEagerState(minBounds); - const tileModels: GridTileModel[] = useMemo( - () => model.grid.map((vm) => ({ type: "grid", vm })), - [model.grid], - ); + const withIndicators = + useObservableEagerState(model.spotlight.media).length > 1; return (
1, + [styles.withIndicators]: withIndicators, })} />
- {tileModels.map((m) => ( + {model.grid.map((m) => ( ))}
diff --git a/src/grid/SpotlightPortraitLayout.tsx b/src/grid/SpotlightPortraitLayout.tsx index 56c5e07bf..e617160e9 100644 --- a/src/grid/SpotlightPortraitLayout.tsx +++ b/src/grid/SpotlightPortraitLayout.tsx @@ -5,16 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { CSSProperties, forwardRef, useMemo } from "react"; +import { CSSProperties, forwardRef } from "react"; import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; -import { - CallLayout, - GridTileModel, - TileModel, - arrangeTiles, -} from "./CallLayout"; +import { CallLayout, arrangeTiles } from "./CallLayout"; import { SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel"; import styles from "./SpotlightPortraitLayout.module.css"; import { useUpdateLayout } from "./Grid"; @@ -40,19 +35,15 @@ export const makeSpotlightPortraitLayout: CallLayout< ref, ) { useUpdateLayout(); - const tileModel: TileModel = useMemo( - () => ({ - type: "spotlight", - vms: model.spotlight, - maximised: true, - }), - [model.spotlight], - ); return (
- +
); @@ -71,10 +62,8 @@ export const makeSpotlightPortraitLayout: CallLayout< width, model.grid.length, ); - const tileModels: GridTileModel[] = useMemo( - () => model.grid.map((vm) => ({ type: "grid", vm })), - [model.grid], - ); + const withIndicators = + useObservableEagerState(model.spotlight.media).length > 1; return (
1, + [styles.withIndicators]: withIndicators, })} />
- {tileModels.map((m) => ( + {model.grid.map((m) => ( ))}
diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 9492b2f01..cd980234c 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -28,7 +28,7 @@ import { import useMeasure from "react-use-measure"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; -import { BehaviorSubject, of } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { logger } from "matrix-js-sdk/src/logger"; @@ -73,7 +73,6 @@ import { E2eeType } from "../e2ee/e2eeType"; import { makeGridLayout } from "../grid/GridLayout"; import { CallLayoutOutputs, - TileModel, defaultPipAlignment, defaultSpotlightAlignment, } from "../grid/CallLayout"; @@ -81,6 +80,7 @@ import { makeOneOnOneLayout } from "../grid/OneOnOneLayout"; import { makeSpotlightExpandedLayout } from "../grid/SpotlightExpandedLayout"; import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout"; import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout"; +import { GridTileViewModel, TileViewModel } from "../state/TileViewModel"; import { ReactionsProvider, useReactions } from "../useReactions"; import handSoundOgg from "../sound/raise_hand.ogg?url"; import handSoundMp3 from "../sound/raise_hand.mp3?url"; @@ -379,7 +379,7 @@ export const InCallView: FC = ({ () => forwardRef< HTMLDivElement, - PropsWithoutRef> + PropsWithoutRef> >(function Tile( { className, style, targetWidth, targetHeight, model }, ref, @@ -388,13 +388,6 @@ export const InCallView: FC = ({ const onToggleExpanded = useObservableEagerState( vm.toggleSpotlightExpanded, ); - const showVideo = useObservableEagerState( - useMemo( - () => - model.type === "grid" ? vm.showGridVideo(model.vm) : of(true), - [model], - ), - ); const showSpeakingIndicatorsValue = useObservableEagerState( vm.showSpeakingIndicators, ); @@ -402,23 +395,21 @@ export const InCallView: FC = ({ vm.showSpotlightIndicators, ); - return model.type === "grid" ? ( + return model instanceof GridTileViewModel ? ( ) : ( = ({ return ( [p.userId, p])); export interface GridLayoutSummary { type: "grid"; @@ -101,38 +109,71 @@ export type LayoutSummary = | OneOnOneLayoutSummary | PipLayoutSummary; -function summarizeLayout(l: Layout): LayoutSummary { - switch (l.type) { - case "grid": - return { - type: l.type, - spotlight: l.spotlight?.map((vm) => vm.id), - grid: l.grid.map((vm) => vm.id), - }; - case "spotlight-landscape": - case "spotlight-portrait": - return { - type: l.type, - spotlight: l.spotlight.map((vm) => vm.id), - grid: l.grid.map((vm) => vm.id), - }; - case "spotlight-expanded": - return { - type: l.type, - spotlight: l.spotlight.map((vm) => vm.id), - pip: l.pip?.id, - }; - case "one-on-one": - return { type: l.type, local: l.local.id, remote: l.remote.id }; - case "pip": - return { type: l.type, spotlight: l.spotlight.map((vm) => vm.id) }; - } +function summarizeLayout(l: Observable): Observable { + return l.pipe( + switchMap((l) => { + switch (l.type) { + case "grid": + return combineLatest( + [ + l.spotlight?.media ?? of(undefined), + ...l.grid.map((vm) => vm.media), + ], + (spotlight, ...grid) => ({ + type: l.type, + spotlight: spotlight?.map((vm) => vm.id), + grid: grid.map((vm) => vm.id), + }), + ); + case "spotlight-landscape": + case "spotlight-portrait": + return combineLatest( + [l.spotlight.media, ...l.grid.map((vm) => vm.media)], + (spotlight, ...grid) => ({ + type: l.type, + spotlight: spotlight.map((vm) => vm.id), + grid: grid.map((vm) => vm.id), + }), + ); + case "spotlight-expanded": + return combineLatest( + [l.spotlight.media, l.pip?.media ?? of(undefined)], + (spotlight, pip) => ({ + type: l.type, + spotlight: spotlight.map((vm) => vm.id), + pip: pip?.id, + }), + ); + case "one-on-one": + return combineLatest( + [l.local.media, l.remote.media], + (local, remote) => ({ + type: l.type, + local: local.id, + remote: remote.id, + }), + ); + case "pip": + return l.spotlight.media.pipe( + map((spotlight) => ({ + type: l.type, + spotlight: spotlight.map((vm) => vm.id), + })), + ); + } + }), + // Sometimes there can be multiple (synchronous) updates per frame. We only + // care about the most recent value for each time step, so discard these + // extra values. + debounceTime(0), + distinctUntilChanged(isEqual), + ); } function withCallViewModel( - { cold }: OurRunHelpers, remoteParticipants: Observable, connectionState: Observable, + speaking: Map>, continuation: (vm: CallViewModel) => void, ): void { const participantsSpy = vi @@ -141,15 +182,17 @@ function withCallViewModel( const mediaSpy = vi .spyOn(ComponentsCore, "observeParticipantMedia") .mockImplementation((p) => - cold("a", { - a: { participant: p } as Partial< - ComponentsCore.ParticipantMedia - > as ComponentsCore.ParticipantMedia, - }), + of({ participant: p } as Partial< + ComponentsCore.ParticipantMedia + > as ComponentsCore.ParticipantMedia), ); const eventsSpy = vi .spyOn(ComponentsCore, "observeParticipantEvents") - .mockImplementation((p) => cold("a", { a: p })); + .mockImplementation((p) => + (speaking.get(p) ?? of(false)).pipe( + map((s) => ({ ...p, isSpeaking: s }) as Participant), + ), + ); const vm = new CallViewModel( mockMatrixRoom({ @@ -176,107 +219,103 @@ function withCallViewModel( } test("participants are retained during a focus switch", () => { - withTestScheduler((helpers) => { - const { hot, expectObservable } = helpers; + withTestScheduler(({ cold, expectObservable }) => { // Participants disappear on frame 2 and come back on frame 3 - const partMarbles = "a-ba"; + const participantMarbles = "a-ba"; // Start switching focus on frame 1 and reconnect on frame 3 - const connMarbles = "ab-a"; + const connectionMarbles = " cs-c"; // The visible participants should remain the same throughout the switch - const laytMarbles = "aaaa 2997ms a 56998ms a"; + const layoutMarbles = " a"; withCallViewModel( - helpers, - hot(partMarbles, { + cold(participantMarbles, { a: [aliceParticipant, bobParticipant], b: [], }), - hot(connMarbles, { - a: ConnectionState.Connected, - b: ECAddonConnectionState.ECSwitchingFocus, + cold(connectionMarbles, { + c: ConnectionState.Connected, + s: ECAddonConnectionState.ECSwitchingFocus, }), + new Map(), (vm) => { - expectObservable(vm.layout.pipe(map(summarizeLayout))).toBe( - laytMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, - ); + }); }, ); }); }); test("screen sharing activates spotlight layout", () => { - withTestScheduler((helpers) => { - const { hot, schedule, expectObservable } = helpers; + withTestScheduler(({ cold, schedule, expectObservable }) => { // Start with no screen shares, then have Alice and Bob share their screens, // then return to no screen shares, then have just Alice share for a bit - const partMarbles = "abc---d---a-b---a"; + const participantMarbles = " abcda-ba"; // While there are no screen shares, switch to spotlight manually, and then // switch back to grid at the end - const modeMarbles = "-----------a--------b"; + const modeMarbles = " -----s--g"; // We should automatically enter spotlight for the first round of screen // sharing, then return to grid, then manually go into spotlight, and // remain in spotlight until we manually go back to grid - const laytMarbles = "ab(cc)(dd)ae(bb)(ee)a 59979ms a"; - // Speaking indicators should always be shown except for when the active - // speaker is present in the spotlight - const showMarbles = "y----------ny---n---y"; - + const layoutMarbles = " abcdaefeg"; + const showSpeakingMarbles = "y----nyny"; withCallViewModel( - helpers, - hot(partMarbles, { + cold(participantMarbles, { a: [aliceParticipant, bobParticipant], b: [aliceSharingScreen, bobParticipant], c: [aliceSharingScreen, bobSharingScreen], d: [aliceParticipant, bobSharingScreen], }), - hot("a", { a: ConnectionState.Connected }), + of(ConnectionState.Connected), + new Map(), (vm) => { schedule(modeMarbles, { - a: () => vm.setGridMode("spotlight"), - b: () => vm.setGridMode("grid"), + s: () => vm.setGridMode("spotlight"), + g: () => vm.setGridMode("grid"), }); - expectObservable(vm.layout.pipe(map(summarizeLayout))).toBe( - laytMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, - b: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0:screen-share`], - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, - c: { - type: "spotlight-landscape", - spotlight: [ - `${aliceId}:0:screen-share`, - `${bobId}:0:screen-share`, - ], - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, - d: { - type: "spotlight-landscape", - spotlight: [`${bobId}:0:screen-share`], - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, - e: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, - ); - expectObservable(vm.showSpeakingIndicators).toBe(showMarbles, { + b: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`], + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + c: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`, `${bobId}:0:screen-share`], + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "spotlight-landscape", + spotlight: [`${bobId}:0:screen-share`], + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + e: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: ["local:0", `${bobId}:0`], + }, + f: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`], + grid: ["local:0", `${bobId}:0`, `${aliceId}:0`], + }, + g: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${bobId}:0`, `${aliceId}:0`], + }, + }); + expectObservable(vm.showSpeakingIndicators).toBe(showSpeakingMarbles, { y: true, n: false, }); @@ -284,3 +323,200 @@ test("screen sharing activates spotlight layout", () => { ); }); }); + +test("participants stay in the same order unless to appear/disappear", () => { + withTestScheduler(({ cold, schedule, expectObservable }) => { + const modeMarbles = "a"; + // First Bob speaks, then Dave, then Alice + const aSpeakingMarbles = "n- 1998ms - 1999ms y"; + const bSpeakingMarbles = "ny 1998ms n 1999ms "; + const dSpeakingMarbles = "n- 1998ms y 1999ms n"; + // Nothing should change when Bob speaks, because Bob is already on screen. + // When Dave speaks he should switch with Alice because she's the one who + // hasn't spoken at all. Then when Alice speaks, she should return to her + // place at the top. + const layoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a"; + + withCallViewModel( + of([aliceParticipant, bobParticipant, daveParticipant]), + of(ConnectionState.Connected), + new Map([ + [aliceParticipant, cold(aSpeakingMarbles, { y: true, n: false })], + [bobParticipant, cold(bSpeakingMarbles, { y: true, n: false })], + [daveParticipant, cold(dSpeakingMarbles, { y: true, n: false })], + ]), + (vm) => { + schedule(modeMarbles, { + a: () => { + // We imagine that only three tiles (the first three) will be visible + // on screen at a time + vm.layout.subscribe((layout) => { + if (layout.type === "grid") { + for (let i = 0; i < layout.grid.length; i++) + layout.grid[i].setVisible(i < 3); + } + }); + }, + }); + + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`, `${daveId}:0`], + }, + b: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${daveId}:0`, `${bobId}:0`, `${aliceId}:0`], + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${daveId}:0`, `${bobId}:0`], + }, + }); + }, + ); + }); +}); + +test("spotlight speakers swap places", () => { + withTestScheduler(({ cold, schedule, expectObservable }) => { + // Go immediately into spotlight mode for the test + const modeMarbles = " s"; + // First Bob speaks, then Dave, then Alice + const aSpeakingMarbles = "n--y"; + const bSpeakingMarbles = "nyn"; + const dSpeakingMarbles = "n-yn"; + // Alice should start in the spotlight, then Bob, then Dave, then Alice + // again. However, the positions of Dave and Bob in the grid should be + // reversed by the end because they've been swapped in and out of the + // spotlight. + const layoutMarbles = " abcd"; + + withCallViewModel( + of([aliceParticipant, bobParticipant, daveParticipant]), + of(ConnectionState.Connected), + new Map([ + [aliceParticipant, cold(aSpeakingMarbles, { y: true, n: false })], + [bobParticipant, cold(bSpeakingMarbles, { y: true, n: false })], + [daveParticipant, cold(dSpeakingMarbles, { y: true, n: false })], + ]), + (vm) => { + schedule(modeMarbles, { s: () => vm.setGridMode("spotlight") }); + + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: ["local:0", `${bobId}:0`, `${daveId}:0`], + }, + b: { + type: "spotlight-landscape", + spotlight: [`${bobId}:0`], + grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], + }, + c: { + type: "spotlight-landscape", + spotlight: [`${daveId}:0`], + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: ["local:0", `${daveId}:0`, `${bobId}:0`], + }, + }); + }, + ); + }); +}); + +test("layout enters picture-in-picture mode when requested", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // Enable then disable picture-in-picture + const pipControlMarbles = "-ed"; + // Should go into picture-in-picture layout then back to grid + const layoutMarbles = " aba"; + + withCallViewModel( + of([aliceParticipant, bobParticipant]), + of(ConnectionState.Connected), + new Map(), + (vm) => { + schedule(pipControlMarbles, { + e: () => window.controls.enablePip(), + d: () => window.controls.disablePip(), + }); + + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + b: { + type: "pip", + spotlight: [`${aliceId}:0`], + }, + }); + }, + ); + }); +}); + +test("spotlight remembers whether it's expanded", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // Start in spotlight mode, then switch to grid and back to spotlight a + // couple times + const modeMarbles = " s-gs-gs"; + // Expand and collapse the spotlight + const expandMarbles = "-a--a"; + // Spotlight should stay expanded during the first mode switch, and stay + // collapsed during the second mode switch + const layoutMarbles = "abcbada"; + + withCallViewModel( + of([aliceParticipant, bobParticipant]), + of(ConnectionState.Connected), + new Map(), + (vm) => { + schedule(modeMarbles, { + s: () => vm.setGridMode("spotlight"), + g: () => vm.setGridMode("grid"), + }); + schedule(expandMarbles, { + a: () => { + let toggle: () => void; + vm.toggleSpotlightExpanded.subscribe((val) => (toggle = val!)); + toggle!(); + }, + }); + + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: ["local:0", `${bobId}:0`], + }, + b: { + type: "spotlight-expanded", + spotlight: [`${aliceId}:0`], + pip: "local:0", + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${bobId}:0`, `${aliceId}:0`], + }, + }); + }, + ); + }); +}); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 39453c601..37531511b 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -46,7 +46,6 @@ import { switchMap, switchScan, take, - throttleTime, timer, withLatestFrom, } from "rxjs"; @@ -70,6 +69,12 @@ import { ObservableScope } from "./ObservableScope"; import { duplicateTiles } from "../settings/settings"; import { isFirefox } from "../Platform"; import { setPipEnabled } from "../controls"; +import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel"; +import { TileStore } from "./TileStore"; +import { gridLikeLayout } from "./GridLikeLayout"; +import { spotlightExpandedLayout } from "./SpotlightExpandedLayout"; +import { oneOnOneLayout } from "./OneOnOneLayout"; +import { pipLayout } from "./PipLayout"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; // How long we wait after a focus switch before showing the real participant @@ -80,39 +85,82 @@ const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; // on mobile. No spotlight tile should be shown below this threshold. const smallMobileCallThreshold = 3; -export interface GridLayout { +export interface GridLayoutMedia { type: "grid"; spotlight?: MediaViewModel[]; grid: UserMediaViewModel[]; } -export interface SpotlightLandscapeLayout { +export interface SpotlightLandscapeLayoutMedia { type: "spotlight-landscape"; spotlight: MediaViewModel[]; grid: UserMediaViewModel[]; } -export interface SpotlightPortraitLayout { +export interface SpotlightPortraitLayoutMedia { type: "spotlight-portrait"; spotlight: MediaViewModel[]; grid: UserMediaViewModel[]; } -export interface SpotlightExpandedLayout { +export interface SpotlightExpandedLayoutMedia { type: "spotlight-expanded"; spotlight: MediaViewModel[]; pip?: UserMediaViewModel; } +export interface OneOnOneLayoutMedia { + type: "one-on-one"; + local: UserMediaViewModel; + remote: UserMediaViewModel; +} + +export interface PipLayoutMedia { + type: "pip"; + spotlight: MediaViewModel[]; +} + +export type LayoutMedia = + | GridLayoutMedia + | SpotlightLandscapeLayoutMedia + | SpotlightPortraitLayoutMedia + | SpotlightExpandedLayoutMedia + | OneOnOneLayoutMedia + | PipLayoutMedia; + +export interface GridLayout { + type: "grid"; + spotlight?: SpotlightTileViewModel; + grid: GridTileViewModel[]; +} + +export interface SpotlightLandscapeLayout { + type: "spotlight-landscape"; + spotlight: SpotlightTileViewModel; + grid: GridTileViewModel[]; +} + +export interface SpotlightPortraitLayout { + type: "spotlight-portrait"; + spotlight: SpotlightTileViewModel; + grid: GridTileViewModel[]; +} + +export interface SpotlightExpandedLayout { + type: "spotlight-expanded"; + spotlight: SpotlightTileViewModel; + pip?: GridTileViewModel; +} + export interface OneOnOneLayout { type: "one-on-one"; - local: LocalUserMediaViewModel; - remote: RemoteUserMediaViewModel; + local: GridTileViewModel; + remote: GridTileViewModel; } export interface PipLayout { type: "pip"; - spotlight: MediaViewModel[]; + spotlight: SpotlightTileViewModel; } /** @@ -161,6 +209,12 @@ enum SortingBin { SelfNotAlwaysShown, } +interface LayoutScanState { + layout: Layout | null; + tiles: TileStore; + visibleTiles: Set; +} + class UserMedia { private readonly scope = new ObservableScope(); public readonly vm: UserMediaViewModel; @@ -426,12 +480,6 @@ export class CallViewModel extends ViewModel { this.scope.state(), ); - private readonly hasRemoteScreenShares: Observable = - this.screenShares.pipe( - map((ms) => ms.some((m) => !m.vm.local)), - distinctUntilChanged(), - ); - private readonly spotlightSpeaker: Observable = this.userMedia.pipe( switchMap((mediaItems) => @@ -466,7 +514,6 @@ export class CallViewModel extends ViewModel { ), map((speaker) => speaker.vm), this.scope.state(), - throttleTime(1600, undefined, { leading: true, trailing: true }), ); private readonly grid: Observable = this.userMedia.pipe( @@ -536,6 +583,14 @@ export class CallViewModel extends ViewModel { this.scope.state(), ); + private readonly hasRemoteScreenShares: Observable = + this.spotlight.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), + distinctUntilChanged(), + ); + private readonly pip: Observable = this.spotlightAndPip.pipe(switchMap(([, pip]) => pip)); @@ -616,7 +671,7 @@ export class CallViewModel extends ViewModel { screenShares.length === 0, ); - private readonly gridLayout: Observable = combineLatest( + private readonly gridLayout: Observable = combineLatest( [this.grid, this.spotlight], (grid, spotlight) => ({ type: "grid", @@ -627,38 +682,44 @@ export class CallViewModel extends ViewModel { }), ); - private readonly spotlightLandscapeLayout: Observable = combineLatest( - [this.grid, this.spotlight], - (grid, spotlight) => ({ type: "spotlight-landscape", spotlight, grid }), - ); + private readonly spotlightLandscapeLayout: Observable = + combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); - private readonly spotlightPortraitLayout: Observable = combineLatest( - [this.grid, this.spotlight], - (grid, spotlight) => ({ type: "spotlight-portrait", spotlight, grid }), - ); + private readonly spotlightPortraitLayout: Observable = + combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); - private readonly spotlightExpandedLayout: Observable = combineLatest( - [this.spotlight, this.pip], - (spotlight, pip) => ({ + private readonly spotlightExpandedLayout: Observable = + combineLatest([this.spotlight, this.pip], (spotlight, pip) => ({ type: "spotlight-expanded", spotlight, pip: pip ?? undefined, - }), - ); + })); - private readonly oneOnOneLayout: Observable = this.grid.pipe( - map((grid) => ({ - type: "one-on-one", - local: grid.find((vm) => vm.local) as LocalUserMediaViewModel, - remote: grid.find((vm) => !vm.local) as RemoteUserMediaViewModel, - })), - ); + private readonly oneOnOneLayout: Observable = + this.mediaItems.pipe( + map((grid) => ({ + type: "one-on-one", + local: grid.find((vm) => vm.vm.local)!.vm as LocalUserMediaViewModel, + remote: grid.find((vm) => !vm.vm.local)!.vm as RemoteUserMediaViewModel, + })), + ); - private readonly pipLayout: Observable = this.spotlight.pipe( + private readonly pipLayout: Observable = this.spotlight.pipe( map((spotlight) => ({ type: "pip", spotlight })), ); - public readonly layout: Observable = this.windowMode.pipe( + /** + * The media to be used to produce a layout. + */ + private readonly layoutMedia: Observable = this.windowMode.pipe( switchMap((windowMode) => { switch (windowMode) { case "normal": @@ -719,48 +780,95 @@ export class CallViewModel extends ViewModel { this.scope.state(), ); + /** + * The layout of tiles in the call interface. + */ + public readonly layout: Observable = this.layoutMedia.pipe( + // Each layout will produce a set of tiles, and these tiles have an + // observable indicating whether they're visible. We loop this information + // back into the layout process by using switchScan. + switchScan< + LayoutMedia, + LayoutScanState, + Observable + >( + ({ tiles: prevTiles, visibleTiles }, media) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout(media, visibleTiles, prevTiles); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout( + media, + visibleTiles, + prevTiles, + ); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, visibleTiles, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, visibleTiles, prevTiles); + break; + } + + // Take all of the 'visible' observables and combine them into one big + // observable array + const visibilities = + newTiles.gridTiles.length === 0 + ? of([]) + : combineLatest(newTiles.gridTiles.map((tile) => tile.visible)); + return visibilities.pipe( + map((visibilities) => ({ + layout: layout, + tiles: newTiles, + visibleTiles: new Set( + newTiles.gridTiles.filter((_tile, i) => visibilities[i]), + ), + })), + ); + }, + { + layout: null, + tiles: TileStore.empty(), + visibleTiles: new Set(), + }, + ), + map(({ layout }) => layout), + this.scope.state(), + ); + public showSpotlightIndicators: Observable = this.layout.pipe( map((l) => l.type !== "grid"), this.scope.state(), ); - /** - * Determines whether video should be shown for a certain piece of media - * appearing in the grid. - */ - public showGridVideo(vm: MediaViewModel): Observable { - return this.layout.pipe( - map( - (l) => - !( - (l.type === "spotlight-landscape" || - l.type === "spotlight-portrait") && - // This media is already visible in the spotlight; avoid duplication - l.spotlight.some((spotlightVm) => spotlightVm === vm) - ), - ), - distinctUntilChanged(), - ); - } - public showSpeakingIndicators: Observable = this.layout.pipe( - map((l) => { + switchMap((l) => { switch (l.type) { case "spotlight-landscape": case "spotlight-portrait": // If the spotlight is showing the active speaker, we can do without // speaking indicators as they're a redundant visual cue. But if // screen sharing feeds are in the spotlight we still need them. - return l.spotlight[0] instanceof ScreenShareViewModel; + return l.spotlight.media.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), + ); // In expanded spotlight layout, the active speaker is always shown in // the picture-in-picture tile so there is no need for speaking // indicators. And in one-on-one layout there's no question as to who is // speaking. case "spotlight-expanded": case "one-on-one": - return false; + return of(false); default: - return true; + return of(true); } }), this.scope.state(), diff --git a/src/state/GridLikeLayout.ts b/src/state/GridLikeLayout.ts new file mode 100644 index 000000000..7fcada952 --- /dev/null +++ b/src/state/GridLikeLayout.ts @@ -0,0 +1,43 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { Layout, LayoutMedia } from "./CallViewModel"; +import { TileStore } from "./TileStore"; +import { GridTileViewModel } from "./TileViewModel"; + +export type GridLikeLayoutType = + | "grid" + | "spotlight-landscape" + | "spotlight-portrait"; + +/** + * Produces a grid-like layout (any layout with a grid and possibly a spotlight) + * with the given media. + */ +export function gridLikeLayout( + media: LayoutMedia & { type: GridLikeLayoutType }, + visibleTiles: Set, + prevTiles: TileStore, +): [Layout & { type: GridLikeLayoutType }, TileStore] { + const update = prevTiles.from(visibleTiles); + if (media.spotlight !== undefined) + update.registerSpotlight( + media.spotlight, + media.type === "spotlight-portrait", + ); + for (const mediaVm of media.grid) update.registerGridTile(mediaVm); + const tiles = update.build(); + + return [ + { + type: media.type, + spotlight: tiles.spotlightTile, + grid: tiles.gridTiles, + } as Layout & { type: GridLikeLayoutType }, + tiles, + ]; +} diff --git a/src/state/OneOnOneLayout.ts b/src/state/OneOnOneLayout.ts new file mode 100644 index 000000000..29ed9fc08 --- /dev/null +++ b/src/state/OneOnOneLayout.ts @@ -0,0 +1,32 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { OneOnOneLayout, OneOnOneLayoutMedia } from "./CallViewModel"; +import { TileStore } from "./TileStore"; +import { GridTileViewModel } from "./TileViewModel"; + +/** + * Produces a one-on-one layout with the given media. + */ +export function oneOnOneLayout( + media: OneOnOneLayoutMedia, + visibleTiles: Set, + prevTiles: TileStore, +): [OneOnOneLayout, TileStore] { + const update = prevTiles.from(visibleTiles); + update.registerGridTile(media.local); + update.registerGridTile(media.remote); + const tiles = update.build(); + return [ + { + type: media.type, + local: tiles.gridTilesByMedia.get(media.local)!, + remote: tiles.gridTilesByMedia.get(media.remote)!, + }, + tiles, + ]; +} diff --git a/src/state/PipLayout.ts b/src/state/PipLayout.ts new file mode 100644 index 000000000..35edeefe7 --- /dev/null +++ b/src/state/PipLayout.ts @@ -0,0 +1,30 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { PipLayout, PipLayoutMedia } from "./CallViewModel"; +import { TileStore } from "./TileStore"; +import { GridTileViewModel } from "./TileViewModel"; + +/** + * Produces a picture-in-picture layout with the given media. + */ +export function pipLayout( + media: PipLayoutMedia, + visibleTiles: Set, + prevTiles: TileStore, +): [PipLayout, TileStore] { + const update = prevTiles.from(visibleTiles); + update.registerSpotlight(media.spotlight, true); + const tiles = update.build(); + return [ + { + type: media.type, + spotlight: tiles.spotlightTile!, + }, + tiles, + ]; +} diff --git a/src/state/SpotlightExpandedLayout.ts b/src/state/SpotlightExpandedLayout.ts new file mode 100644 index 000000000..83c5a95e3 --- /dev/null +++ b/src/state/SpotlightExpandedLayout.ts @@ -0,0 +1,36 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { + SpotlightExpandedLayout, + SpotlightExpandedLayoutMedia, +} from "./CallViewModel"; +import { TileStore } from "./TileStore"; +import { GridTileViewModel } from "./TileViewModel"; + +/** + * Produces an expanded spotlight layout with the given media. + */ +export function spotlightExpandedLayout( + media: SpotlightExpandedLayoutMedia, + visibleTiles: Set, + prevTiles: TileStore, +): [SpotlightExpandedLayout, TileStore] { + const update = prevTiles.from(visibleTiles); + update.registerSpotlight(media.spotlight, true); + if (media.pip !== undefined) update.registerGridTile(media.pip); + const tiles = update.build(); + + return [ + { + type: media.type, + spotlight: tiles.spotlightTile!, + pip: tiles.gridTiles[0], + }, + tiles, + ]; +} diff --git a/src/state/TileStore.ts b/src/state/TileStore.ts new file mode 100644 index 000000000..0288830c7 --- /dev/null +++ b/src/state/TileStore.ts @@ -0,0 +1,259 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject } from "rxjs"; + +import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel"; +import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel"; +import { fillGaps } from "../utils/iter"; + +class SpotlightTileData { + private readonly media_: BehaviorSubject; + public get media(): MediaViewModel[] { + return this.media_.value; + } + public set media(value: MediaViewModel[]) { + this.media_.next(value); + } + + private readonly maximised_: BehaviorSubject; + public get maximised(): boolean { + return this.maximised_.value; + } + public set maximised(value: boolean) { + this.maximised_.next(value); + } + + public readonly vm: SpotlightTileViewModel; + + public constructor(media: MediaViewModel[], maximised: boolean) { + this.media_ = new BehaviorSubject(media); + this.maximised_ = new BehaviorSubject(maximised); + this.vm = new SpotlightTileViewModel(this.media_, this.maximised_); + } + + public destroy(): void { + this.vm.destroy(); + } +} + +class GridTileData { + private readonly media_: BehaviorSubject; + public get media(): UserMediaViewModel { + return this.media_.value; + } + public set media(value: UserMediaViewModel) { + this.media_.next(value); + } + + public readonly vm: GridTileViewModel; + + public constructor(media: UserMediaViewModel) { + this.media_ = new BehaviorSubject(media); + this.vm = new GridTileViewModel(this.media_); + } + + public destroy(): void { + this.vm.destroy(); + } +} + +/** + * A collection of tiles to be mapped to a layout. + */ +export class TileStore { + private constructor( + private readonly spotlight: SpotlightTileData | null, + private readonly grid: GridTileData[], + ) {} + + public readonly spotlightTile = this.spotlight?.vm; + public readonly gridTiles = this.grid.map(({ vm }) => vm); + public readonly gridTilesByMedia = new Map( + this.grid.map(({ vm, media }) => [media, vm]), + ); + + /** + * Creates an an empty collection of tiles. + */ + public static empty(): TileStore { + return new TileStore(null, []); + } + + /** + * Creates a builder which can be used to update the collection, passing + * ownership of the tiles to the updated collection. + */ + public from(visibleTiles: Set): TileStoreBuilder { + return new TileStoreBuilder( + this.spotlight, + this.grid, + (spotlight, grid) => new TileStore(spotlight, grid), + visibleTiles, + ); + } +} + +/** + * A builder for a new collection of tiles. Will reuse tiles and destroy unused + * tiles from a previous collection where appropriate. + */ +export class TileStoreBuilder { + private spotlight: SpotlightTileData | null = null; + private readonly prevSpotlightSpeaker = + this.prevSpotlight?.media.length === 1 && + "speaking" in this.prevSpotlight.media[0] && + this.prevSpotlight.media[0]; + + private readonly prevGridByMedia = new Map( + this.prevGrid.map((entry, i) => [entry.media, [entry, i]] as const), + ); + + // The total number of grid entries that we have so far + private numGridEntries = 0; + // A sparse array of grid entries which should be kept in the same spots as + // which they appeared in the previous grid + private readonly stationaryGridEntries: GridTileData[] = new Array( + this.prevGrid.length, + ); + // Grid entries which should now enter the visible section of the grid + private readonly visibleGridEntries: GridTileData[] = []; + // Grid entries which should now enter the invisible section of the grid + private readonly invisibleGridEntries: GridTileData[] = []; + + public constructor( + private readonly prevSpotlight: SpotlightTileData | null, + private readonly prevGrid: GridTileData[], + private readonly construct: ( + spotlight: SpotlightTileData | null, + grid: GridTileData[], + ) => TileStore, + private readonly visibleTiles: Set, + ) {} + + /** + * Sets the contents of the spotlight tile. If this is never called, there + * will be no spotlight tile. + */ + public registerSpotlight(media: MediaViewModel[], maximised: boolean): void { + if (this.spotlight !== null) throw new Error("Spotlight already set"); + if (this.numGridEntries > 0) + throw new Error("Spotlight must be registered before grid tiles"); + + // Reuse the previous spotlight tile if it exists + if (this.prevSpotlight === null) { + this.spotlight = new SpotlightTileData(media, maximised); + } else { + this.spotlight = this.prevSpotlight; + this.spotlight.media = media; + this.spotlight.maximised = maximised; + } + } + + /** + * Sets up a grid tile for the given media. If this is never called for some + * media, then that media will have no grid tile. + */ + public registerGridTile(media: UserMediaViewModel): void { + if (this.spotlight !== null) { + // We actually *don't* want spotlight speakers to appear in both the + // spotlight and the grid, so they're filtered out here + if (!media.local && this.spotlight.media.includes(media)) return; + // When the spotlight speaker changes, we would see one grid tile appear + // and another grid tile disappear. This would be an undesirable layout + // shift, so instead what we do is take the speaker's grid tile and swap + // the media out, so it can remain where it is in the layout. + if ( + media === this.prevSpotlightSpeaker && + this.spotlight.media.length === 1 && + "speaking" in this.spotlight.media[0] && + this.prevSpotlightSpeaker !== this.spotlight.media[0] + ) { + const prev = this.prevGridByMedia.get(this.spotlight.media[0]); + if (prev !== undefined) { + const [entry, prevIndex] = prev; + const previouslyVisible = this.visibleTiles.has(entry.vm); + const nowVisible = this.visibleTiles.has( + this.prevGrid[this.numGridEntries]?.vm, + ); + + // If it doesn't need to move between the visible/invisible sections of + // the grid, then we can keep it where it was and swap the media + if (previouslyVisible === nowVisible) { + this.stationaryGridEntries[prevIndex] = entry; + // Do the media swap + entry.media = media; + this.prevGridByMedia.delete(this.spotlight.media[0]); + this.prevGridByMedia.set(media, prev); + } else { + // Create a new tile; this will cause a layout shift but I'm not + // sure there's any other straightforward option in this case + (nowVisible + ? this.visibleGridEntries + : this.invisibleGridEntries + ).push(new GridTileData(media)); + } + + this.numGridEntries++; + return; + } + } + } + + // Was there previously a tile with this same media? + const prev = this.prevGridByMedia.get(media); + if (prev === undefined) { + // Create a new tile + (this.visibleTiles.has(this.prevGrid[this.numGridEntries]?.vm) + ? this.visibleGridEntries + : this.invisibleGridEntries + ).push(new GridTileData(media)); + } else { + // Reuse the existing tile + const [entry, prevIndex] = prev; + const previouslyVisible = this.visibleTiles.has(entry.vm); + const nowVisible = this.visibleTiles.has( + this.prevGrid[this.numGridEntries]?.vm, + ); + // If it doesn't need to move between the visible/invisible sections of + // the grid, then we can keep it exactly where it was previously + if (previouslyVisible === nowVisible) + this.stationaryGridEntries[prevIndex] = entry; + // Otherwise, queue this tile to be moved + else + (nowVisible ? this.visibleGridEntries : this.invisibleGridEntries).push( + entry, + ); + } + + this.numGridEntries++; + } + + /** + * Constructs a new collection of all registered tiles, transferring ownership + * of the tiles to the new collection. Any tiles present in the previous + * collection but not the new collection will be destroyed. + */ + public build(): TileStore { + // Piece together the grid + const grid = [ + ...fillGaps(this.stationaryGridEntries, [ + ...this.visibleGridEntries, + ...this.invisibleGridEntries, + ]), + ]; + + // Destroy unused tiles + if (this.spotlight === null && this.prevSpotlight !== null) + this.prevSpotlight.destroy(); + const gridEntries = new Set(grid); + for (const entry of this.prevGrid) + if (!gridEntries.has(entry)) entry.destroy(); + + return this.construct(this.spotlight, grid); + } +} diff --git a/src/state/TileViewModel.ts b/src/state/TileViewModel.ts new file mode 100644 index 000000000..3c25907ea --- /dev/null +++ b/src/state/TileViewModel.ts @@ -0,0 +1,43 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject, Observable } from "rxjs"; + +import { ViewModel } from "./ViewModel"; +import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel"; + +let nextId = 0; +function createId(): string { + return (nextId++).toString(); +} + +export class GridTileViewModel extends ViewModel { + public readonly id = createId(); + + private readonly visible_ = new BehaviorSubject(false); + /** + * Whether the tile is visible within the current viewport. + */ + public readonly visible: Observable = this.visible_; + + public setVisible = (value: boolean): void => this.visible_.next(value); + + public constructor(public readonly media: Observable) { + super(); + } +} + +export class SpotlightTileViewModel extends ViewModel { + public constructor( + public readonly media: Observable, + public readonly maximised: Observable, + ) { + super(); + } +} + +export type TileViewModel = GridTileViewModel | SpotlightTileViewModel; diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 0bf6cab82..81e501102 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -9,12 +9,20 @@ import { RemoteTrackPublication } from "livekit-client"; import { test, expect } from "vitest"; import { render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; +import { of } from "rxjs"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import { GridTile } from "./GridTile"; import { withRemoteMedia } from "../utils/test"; +import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsProvider } from "../useReactions"; +global.IntersectionObserver = class MockIntersectionObserver { + public observe(): void {} + public unobserve(): void {} + public disconnect(): void {} +} as unknown as typeof IntersectionObserver; + test("GridTile is accessible", async () => { await withRemoteMedia( { @@ -42,11 +50,10 @@ test("GridTile is accessible", async () => { const { container } = render( {}} targetWidth={300} targetHeight={200} - showVideo showSpeakingIndicators /> , diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 3675e9a7c..8252d1086 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -10,6 +10,7 @@ import { ReactNode, forwardRef, useCallback, + useRef, useState, } from "react"; import { animated } from "@react-spring/web"; @@ -44,6 +45,8 @@ import { import { Slider } from "../Slider"; import { MediaView } from "./MediaView"; import { useLatest } from "../useLatest"; +import { GridTileViewModel } from "../state/TileViewModel"; +import { useMergedRefs } from "../useMergedRefs"; import { useReactions } from "../useReactions"; interface TileProps { @@ -52,7 +55,6 @@ interface TileProps { targetWidth: number; targetHeight: number; displayName: string; - showVideo: boolean; showSpeakingIndicators: boolean; } @@ -67,7 +69,6 @@ const UserMediaTile = forwardRef( ( { vm, - showVideo, showSpeakingIndicators, menuStart, menuEnd, @@ -119,7 +120,7 @@ const UserMediaTile = forwardRef( video={video} member={vm.member} unencryptedWarning={unencryptedWarning} - videoEnabled={videoEnabled && showVideo} + videoEnabled={videoEnabled} videoFit={cropVideo ? "cover" : "contain"} className={classNames(className, styles.tile, { [styles.speaking]: showSpeaking, @@ -277,25 +278,27 @@ const RemoteUserMediaTile = forwardRef< RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; interface GridTileProps { - vm: UserMediaViewModel; + vm: GridTileViewModel; onOpenProfile: (() => void) | null; targetWidth: number; targetHeight: number; className?: string; style?: ComponentProps["style"]; - showVideo: boolean; showSpeakingIndicators: boolean; } export const GridTile = forwardRef( - ({ vm, onOpenProfile, ...props }, ref) => { - const displayName = useDisplayName(vm); + ({ vm, onOpenProfile, ...props }, theirRef) => { + const ourRef = useRef(null); + const ref = useMergedRefs(ourRef, theirRef); + const media = useObservableEagerState(vm.media); + const displayName = useDisplayName(media); - if (vm instanceof LocalUserMediaViewModel) { + if (media instanceof LocalUserMediaViewModel) { return ( ( return ( diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index a0fbed45a..cedeea626 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -9,9 +9,11 @@ import { test, expect, vi } from "vitest"; import { isInaccessible, render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import userEvent from "@testing-library/user-event"; +import { of } from "rxjs"; import { SpotlightTile } from "./SpotlightTile"; import { withLocalMedia, withRemoteMedia } from "../utils/test"; +import { SpotlightTileViewModel } from "../state/TileViewModel"; global.IntersectionObserver = class MockIntersectionObserver { public observe(): void {} @@ -36,10 +38,9 @@ test("SpotlightTile is accessible", async () => { const toggleExpanded = vi.fn(); const { container } = render( = { + ref, videoEnabled, videoFit: cropVideo ? "cover" : "contain", ...props, }; return vm instanceof LocalUserMediaViewModel ? ( - + ) : ( ); @@ -175,8 +178,7 @@ const SpotlightItem = forwardRef( SpotlightItem.displayName = "SpotlightItem"; interface Props { - vms: MediaViewModel[]; - maximised: boolean; + vm: SpotlightTileViewModel; expanded: boolean; onToggleExpanded: (() => void) | null; targetWidth: number; @@ -189,8 +191,7 @@ interface Props { export const SpotlightTile = forwardRef( ( { - vms, - maximised, + vm, expanded, onToggleExpanded, targetWidth, @@ -204,12 +205,14 @@ export const SpotlightTile = forwardRef( const { t } = useTranslation(); const [root, ourRef] = useObservableRef(null); const ref = useMergedRefs(ourRef, theirRef); - const [visibleId, setVisibleId] = useState(vms[0].id); - const latestVms = useLatest(vms); + const maximised = useObservableEagerState(vm.maximised); + const media = useObservableEagerState(vm.media); + const [visibleId, setVisibleId] = useState(media[0].id); + const latestMedia = useLatest(media); const latestVisibleId = useLatest(visibleId); - const visibleIndex = vms.findIndex((vm) => vm.id === visibleId); + const visibleIndex = media.findIndex((vm) => vm.id === visibleId); const canGoBack = visibleIndex > 0; - const canGoToNext = visibleIndex !== -1 && visibleIndex < vms.length - 1; + const canGoToNext = visibleIndex !== -1 && visibleIndex < media.length - 1; // To keep track of which item is visible, we need an intersection observer // hooked up to the root element and the items. Because the items will run @@ -234,28 +237,30 @@ export const SpotlightTile = forwardRef( const [scrollToId, setScrollToId] = useReactiveState( (prev) => - prev == null || prev === visibleId || vms.every((vm) => vm.id !== prev) + prev == null || + prev === visibleId || + media.every((vm) => vm.id !== prev) ? null : prev, [visibleId], ); const onBackClick = useCallback(() => { - const vms = latestVms.current; - const visibleIndex = vms.findIndex( + const media = latestMedia.current; + const visibleIndex = media.findIndex( (vm) => vm.id === latestVisibleId.current, ); - if (visibleIndex > 0) setScrollToId(vms[visibleIndex - 1].id); - }, [latestVisibleId, latestVms, setScrollToId]); + if (visibleIndex > 0) setScrollToId(media[visibleIndex - 1].id); + }, [latestVisibleId, latestMedia, setScrollToId]); const onNextClick = useCallback(() => { - const vms = latestVms.current; - const visibleIndex = vms.findIndex( + const media = latestMedia.current; + const visibleIndex = media.findIndex( (vm) => vm.id === latestVisibleId.current, ); - if (visibleIndex !== -1 && visibleIndex !== vms.length - 1) - setScrollToId(vms[visibleIndex + 1].id); - }, [latestVisibleId, latestVms, setScrollToId]); + if (visibleIndex !== -1 && visibleIndex !== media.length - 1) + setScrollToId(media[visibleIndex + 1].id); + }, [latestVisibleId, latestMedia, setScrollToId]); const ToggleExpandIcon = expanded ? CollapseIcon : ExpandIcon; @@ -277,7 +282,7 @@ export const SpotlightTile = forwardRef( )}
- {vms.map((vm) => ( + {media.map((vm) => ( ( {!expanded && (
1, + [styles.show]: showIndicators && media.length > 1, })} > - {vms.map((vm) => ( + {media.map((vm) => (
{ + expect([ + ...fillGaps([1, undefined, undefined, undefined, 3], [2]), + ]).toStrictEqual([1, 2, 3]); +}); + +test("fillGaps adds extra filler elements to the end", () => { + expect([ + ...fillGaps([1, undefined, 3, undefined], [2, 4, 5, 6]), + ]).toStrictEqual([1, 2, 3, 4, 5, 6]); +}); diff --git a/src/utils/iter.ts b/src/utils/iter.ts new file mode 100644 index 000000000..cf40ae8fe --- /dev/null +++ b/src/utils/iter.ts @@ -0,0 +1,36 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +/** + * Fills in the 'undefined' gaps in a collection by drawing items from a second + * collection, or simply filtering out the gap if no items are left. If filler + * items remain at the end, they will be appended to the resulting collection. + */ +export function fillGaps( + gappy: Iterable, + filler: Iterable, +): Iterable { + return { + [Symbol.iterator](): Iterator { + const gappyIter = gappy[Symbol.iterator](); + const fillerIter = filler[Symbol.iterator](); + return { + next(): IteratorResult { + let gappyItem: IteratorResult; + do { + gappyItem = gappyIter.next(); + if (!gappyItem.done && gappyItem.value !== undefined) + return gappyItem as IteratorYieldResult; + const fillerItem = fillerIter.next(); + if (!fillerItem.done) return fillerItem; + } while (!gappyItem.done); + return gappyItem; + }, + }; + }, + }; +}