From e7baea98d843cdefd07aa429445cc2a160008582 Mon Sep 17 00:00:00 2001 From: Chris Klochek Date: Fri, 5 Jul 2024 16:33:36 -0400 Subject: [PATCH] HACK DO NOT SUBMIT. Integration example for replay videos. --- .../src/suspense/ScreenshotCache.ts | 15 +++++- packages/shared/client/ReplayClient.ts | 10 ++++ pages/_document.tsx | 2 + src/ui/components/Video/Video.tsx | 31 +++++++++--- .../Video/imperative/MutableGraphicsState.tsx | 48 ++++++++++++++++++- .../Video/imperative/updateGraphics.ts | 39 ++++++++++----- 6 files changed, 125 insertions(+), 20 deletions(-) diff --git a/packages/replay-next/src/suspense/ScreenshotCache.ts b/packages/replay-next/src/suspense/ScreenshotCache.ts index 51b18022738..fed17cd4039 100644 --- a/packages/replay-next/src/suspense/ScreenshotCache.ts +++ b/packages/replay-next/src/suspense/ScreenshotCache.ts @@ -1,4 +1,4 @@ -import { ExecutionPoint, ScreenShot } from "@replayio/protocol"; +import { ExecutionPoint, ScreenShot, Video } from "@replayio/protocol"; import { createCache } from "suspense"; import { paintHashCache } from "replay-next/src/suspense/PaintHashCache"; @@ -19,3 +19,16 @@ export const screenshotCache = createCache< return screenShot; }, }); + +export const videoCache = createCache< + [replayClient: ReplayClientInterface], + Video[] +>({ + config: { immutable: true }, + debugLabel: "ScreenshotCache", + getKey: ([client]) => client.getRecordingId() || "", + load: async ([client]) => { + const videos = await client.getVideos(); + return videos; + }, +}); diff --git a/packages/shared/client/ReplayClient.ts b/packages/shared/client/ReplayClient.ts index af33e184ef7..dd4c649b681 100644 --- a/packages/shared/client/ReplayClient.ts +++ b/packages/shared/client/ReplayClient.ts @@ -45,6 +45,7 @@ import { TimeStampedPoint, TimeStampedPointRange, VariableMapping, + Video, annotations, createPauseResult, findPointsResults, @@ -810,6 +811,15 @@ export class ReplayClient implements ReplayClientInterface { return screen; } + async getVideos(): Promise { + const sessionId = await this.waitForSession(); + const { videos } = await client.Graphics.getVideos( + { mimeType: "video/webm" }, + sessionId + ); + return videos; + } + async mapExpressionToGeneratedScope(expression: string, location: Location): Promise { const sessionId = await this.waitForSession(); const result = await client.Debugger.mapExpressionToGeneratedScope( diff --git a/pages/_document.tsx b/pages/_document.tsx index 6f052a2cfe9..8f325535b57 100644 --- a/pages/_document.tsx +++ b/pages/_document.tsx @@ -48,6 +48,8 @@ const csp = (props: any) => { // Required to inline images from the database and from external avaters `img-src 'self' data: https:`, + `media-src 'self' data: https:`, + // Required for our logpoint analysis cache (which uses a Web worker) `worker-src 'self' blob:`, ] diff --git a/src/ui/components/Video/Video.tsx b/src/ui/components/Video/Video.tsx index a6caa454e8c..bf207c4e190 100644 --- a/src/ui/components/Video/Video.tsx +++ b/src/ui/components/Video/Video.tsx @@ -31,6 +31,8 @@ export default function Video() { useLayoutEffect(() => { const containerElement = document.getElementById("video") as HTMLDivElement; const graphicsElement = document.getElementById("graphics") as HTMLImageElement; + const videoElement = document.getElementById("webmvideo") as HTMLVideoElement; + const videoSourceElement = document.getElementById("videoSrc") as HTMLSourceElement; const graphicsOverlayElement = document.getElementById("overlay-graphics") as HTMLDivElement; let prevState: Partial = {}; @@ -38,15 +40,29 @@ export default function Video() { // Keep graphics in sync with the imperatively managed screenshot state state.listen(nextState => { - if (nextState.screenShot != prevState.screenShot) { - const { screenShot } = nextState; - if (screenShot) { - graphicsElement.src = `data:${screenShot.mimeType};base64,${screenShot.data}`; - } else { - graphicsElement.src = ""; + if (nextState.videos.length > 0 && !videoElement.src) { + graphicsElement.hidden = true; + videoElement.hidden = false; + videoElement.src = `data:video/webm;base64,${nextState.videos[0].data}`; + } else { + if (nextState.screenShot != prevState.screenShot) { + const { screenShot } = nextState; + if (screenShot) { + graphicsElement.src = `data:${screenShot.mimeType};base64,${screenShot.data}`; + graphicsElement.hidden = false; + videoElement.hidden = true; + } else { + graphicsElement.src = ""; + videoSourceElement.src = ""; + } } } + const { videos, paintIndex } = nextState; + if (videos.length > 0 && paintIndex !== null && paintIndex > 0) { + videoElement.currentTime = (paintIndex - 1) * (1.0 / videos[0].fps); + } + // Show loading progress bar if graphics stall for longer than 5s const isLoading = nextState.status === "loading"; const wasLoading = prevState.status === "loading"; @@ -116,6 +132,9 @@ export default function Video() { + {/* Graphics that are relative to the rendered screenshot go here; this container is automatically positioned to align with the screenshot */}
diff --git a/src/ui/components/Video/imperative/MutableGraphicsState.tsx b/src/ui/components/Video/imperative/MutableGraphicsState.tsx index 343bd996c13..6194af88b27 100644 --- a/src/ui/components/Video/imperative/MutableGraphicsState.tsx +++ b/src/ui/components/Video/imperative/MutableGraphicsState.tsx @@ -11,7 +11,7 @@ // // This approach is unusual, but it's arguably cleaner than sharing these values via the DOM. -import { ExecutionPoint, ScreenShot } from "@replayio/protocol"; +import { ExecutionPoint, ScreenShot, Video } from "@replayio/protocol"; import { fitImageToContainer, getDimensions } from "replay-next/src/utils/image"; import { shallowEqual } from "shared/utils/compare"; @@ -34,8 +34,10 @@ export interface State { localScale: number; recordingScale: number; screenShot: ScreenShot | undefined; + videos: Video[]; screenShotType: ScreenShotType | undefined; status: Status; + paintIndex: number | null; } export const state = createState({ @@ -52,6 +54,8 @@ export const state = createState({ screenShot: undefined, screenShotType: undefined, status: "loading", + paintIndex: null, + videos: [], }); let lock: Object | null = null; @@ -62,9 +66,11 @@ export async function updateState( didResize?: boolean; executionPoint: ExecutionPoint | null; screenShot: ScreenShot | null; + videos: Video[]; screenShotType: ScreenShotType | null; status: Status; time: number; + paintIndex: number | null; }> = {} ) { const prevState = state.read(); @@ -76,6 +82,8 @@ export async function updateState( screenShotType = prevState.screenShotType, status = prevState.status, time = prevState.currentTime, + paintIndex = prevState.paintIndex, + videos = prevState.videos, } = options; if (shallowEqual(options, { didResize })) { @@ -92,7 +100,41 @@ export async function updateState( let graphicsRect = prevState.graphicsRect; let localScale = prevState.localScale; let recordingScale = prevState.recordingScale; - if (screenShot && (screenShot != prevState.screenShot || didResize)) { + if (videos.length > 0) { + const naturalDimensions = { + aspectRatio: videos[0].width / videos[0].height, + height: videos[0].height, + width: videos[0].width, + } + + if (lock !== localLock) { + return; + } + + const naturalHeight = naturalDimensions.height; + const naturalWidth = naturalDimensions.width; + + const containerRect = graphicsElement.getBoundingClientRect(); + const scaledDimensions = fitImageToContainer({ + containerHeight: containerRect.height, + containerWidth: containerRect.width, + imageHeight: naturalHeight, + imageWidth: naturalWidth, + }); + + const clientHeight = scaledDimensions.height; + const clientWidth = scaledDimensions.width; + + localScale = clientWidth / naturalWidth; + recordingScale = 1.0; + + graphicsRect = { + height: clientHeight, + left: containerRect.left + (containerRect.width - clientWidth) / 2, + top: containerRect.top + (containerRect.height - clientHeight) / 2, + width: clientWidth, + }; + } else if (screenShot && (screenShot != prevState.screenShot || didResize)) { const naturalDimensions = await getDimensions(screenShot.data, screenShot.mimeType); if (lock !== localLock) { return; @@ -132,6 +174,8 @@ export async function updateState( screenShot: screenShot || undefined, screenShotType: screenShotType || undefined, status, + paintIndex, + videos }; if (!shallowEqual(prevState, nextState)) { diff --git a/src/ui/components/Video/imperative/updateGraphics.ts b/src/ui/components/Video/imperative/updateGraphics.ts index 5cfc3179db4..3b66c686664 100644 --- a/src/ui/components/Video/imperative/updateGraphics.ts +++ b/src/ui/components/Video/imperative/updateGraphics.ts @@ -1,9 +1,9 @@ import { ExecutionPoint, ScreenShot } from "@replayio/protocol"; -import { PaintsCache, findMostRecentPaint } from "protocol/PaintsCache"; +import { PaintsCache, findMostRecentPaint, findMostRecentPaintIndex } from "protocol/PaintsCache"; import { RepaintGraphicsCache } from "protocol/RepaintGraphicsCache"; import { paintHashCache } from "replay-next/src/suspense/PaintHashCache"; -import { screenshotCache } from "replay-next/src/suspense/ScreenshotCache"; +import { screenshotCache, videoCache } from "replay-next/src/suspense/ScreenshotCache"; import { ReplayClientInterface } from "shared/client/types"; import { updateState } from "ui/components/Video/imperative/MutableGraphicsState"; @@ -32,9 +32,11 @@ export async function updateGraphics({ } const promises: Promise[] = []; + const videos = await videoCache.readAsync(replayClient); // If the current time is before the first paint, we should show nothing const paintPoint = findMostRecentPaint(time); + const paintIndex = findMostRecentPaintIndex(time); const isBeforeFirstCachedPaint = !paintPoint || !paintPoint.paintHash; if (isBeforeFirstCachedPaint) { updateState(graphicsElement, { @@ -44,7 +46,7 @@ export async function updateGraphics({ status: executionPoint ? "loading" : "loaded", time, }); - } else { + } else if (videos.length == 0) { const cachedScreenShot = paintHashCache.getValueIfCached(paintPoint.paintHash); if (cachedScreenShot) { // If this screenshot has already been cached, skip fetching it again @@ -63,20 +65,34 @@ export async function updateGraphics({ } let repaintGraphicsScreenShot: ScreenShot | undefined = undefined; - if (executionPoint) { - const promise = fetchRepaintGraphics({ + // if (executionPoint) { + // const promise = fetchRepaintGraphics({ + // executionPoint, + // replayClient, + // time, + // }).then(screenShot => { + // repaintGraphicsScreenShot = screenShot; + + // return screenShot; + // }); + + // promises.push(promise); + // } + + if (videos.length > 0) { + updateState(graphicsElement, { executionPoint, - replayClient, + screenShotType: repaintGraphicsScreenShot != null ? "repaint" : "cached-paint", + status: "loaded", time, - }).then(screenShot => { - repaintGraphicsScreenShot = screenShot; - - return screenShot; + paintIndex, + videos }); - promises.push(promise); + return true; } + if (promises.length === 0) { // If we are before the first paint and have no execution point to request a repaint, // then we should clear out the currently visible graphics and bail out @@ -97,6 +113,7 @@ export async function updateGraphics({ screenShotType: repaintGraphicsScreenShot != null ? "repaint" : "cached-paint", status: "loaded", time, + paintIndex, }); if (repaintGraphicsScreenShot != null) {