WIP

package.json

@@ -13,7 +13,7 @@
   },
   "homepage": "https://github.com/WofWca/jumpcutter#readme",
   "scripts": {
-    "_abstract-build": "yarn lint && NODE_ENV=production webpack --mode=production",
+    "_abstract-build": "NODE_ENV=production webpack --mode=production",
     "build:gecko": "yarn run _abstract-build --env browser=gecko",
     "build:chromium": "yarn run _abstract-build --env browser=chromium",
     "_abstract-build-and-package": "yarn run build:$BROWSER -- --env noreport && cd dist-$BROWSER && rm -f ../dist-$BROWSER.zip && zip -r ../dist-$BROWSER.zip .",

src/entry-points/content/ElementPlaybackControllerCloning/ElementPlaybackControllerCloning.ts

@@ -188,6 +188,13 @@ export default class Controller {
 
   _didNotDoDesyncCorrectionForNSpeedSwitches = 0;
 
+  // TODO refactor: make this a constructor parameter for this Controller.
+  private readonly getMediaSourceCloneElement: ConstructorParameters<typeof Lookahead>[2] =
+    (originalElement) => import(
+      /* webpackExports: ['getMediaSourceCloneElement']*/
+      '@/entry-points/content/cloneMediaSources/getMediaSourceCloneElement'
+    ).then(({ getMediaSourceCloneElement }) => getMediaSourceCloneElement(originalElement));
+
   constructor(
     element: HTMLMediaElement,
     controllerSettings: ControllerSettings,
@@ -196,7 +203,11 @@ export default class Controller {
     this.element = element;
     this.settings = controllerSettings;
 
-    const lookahead = this.lookahead = new Lookahead(element, this.settings);
+    const lookahead = this.lookahead = new Lookahead(
+      element,
+      this.settings,
+      this.getMediaSourceCloneElement
+    );
     // Destruction is performed in `this.destroy` directly.
     lookahead.ensureInit();
 
@@ -677,7 +688,11 @@ export default class Controller {
   }
 
   private _initLookahead() {
-    const lookahead = this.lookahead = new Lookahead(this.element, this.settings);
+    const lookahead = this.lookahead = new Lookahead(
+      this.element,
+      this.settings,
+      this.getMediaSourceCloneElement
+    );
     // Destruction is performed in `this.destroy` directly.
     lookahead.ensureInit();
   }

src/entry-points/content/ElementPlaybackControllerCloning/Lookahead.ts

@@ -1,6 +1,6 @@
 /**
  * @license
- * Copyright (C) 2021, 2022  WofWca <[email protected]>
+ * Copyright (C) 2021, 2022, 2023  WofWca <[email protected]>
  *
  * This file is part of Jump Cutter Browser Extension.
  *
@@ -28,6 +28,17 @@ import SilenceDetectorNode, { SilenceDetectorEventType, SilenceDetectorMessage }
   from '@/entry-points/content/SilenceDetector/SilenceDetectorNode';
 import VolumeFilterNode from '@/entry-points/content/VolumeFilter/VolumeFilterNode';
 import lookaheadVolumeFilterSmoothing from './lookaheadVolumeFilterSmoothing.json'
+// import * as mediaSourcesCloning from '../cloneMediaSources';
+
+
+function getBufferedRanges(el: HTMLMediaElement) {
+  const arr = [];
+  for (let i = 0; i < el.buffered.length; i++) {
+    arr.push([el.buffered.start(i), el.buffered.end(i)]);
+  }
+  return arr;
+}
+
 
 // A more semantically correct version would be `Array<[start: MediaTime, end: MediaTime]>`,
 // but I think this is a bit faster.
@@ -59,7 +70,9 @@ type LookaheadSettings = Pick<ExtensionSettings, 'volumeThreshold' | 'marginBefo
 // and reduces time accuracy of silence ranges, which can cause it to miss short silence ranges.
 // This is practically the greatest effective absolute playbackRate one expects to achieve.
 // TODO improvement: turn this into an option.
-const maxClonePlaybackRate = Math.min(8, maxPlaybackRate);
+const maxClonePlaybackRate = Math.min(3, maxPlaybackRate);
+// TODO I don't think it should be about max playback rate. Rather about `buffered`.
+const maxClonePlaybackRateWhenMediaSourceSrc = Math.min(3, maxPlaybackRate);
 
 export default class Lookahead {
   clone: HTMLAudioElement; // Always <audio> for performance - so the browser doesn't have to decode video frames.
@@ -75,17 +88,26 @@ export default class Lookahead {
 
   private _resolveDestroyedPromise!: () => void;
   private _destroyedPromise = new Promise<void>(r => this._resolveDestroyedPromise = r);
+  /**
+   * @param getFallbackCloneElement a function that returns a clone element. It is used when
+   * the `Lookahead` could not reuse the same source as the original element. The current
+   * use case is when the original element uses `MediaSource`. The function may return the same
+   * clone element for different calls.
+   */
   constructor(
     private originalElement: HTMLMediaElement,
     private settings: LookaheadSettings,
     // public onNewSilenceRange: (start: Time, end: Time) => void,
+    private readonly getFallbackCloneElement:
+      undefined | ((originalElement: HTMLMediaElement) => Promise<HTMLAudioElement | undefined>)
   ) {
     const clone = document.createElement('audio');
     this.clone = clone;
 
     // TODO this probably doesn't cover all cases. Maybe it's better to just `originalElement.cloneNode(true)`?
     // TODO also need to watch for changes of `crossOrigin`
     // (in `ElementPlaybackControllerCloning.ts`).
+    // TODO wait, we gotta do the same for the `MediaSource` clone element, no?
     clone.crossOrigin = originalElement.crossOrigin;
     clone.src = originalElement.currentSrc;
 
@@ -115,10 +137,77 @@ export default class Lookahead {
   private async _init(): Promise<void> {
     const originalElement = this.originalElement;
 
-    const clone = this.clone;
+    // const clone = this.clone;
 
     const toAwait: Array<Promise<void>> = [];
 
+
+    // TODO what if `originalElement.srcObject`
+    // const mediaSourceFromSrcUrl =
+    //   mediaSourcesCloning.getMediaSourceFromObjectUrl(originalElement.src);
+
+    // URLs returned by `createObjectURL` are guaranteed to `.startsWith('blob:')`:
+    // https://w3c.github.io/FileAPI/#unicodeBlobURL
+    //
+    // TODO fuck. What if it's not `MediaSource` but a `File` (like it is in the
+    // "local-file-player") or a `Blob`? We could just use the same `src`. How do we determine
+    // which one it is? How about try to `fetch` it? If it's `MediaSource` the fetch will fail
+    // (based on my testing).
+    // Do we have to wait for the `MediaSource`-tracking script to return the response??
+    const isSrcObjectUrl = originalElement.src.startsWith('blob:')
+    if (isSrcObjectUrl) {
+      // Reusing the same `mediaSourceFromSrcUrl` for the clone element is not possible, see
+      // https://github.com/WofWca/jumpcutter/issues/2
+      toAwait.push(this.getFallbackCloneElement?.(originalElement)!.then(cloneEl => {
+        const clone = cloneEl!;
+        this.clone = clone;
+
+
+        // const cloneUrl = URL.createObjectURL(cloneMediaSource);
+        // clone.src = cloneUrl;
+        console.log(clone);
+        // TODO refactor: see the same line of code below.
+        // But make sure to `clone.src = ''` _before_ `revokeObjectURL`.
+
+        // this._destroyedPromise.then(() => clone.src = '');
+
+        // this._destroyedPromise.then(() => URL.revokeObjectURL(cloneUrl));
+
+        // // Yes, we could also do `clone.src = URL.createObjectURL(cloneMediaSource)`
+        // clone.srcObject = cloneMediaSource;
+      }));
+    } else {
+      this.clone.src = originalElement.currentSrc;
+      // TODO refactor: see the same line of code below.
+      this._destroyedPromise.then(() => clone.src = '');
+    }
+
+    await Promise.all(toAwait);
+    const clone = this.clone;
+
+
+    // Not doing this appears to cause a resource (memory and processing) leak in Chromium
+    // manifesting itself when creating new instances of Lookahead (and discarding the old ones).
+    // Looks like it's because
+    // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource
+    // > only once a media element is in a state where no further audio could ever be
+    // > played by that element may the element be garbage collected
+    // Here's the advice that tells us to do exactly this:
+    // https://html.spec.whatwg.org/multipage/media.html#best-practices-for-authors-using-media-elements
+    // > or, even better, by setting the element's src attribute to an empty string
+    // BTW, `clone.pause()` also works (sometimes?)
+    this._destroyedPromise.then(() => clone.src = '');
+
+
+
+
+    // clone.addEventListener('timeupdate', e => {
+    //   console.warn('timeupdate', clone.currentTime, clone.duration, getBufferedRanges(clone))
+    // })
+    console.log(originalElement, clone);
+    originalElement.addEventListener('error', e => console.error(e, e.target.error));
+    clone.addEventListener('error', e => console.error(e, e.target.error));
+
     const ctx = new AudioContext({
       latencyHint: 'playback',
     });
@@ -238,6 +327,9 @@ export default class Lookahead {
       : (seekTo: MediaTime) => { clone.currentTime = seekTo };
     const seekCloneIfOriginalElIsPlayingUnprocessedRange = () => {
       const originalElementTime = originalElement.currentTime;
+      // TODO fix: `clone.played` doesn't mean that we've actually played back that part.
+      // E.g. if you play a video in Odysee then reload the page such that the video starts
+      // playing from the middle then this is gonna say that it `played` it from start to middle.
       const playingUnprocessedRange = !inRanges(clone.played, originalElementTime);
       // Keep in mind that `originalElement.seeking` could be `true`, so make sure not to repeatedly
       // call this so that it gets stuck in that state.
@@ -291,11 +383,21 @@ export default class Lookahead {
         if (IS_DEV_MODE) {
           // TODO improvement: this can happen when a seek is performed. In that case speed will jump to max.
           if (aheadSeconds < 0) {
-            console.warn('aheadSeconds < 0:', aheadSeconds, clone.currentTime, originalElement.currentTime);
+            console.warn(
+              'aheadSeconds < 0:',
+              aheadSeconds,
+              clone.currentTime,
+              getBufferedRanges(clone),
+              originalElement.currentTime,
+            );
           }
         }
         // Speed is max when 0 seconds ahead and is 0 when 3 minutes ahead (but we'll clamp it, below).
-        const playbackRateUnclamped = maxClonePlaybackRate * (1 - (aheadSeconds / (3 * 60)));
+        // TODO improvement: make this depend on `clone.buffered`? Especially useful for the
+        // `MediaSource` cloning algorithm where the clone's `buffered` range is the same as
+        // the original element's `buffered` range so it can't run ahead too much, only as far as
+        // `buffered` goes.
+        const playbackRateUnclamped = maxClonePlaybackRate * (1 - (aheadSeconds / (1 * 60)));
         // Min clamp shouldn't be low because of performance - playing a video at 0.01 speed for 100 seconds
         // is much worse than playing it at 1 speed for 1 second.
         // TODO perf: Need to pause the video instead when it goes below the lower bound.

src/entry-points/content/ElementPlaybackControllerCloning/canPlayCurrentSrc.ts

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright (C) 2023  WofWca <[email protected]>
+ *
+ * This file is part of Jump Cutter Browser Extension.
+ *
+ * Jump Cutter Browser Extension is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Jump Cutter Browser Extension is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Jump Cutter Browser Extension.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+export async function canPlayCurrentSrc(element: HTMLMediaElement): Promise<boolean> {
+
+}

src/entry-points/content/ElementPlaybackControllerCloning/createCloneElementWithSameSrc.ts

@@ -0,0 +1,68 @@
+/**
+ * @license
+ * Copyright (C) 2023  WofWca <[email protected]>
+ *
+ * This file is part of Jump Cutter Browser Extension.
+ *
+ * Jump Cutter Browser Extension is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Jump Cutter Browser Extension is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Jump Cutter Browser Extension.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+
+// * Try to create a clone `HTMLMediaElement` that can play the same source as the `originalElement`.
+
+/**
+ * Create a clone `HTMLMediaElement` that uses the same source as the original one.
+ */
+export function createCloneElementWithSameSrc(
+  originalElement: HTMLMediaElement,
+): HTMLAudioElement {
+  // Also see {@link `getOriginalMediaSource`}. It is very similar.
+  // Maybe even too similar.
+
+  // To recap:
+  // * attempting to play a `MediaSource` will [most likely](https://github.com/WofWca/jumpcutter/issues/2#issuecomment-1571654947)
+  // fail.
+  // * For `Blob` (including `File`) it's gonna work (I think?).
+  // * Not sure about `MediaStream`, but either way, I think our extension is not applicable to streams.
+  // * `MediaSource`, `Blob`, `MediaStream` can be used either as `srcObject` or
+  //   `src = URL.createObjectURL(object)`
+
+
+  const cloneEl = document.createElement('audio');
+
+  // https://html.spec.whatwg.org/multipage/media.html#concept-media-load-algorithm
+  // > If mode is object
+  // > 1. Set the currentSrc attribute to the empty string.
+  const { currentSrc } = originalElement;
+  const isSrcObjectUsedOrNoSourceAtAll = !currentSrc;
+  if (isSrcObjectUsedOrNoSourceAtAll) {
+    const { srcObject } = originalElement;
+
+    if (!srcObject) {
+      if (IS_DEV_MODE) {
+        console.warn('Making a clone element for an element with no source. You probably'
+          + ' should have waited before the original element gets a source');
+      }
+      return cloneEl;
+    }
+
+    cloneEl.srcObject = srcObject;
+  } else {
+    cloneEl.src = currentSrc;
+  }
+  return cloneEl;
+
+  // Playback may also fail due to internet getting cut off, or the server returning an error
+  // (e.g. HTTP 429).
+}

src/entry-points/content/ElementPlaybackControllerCloning/getFinalCloneElement.ts

@@ -0,0 +1,72 @@
+/**
+ * @license
+ * Copyright (C) 2023  WofWca <[email protected]>
+ *
+ * This file is part of Jump Cutter Browser Extension.
+ *
+ * Jump Cutter Browser Extension is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Jump Cutter Browser Extension is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Jump Cutter Browser Extension.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { createCloneElementWithSameSrc } from "./createCloneElementWithSameSrc";
+import { sourceMayBeMediaSource } from "./sourceMayBeMediaSource";
+
+/**
+ * This must be called _synchronously_ after the source has been assigned to `element`, otherwise
+ * it may not work.
+ */
+async function canPlayCurrentSrc(element: HTMLMediaElement): Promise<boolean> {
+
+}
+
+export async function getFinalCloneElement(
+  originalElement: HTMLMediaElement,
+  getFallbackCloneElement:
+    undefined | ((originalElement: HTMLMediaElement) => Promise<HTMLAudioElement | undefined>),
+): Promise<HTMLAudioElement> {
+  const sameSourceClone = createCloneElementWithSameSrc(originalElement);
+  if (await canPlayCurrentSrc(sameSourceClone)) {
+    return sameSourceClone;
+  }
+  const fallbackElementIsSuposedToExist =
+    getFallbackCloneElement
+    // You might ask "what about `MediaSourceHandle`? Aren't we supposed to have a clone
+    // in this case?". Well, better check the `../cloneMediaSources` folder for the answer.
+    // Search for `MediaSourceHandle`.
+    && sourceMayBeMediaSource(originalElement);
+  // You might ask "why don't we try to `getFallbackCloneElement` unconditionally at this point?
+  // If there is one, let's use it". The answer is that we get the fallback element from the page's
+  // world's scripts which we don't really trust, while `sourceMayBeMediaSource` is fully under
+  // our control.
+  // But then, the page might still make the original element use `MediaSource`, and forge
+  // the fallback element the way it likes, so maybe there isn't really a point to this check,
+  // maybe it only adds a point of failure for no good reason.
+  // Not only that, making this check here IMO adds unnecessary code coupling.
+  // So, TODO refactor: reconsider the `sourceMayBeMediaSource` check.
+  if (fallbackElementIsSuposedToExist) {
+    const fallbackCloneElement = await getFallbackCloneElement(originalElement);
+    if (fallbackCloneElement) {
+      return fallbackCloneElement;
+    }
+  } else {
+    if (IS_DEV_MODE) {
+      if (await getFallbackCloneElement?.(originalElement)) {
+        console.warn('Expected no fallback element to exist, but it actually does.'
+          + ' Is the pre-condition check outdated, or did the website\'s script make one itself?');
+      }
+    }
+  }
+  // No fallback element, the only option is to return `sameSourceClone` that we can't play.
+  // Maybe we'll be able to play it later for some reason idk.
+  return sameSourceClone;
+}