Bug 1752367 - added tests to ensure pip cc order using vttcue.line r=mhowell,niklas

Differential Revision: https://phabricator.services.mozilla.com/D138233

toolkit/actors/PictureInPictureChild.jsm

@@ -1296,15 +1296,9 @@ class PictureInPictureChild extends JSWindowActorChild {
       return;
     }
 
-    const allCuesArray = [...textTrackCues];
-    let lineNumberUsed = allCuesArray.find(cue => cue.line !== "auto");
-
-    // If VTTCue.line is not set to "auto", simplying reading textTrackCues does
-    // not guarantee that text tracks are displayed in their intended order. In this case,
-    // sort the cues according to line number.
-    if (lineNumberUsed) {
-      allCuesArray.sort((cue1, cue2) => cue1.line - cue2.line);
-    }
+    let allCuesArray = [...textTrackCues];
+    // Re-order cues
+    this.getOrderedWebVTTCues(allCuesArray);
     // Parse through WebVTT cue using vtt.js to ensure
     // semantic markup like <b> and <i> tags are rendered.
     allCuesArray.forEach(cue => {
@@ -1319,6 +1313,41 @@ class PictureInPictureChild extends JSWindowActorChild {
     });
   }
 
+  /**
+   * Re-orders list of multiple active cues to ensure cues are rendered in the correct order.
+   * How cues are ordered depends on the VTTCue.line value of the cue.
+   *
+   * If line is string "auto", we want to reverse the order of cues.
+   * Cues are read from top to bottom in a vtt file, but are inserted into a video from bottom to top.
+   * Ensure this order is followed.
+   *
+   * If line is an integer or percentage, we want to order cues according to numeric value.
+   * Assumptions:
+   *  1) all active cues are numeric
+   *  2) all active cues are in range 0..100
+   *  3) all actives cue are horizontal (no VTTCue.vertical)
+   *  4) all active cues with VTTCue.line integer have VTTCue.snapToLines = true
+   *  5) all active cues with VTTCue.line percentage have VTTCue.snapToLines = false
+   *
+   * vtt.jsm currently sets snapToLines to false if line is a percentage value, but
+   * cues are still ordered by line. In most cases, snapToLines is set to true by default,
+   * unless intentionally overridden.
+   * @param allCuesArray {Array<VTTCue>} array of active cues
+   */
+  getOrderedWebVTTCues(allCuesArray) {
+    if (!allCuesArray || allCuesArray.length <= 1) {
+      return;
+    }
+
+    let allCuesHaveNumericLines = allCuesArray.find(cue => cue.line !== "auto");
+
+    if (allCuesHaveNumericLines) {
+      allCuesArray.sort((cue1, cue2) => cue1.line - cue2.line);
+    } else if (allCuesArray.length >= 2) {
+      allCuesArray.reverse();
+    }
+  }
+
   /**
    * Returns a reference to the PiP's <video> element being displayed in Picture-in-Picture
    * mode.

toolkit/components/pictureinpicture/tests/browser.ini

@@ -21,6 +21,8 @@ support-files =
   test-webvtt-1.vtt
   test-webvtt-2.vtt
   test-webvtt-3.vtt
+  test-webvtt-4.vtt
+  test-webvtt-5.vtt
   short.mp4
   ../../../../dom/media/test/gizmo.mp4
   ../../../../dom/media/test/owl.mp3
@@ -87,6 +89,7 @@ skip-if = os == "win" && bits == 64 && debug # Bug 1683002
 [browser_telemetry_togglePiP.js]
 [browser_text_tracks_webvtt_1.js]
 [browser_text_tracks_webvtt_2.js]
+[browser_text_tracks_webvtt_3.js]
 [browser_thirdPartyIframe.js]
 [browser_toggleButtonOnNanDuration.js]
 skip-if =

toolkit/components/pictureinpicture/tests/browser_text_tracks_webvtt_1.js

@@ -3,45 +3,6 @@
 
 "use strict";
 
-/**
- * Initializes videos and text tracks for the current test case.
- * First track is the default track to be loaded onto the video.
- * Once initialization is done, play then pause the requested video.
- * so that text tracks are loaded.
- * @param {Element} browser The <xul:browser> hosting the <video>
- * @param {String} videoID The ID of the video being checked
- * @param {Integer} defaultTrackIndex The index of the track to be loaded, or none if -1
- */
-async function prepareVideosAndTracks(browser, videoID, defaultTrackIndex = 0) {
-  info("Preparing video and initial text tracks");
-  await ensureVideosReady(browser);
-  await SpecialPowers.spawn(
-    browser,
-    [{ videoID, defaultTrackIndex }],
-    async args => {
-      let video = content.document.getElementById(args.videoID);
-      let tracks = video.textTracks;
-
-      is(tracks.length, 3, "Number of tracks loaded should be 3");
-
-      // Enable track for originating video
-      if (args.defaultTrackIndex >= 0) {
-        info(`Loading track ${args.defaultTrackIndex + 1}`);
-        let track = tracks[args.defaultTrackIndex];
-        tracks.mode = "showing";
-        track.mode = "showing";
-      }
-
-      // Briefly play the video to load text tracks onto the pip window.
-      info("Playing video to load text tracks");
-      video.play();
-      info("Pausing video");
-      video.pause();
-      ok(video.paused, "Video should be paused before proceeding with test");
-    }
-  );
-}
-
 /**
  * This test ensures that text tracks shown on the source video
  * do not appear on a newly created pip window if the pref
@@ -65,7 +26,7 @@ add_task(async function test_text_tracks_new_window_pref_disabled() {
       gBrowser,
     },
     async browser => {
-      await prepareVideosAndTracks(browser, videoID);
+      await prepareVideosAndWebVTTTracks(browser, videoID);
 
       let pipWin = await triggerPictureInPicture(browser, videoID);
       ok(pipWin, "Got Picture-in-Picture window.");
@@ -107,7 +68,7 @@ add_task(async function test_text_tracks_new_window_pref_enabled() {
       gBrowser,
     },
     async browser => {
-      await prepareVideosAndTracks(browser, videoID);
+      await prepareVideosAndWebVTTTracks(browser, videoID);
 
       let pipWin = await triggerPictureInPicture(browser, videoID);
       ok(pipWin, "Got Picture-in-Picture window.");
@@ -147,7 +108,7 @@ add_task(async function test_text_tracks_new_window_no_track() {
       gBrowser,
     },
     async browser => {
-      await prepareVideosAndTracks(browser, videoID, -1);
+      await prepareVideosAndWebVTTTracks(browser, videoID, -1);
 
       let pipWin = await triggerPictureInPicture(browser, videoID);
       ok(pipWin, "Got Picture-in-Picture window.");

toolkit/components/pictureinpicture/tests/browser_text_tracks_webvtt_2.js

@@ -3,79 +3,6 @@
 
 "use strict";
 
-/**
- * Initializes videos and text tracks for the current test case.
- * First track is the default track to be loaded onto the video.
- * Once initialization is done, play then pause the requested video.
- * so that text tracks are loaded.
- * @param {Element} browser The <xul:browser> hosting the <video>
- * @param {String} videoID The ID of the video being checked
- * @param {Integer} defaultTrackIndex The index of the track to be loaded, or none if -1
- */
-async function prepareVideosAndTracks(browser, videoID, defaultTrackIndex = 0) {
-  info("Preparing video and initial text tracks");
-  await ensureVideosReady(browser);
-  await SpecialPowers.spawn(
-    browser,
-    [{ videoID, defaultTrackIndex }],
-    async args => {
-      let video = content.document.getElementById(args.videoID);
-      let tracks = video.textTracks;
-
-      is(tracks.length, 3, "Number of tracks loaded should be 3");
-
-      // Enable track for originating video
-      if (args.defaultTrackIndex >= 0) {
-        info(`Loading track ${args.defaultTrackIndex + 1}`);
-        let track = tracks[args.defaultTrackIndex];
-        tracks.mode = "showing";
-        track.mode = "showing";
-      }
-
-      // Briefly play the video to load text tracks onto the pip window.
-      info("Playing video to load text tracks");
-      video.play();
-      info("Pausing video");
-      video.pause();
-      ok(video.paused, "Video should be paused before proceeding with test");
-    }
-  );
-}
-
-/**
- * Plays originating video until the next cue is loaded.
- * Once the next cue is loaded, pause the video.
- * @param {Element} browser The <xul:browser> hosting the <video>
- * @param {String} videoID The ID of the video being checked
- * @param {Integer} textTrackIndex The index of the track to be loaded, or none if -1
- */
-async function waitForNextCue(browser, videoID, textTrackIndex = 0) {
-  if (textTrackIndex < 0) {
-    ok(false, "Cannot wait for next cue with invalid track index");
-  }
-
-  await SpecialPowers.spawn(
-    browser,
-    [{ videoID, textTrackIndex }],
-    async args => {
-      let video = content.document.getElementById(args.videoID);
-      info("Playing video to activate next cue");
-      video.play();
-      ok(!video.paused, "Video is playing");
-
-      info("Waiting until cuechange is called");
-      await ContentTaskUtils.waitForEvent(
-        video.textTracks[args.textTrackIndex],
-        "cuechange"
-      );
-
-      info("Pausing video to read text track");
-      video.pause();
-      ok(video.paused, "Video is paused");
-    }
-  );
-}
-
 /**
  * This test ensures that text tracks disappear from the pip window
  * when the pref is disabled.
@@ -98,7 +25,7 @@ add_task(async function test_text_tracks_existing_window_pref_disabled() {
       gBrowser,
     },
     async browser => {
-      await prepareVideosAndTracks(browser, videoID);
+      await prepareVideosAndWebVTTTracks(browser, videoID);
 
       let pipWin = await triggerPictureInPicture(browser, videoID);
       ok(pipWin, "Got Picture-in-Picture window.");
@@ -159,7 +86,7 @@ add_task(async function test_text_tracks_existing_window_pref_enabled() {
       gBrowser,
     },
     async browser => {
-      await prepareVideosAndTracks(browser, videoID);
+      await prepareVideosAndWebVTTTracks(browser, videoID);
 
       let pipWin = await triggerPictureInPicture(browser, videoID);
       ok(pipWin, "Got Picture-in-Picture window.");
@@ -221,7 +148,7 @@ add_task(async function test_text_tracks_existing_window_new_track() {
       gBrowser,
     },
     async browser => {
-      await prepareVideosAndTracks(browser, videoID);
+      await prepareVideosAndWebVTTTracks(browser, videoID);
 
       let pipWin = await triggerPictureInPicture(browser, videoID);
       ok(pipWin, "Got Picture-in-Picture window.");
@@ -290,7 +217,7 @@ add_task(async function test_text_tracks_existing_window_cues() {
       gBrowser,
     },
     async browser => {
-      await prepareVideosAndTracks(browser, videoID);
+      await prepareVideosAndWebVTTTracks(browser, videoID);
 
       let pipWin = await triggerPictureInPicture(browser, videoID);
       ok(pipWin, "Got Picture-in-Picture window.");
@@ -366,7 +293,7 @@ add_task(async function test_text_tracks_existing_window_no_track() {
       gBrowser,
     },
     async browser => {
-      await prepareVideosAndTracks(browser, videoID);
+      await prepareVideosAndWebVTTTracks(browser, videoID);
 
       let pipWin = await triggerPictureInPicture(browser, videoID);
       ok(pipWin, "Got Picture-in-Picture window.");
@@ -426,7 +353,7 @@ add_task(async function test_text_tracks_existing_window_multi_cue() {
       gBrowser,
     },
     async browser => {
-      await prepareVideosAndTracks(browser, videoID, 2);
+      await prepareVideosAndWebVTTTracks(browser, videoID, 2);
       let pipWin = await triggerPictureInPicture(browser, videoID);
       ok(pipWin, "Got Picture-in-Picture window.");
       let pipBrowser = pipWin.document.getElementById("browser");

toolkit/components/pictureinpicture/tests/browser_text_tracks_webvtt_3.js

@@ -0,0 +1,218 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * Verifies the value of a cue's .line property.
+ * @param {Element} browser The <xul:browser> hosting the <video>
+ * @param {String} videoID The ID of the video being checked
+ * @param {Integer} trackIndex The index of the track to be loaded
+ * @param {Integer} cueIndex The index of the cue to be tested on
+ * @param {Integer|String} expectedValue The expected line value of the cue
+ */
+async function verifyLineForCue(
+  browser,
+  videoID,
+  trackIndex,
+  cueIndex,
+  expectedValue
+) {
+  await SpecialPowers.spawn(
+    browser,
+    [{ videoID, trackIndex, cueIndex, expectedValue }],
+    async args => {
+      info("Checking .line property values");
+      const video = content.document.getElementById(args.videoID);
+      const activeCues = video.textTracks[args.trackIndex].activeCues;
+      const vttCueLine = activeCues[args.cueIndex].line;
+      is(vttCueLine, args.expectedValue, "Cue line should have expected value");
+    }
+  );
+}
+
+/**
+ * This test ensures that text tracks appear in expected order if
+ * VTTCue.line property is auto.
+ */
+add_task(async function test_text_tracks_new_window_line_auto() {
+  info("Running test: new window - line auto");
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      [
+        "media.videocontrols.picture-in-picture.display-text-tracks.enabled",
+        true,
+      ],
+    ],
+  });
+  let videoID = "with-controls";
+
+  await BrowserTestUtils.withNewTab(
+    {
+      url: TEST_PAGE_WITH_WEBVTT,
+      gBrowser,
+    },
+    async browser => {
+      let trackIndex = 2;
+      await prepareVideosAndWebVTTTracks(browser, videoID, trackIndex);
+
+      let pipWin = await triggerPictureInPicture(browser, videoID);
+      ok(pipWin, "Got Picture-in-Picture window.");
+      let pipBrowser = pipWin.document.getElementById("browser");
+
+      await verifyLineForCue(browser, videoID, trackIndex, 0, "auto");
+      await verifyLineForCue(browser, videoID, trackIndex, 1, "auto");
+
+      await SpecialPowers.spawn(pipBrowser, [], async () => {
+        info("Checking text track content in pip window");
+        let textTracks = content.document.getElementById("texttracks");
+        ok(textTracks, "TextTracks container should exist in the pip window");
+        await ContentTaskUtils.waitForCondition(() => {
+          return textTracks.textContent;
+        }, `Text track is still not visible on the pip window. Got ${textTracks.textContent}`);
+
+        let cueDivs = textTracks.children;
+
+        is(cueDivs.length, 2, "There should be 2 active cues");
+        // cue1 in this case refers to the first cue to be defined in the vtt file.
+        // cue2 is therefore the next cue to be defined right after in the vtt file.
+        ok(
+          cueDivs[0].textContent.includes("cue 2"),
+          `cue 2 should be above. Got: ${cueDivs[0].textContent}`
+        );
+        ok(
+          cueDivs[1].textContent.includes("cue 1"),
+          `cue 1 should be below. Got: ${cueDivs[1].textContent}`
+        );
+      });
+    }
+  );
+});
+
+/**
+ * This test ensures that text tracks appear in expected order if
+ * VTTCue.line property is an integer.
+ */
+add_task(async function test_text_tracks_new_window_line_integer() {
+  info("Running test: new window - line integer");
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      [
+        "media.videocontrols.picture-in-picture.display-text-tracks.enabled",
+        true,
+      ],
+    ],
+  });
+  let videoID = "with-controls";
+
+  await BrowserTestUtils.withNewTab(
+    {
+      url: TEST_PAGE_WITH_WEBVTT,
+      gBrowser,
+    },
+    async browser => {
+      let trackIndex = 3;
+      await prepareVideosAndWebVTTTracks(browser, videoID, trackIndex);
+
+      let pipWin = await triggerPictureInPicture(browser, videoID);
+      ok(pipWin, "Got Picture-in-Picture window.");
+      let pipBrowser = pipWin.document.getElementById("browser");
+
+      await verifyLineForCue(browser, videoID, trackIndex, 0, 2);
+      await verifyLineForCue(browser, videoID, trackIndex, 1, 3);
+      await verifyLineForCue(browser, videoID, trackIndex, 2, 1);
+
+      await SpecialPowers.spawn(pipBrowser, [], async () => {
+        info("Checking text track content in pip window");
+        let textTracks = content.document.getElementById("texttracks");
+        ok(textTracks, "TextTracks container should exist in the pip window");
+        await ContentTaskUtils.waitForCondition(() => {
+          return textTracks.textContent;
+        }, `Text track is still not visible on the pip window. Got ${textTracks.textContent}`);
+
+        let cueDivs = textTracks.children;
+
+        is(cueDivs.length, 3, "There should be 3 active cues");
+
+        // cue1 in this case refers to the first cue to be defined in the vtt file.
+        // cue2 is therefore the next cue to be defined right after in the vtt file.
+        ok(
+          cueDivs[0].textContent.includes("cue 3"),
+          `cue 3 should be above. Got: ${cueDivs[0].textContent}`
+        );
+        ok(
+          cueDivs[1].textContent.includes("cue 1"),
+          `cue 1 should be next. Got: ${cueDivs[1].textContent}`
+        );
+        ok(
+          cueDivs[2].textContent.includes("cue 2"),
+          `cue 2 should be below. Got: ${cueDivs[2].textContent}`
+        );
+      });
+    }
+  );
+});
+
+/**
+ * This test ensures that text tracks appear in expected order if
+ * VTTCue.line property is a percentage value.
+ */
+add_task(async function test_text_tracks_new_window_line_percent() {
+  info("Running test: new window - line percent");
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      [
+        "media.videocontrols.picture-in-picture.display-text-tracks.enabled",
+        true,
+      ],
+    ],
+  });
+  let videoID = "with-controls";
+
+  await BrowserTestUtils.withNewTab(
+    {
+      url: TEST_PAGE_WITH_WEBVTT,
+      gBrowser,
+    },
+    async browser => {
+      let trackIndex = 4;
+      await prepareVideosAndWebVTTTracks(browser, videoID, trackIndex);
+
+      let pipWin = await triggerPictureInPicture(browser, videoID);
+      ok(pipWin, "Got Picture-in-Picture window.");
+      let pipBrowser = pipWin.document.getElementById("browser");
+
+      await verifyLineForCue(browser, videoID, trackIndex, 0, 90);
+      await verifyLineForCue(browser, videoID, trackIndex, 1, 10);
+      await verifyLineForCue(browser, videoID, trackIndex, 2, 50);
+
+      await SpecialPowers.spawn(pipBrowser, [], async () => {
+        info("Checking text track content in pip window");
+        let textTracks = content.document.getElementById("texttracks");
+        ok(textTracks, "TextTracks container should exist in the pip window");
+
+        await ContentTaskUtils.waitForCondition(() => {
+          return textTracks.textContent;
+        }, `Text track is still not visible on the pip window. Got ${textTracks.textContent}`);
+
+        let cueDivs = textTracks.children;
+        is(cueDivs.length, 3, "There should be 3 active cues");
+
+        // cue1 in this case refers to the first cue to be defined in the vtt file.
+        // cue2 is therefore the next cue to be defined right after in the vtt file.
+        ok(
+          cueDivs[0].textContent.includes("cue 2"),
+          `cue 2 should be above. Got: ${cueDivs[0].textContent}`
+        );
+        ok(
+          cueDivs[1].textContent.includes("cue 3"),
+          `cue 3 should be next. Got: ${cueDivs[1].textContent}`
+        );
+        ok(
+          cueDivs[2].textContent.includes("cue 1"),
+          `cue 1 should be below. Got: ${cueDivs[2].textContent}`
+        );
+      });
+    }
+  );
+});

toolkit/components/pictureinpicture/tests/head.js

@@ -849,3 +849,80 @@ async function isVideoMuted(browser, videoID) {
     return content.document.getElementById(videoID).muted;
   });
 }
+
+/**
+ * Initializes videos and text tracks for the current test case.
+ * First track is the default track to be loaded onto the video.
+ * Once initialization is done, play then pause the requested video.
+ * so that text tracks are loaded.
+ * @param {Element} browser The <xul:browser> hosting the <video>
+ * @param {String} videoID The ID of the video being checked
+ * @param {Integer} defaultTrackIndex The index of the track to be loaded, or none if -1
+ */
+async function prepareVideosAndWebVTTTracks(
+  browser,
+  videoID,
+  defaultTrackIndex = 0
+) {
+  info("Preparing video and initial text tracks");
+  await ensureVideosReady(browser);
+  await SpecialPowers.spawn(
+    browser,
+    [{ videoID, defaultTrackIndex }],
+    async args => {
+      let video = content.document.getElementById(args.videoID);
+      let tracks = video.textTracks;
+
+      is(tracks.length, 5, "Number of tracks loaded should be 5");
+
+      // Enable track for originating video
+      if (args.defaultTrackIndex >= 0) {
+        info(`Loading track ${args.defaultTrackIndex + 1}`);
+        let track = tracks[args.defaultTrackIndex];
+        tracks.mode = "showing";
+        track.mode = "showing";
+      }
+
+      // Briefly play the video to load text tracks onto the pip window.
+      info("Playing video to load text tracks");
+      video.play();
+      info("Pausing video");
+      video.pause();
+      ok(video.paused, "Video should be paused before proceeding with test");
+    }
+  );
+}
+
+/**
+ * Plays originating video until the next cue is loaded.
+ * Once the next cue is loaded, pause the video.
+ * @param {Element} browser The <xul:browser> hosting the <video>
+ * @param {String} videoID The ID of the video being checked
+ * @param {Integer} textTrackIndex The index of the track to be loaded, or none if -1
+ */
+async function waitForNextCue(browser, videoID, textTrackIndex = 0) {
+  if (textTrackIndex < 0) {
+    ok(false, "Cannot wait for next cue with invalid track index");
+  }
+
+  await SpecialPowers.spawn(
+    browser,
+    [{ videoID, textTrackIndex }],
+    async args => {
+      let video = content.document.getElementById(args.videoID);
+      info("Playing video to activate next cue");
+      video.play();
+      ok(!video.paused, "Video is playing");
+
+      info("Waiting until cuechange is called");
+      await ContentTaskUtils.waitForEvent(
+        video.textTracks[args.textTrackIndex],
+        "cuechange"
+      );
+
+      info("Pausing video to read text track");
+      video.pause();
+      ok(video.paused, "Video is paused");
+    }
+  );
+}

toolkit/components/pictureinpicture/tests/test-page-with-webvtt.html

@@ -35,6 +35,20 @@
     srclang="eo"
     src="test-webvtt-3.vtt"
     />
+    <track
+    id="track4"
+    kind="subtitles"
+    label="[test] zh"
+    srclang="zh"
+    src="test-webvtt-4.vtt"
+    />
+    <track
+    id="track5"
+    kind="subtitles"
+    label="[test] es"
+    srclang="es"
+    src="test-webvtt-5.vtt"
+    />
   </video>
 
   <script>

toolkit/components/pictureinpicture/tests/test-webvtt-3.vtt

@@ -1,9 +1,11 @@
 WEBVTT
 
+Test file with multiple active cues and VTTCue.line as "auto"
+
 1
 00:00:00.000 --> 00:00:01.000
 track 3 - cue 1
 
 2
-00:00:00.000 --> 00:00:05.000
+00:00:00.000 --> 00:00:01.000
 track 3 - cue 2

toolkit/components/pictureinpicture/tests/test-webvtt-4.vtt

@@ -0,0 +1,15 @@
+WEBVTT
+
+Test file with multiple active cues and VTTCue.line as an integer
+
+1
+00:00:00.000 --> 00:00:01.000 line:2
+track 4 - cue 1 - integer line
+
+2
+00:00:00.000 --> 00:00:01.000 line:3
+track 4 - cue 2 - integer line
+
+3
+00:00:00.000 --> 00:00:01.000 line:1
+track 4 - cue 3 - integer line

toolkit/components/pictureinpicture/tests/test-webvtt-5.vtt

@@ -0,0 +1,12 @@
+WEBVTT
+
+Test file with multiple active cues and VTTCue.line as a percentage value
+
+00:00:00.000 --> 00:00:01.000 line:90%
+track 5 - cue 1 - percent line
+
+00:00:00.000 --> 00:00:01.000 line:10%
+track 5 - cue 2 - percent line
+
+00:00:00.000 --> 00:00:01.000 line:50%
+track 5 - cue 3 - percent line