Bug 1541189 - Fix intermittents on stream test - r=whimboo

Tweak the Streaming test to fix intermittents.

Differential Revision: https://phabricator.services.mozilla.com/D26288

--HG--
extra : moz-landing-system : lando

.eslintignore

@@ -171,6 +171,7 @@ dom/grid/**
 dom/html/**
 dom/jsurl/**
 dom/media/test/**
+!dom/media/test/marionette/yttest/*.js
 dom/media/tests/**
 dom/media/webaudio/**
 dom/media/webspeech/**

dom/media/test/marionette/test_youtube.py

@@ -11,18 +11,12 @@ from yttest.support import VideoStreamTestCase
 class YoutubeTest(VideoStreamTestCase):
 
     # bug 1513511
-    def test_stream_30_seconds(self):
-        # XXX use the VP9 video we will settle on.
-        with self.youtube_video("BZP1rYjoBgI") as page:
+    def test_stream_4K(self):
+        with self.youtube_video("uR0N3DrybGQ", duration=15) as page:
             res = page.run_test()
-            self.assertTrue(res is not None, "We did not get back the results")
-            self.assertLess(res["droppedVideoFrames"], res["totalVideoFrames"] * 0.04)
-            # extracting in/out from the debugInfo
-            video_state = res["debugInfo"][7]
-            video_in = int(video_state.split(" ")[10].split("=")[-1])
-            video_out = int(video_state.split(" ")[11].split("=")[-1])
-            # what's the ratio ? we want 99%+
-            if video_out == video_in:
-                return
-            in_out_ratio = float(video_out) / float(video_in) * 100
-            self.assertMore(in_out_ratio, 99.0)
+            self.assertVideoQuality(res)
+
+    def test_stream_480p(self):
+        with self.youtube_video("BZP1rYjoBgI", duration=15) as page:
+            res = page.run_test()
+            self.assertVideoQuality(res)

dom/media/test/marionette/yttest/debug_info.js

@@ -1,4 +1,5 @@
 video.mozRequestDebugInfo().then(debugInfo => {
+  // The parsing won't be necessary once we have bug 1542674
   try {
     debugInfo = debugInfo.replace(/\t/g, '').split(/\n/g);
     var JSONDebugInfo = "{";
@@ -8,11 +9,10 @@ video.mozRequestDebugInfo().then(debugInfo => {
       }
       JSONDebugInfo = JSONDebugInfo.slice(0,JSONDebugInfo.length-1);
       JSONDebugInfo += "}";
-      result["debugInfo"] = JSON.parse(JSONDebugInfo);
+      result["mozRequestDebugInfo"] = JSON.parse(JSONDebugInfo);
   } catch (err) {
     console.log(`Error '${err.toString()} in JSON.parse(${debugInfo})`);
-    result["debugInfo"] = debugInfo;
+    result["mozRequestDebugInfo"] = debugInfo;
   }
-  result["debugInfo"] = debugInfo;
   resolve(result);
 });

dom/media/test/marionette/yttest/duration_test.js

@@ -10,9 +10,11 @@ if (!video) {
 
 video.addEventListener("timeupdate", () => {
     if (video.currentTime >= %(duration)s) {
-      video.pause();
       %(video_playback_quality)s
       %(debug_info)s
+      // Pausing after we get the debug info so
+      // we can also look at in/out data in buffers
+      video.pause();
     }
   }
 );

dom/media/test/marionette/yttest/playback.py

@@ -12,6 +12,11 @@ import sys
 import datetime
 import time
 
+try:
+    from urllib import unquote
+except ImportError:
+    from urllib.parse import unquote
+
 
 itags = {
     "5": {
@@ -596,6 +601,13 @@ def OK(flow, code=204):
 
 
 def request(flow):
+    # in some cases, the YT client sends requests with a methode of the form:
+    #   VAR=XX%3GET /xxx
+    # this will clean it up:
+    method = flow.request.method
+    method = unquote(method).split("=")
+    flow.request.method = method[-1]
+
     # All requests made for stats purposes can be discarded and
     # a 204 sent back to the client.
     if flow.request.url.startswith("https://www.youtube.com/ptracking"):
@@ -611,6 +623,9 @@ def request(flow):
     if "push.services.mozilla.com" in flow.request.url:
         OK(flow, code=200)
         return
+    if "tracking-protection.cdn.mozilla.net" in flow.request.url:
+        OK(flow, code=200)
+        return
     if "gen_204" in flow.request.url:
         OK(flow)
         return

dom/media/test/marionette/yttest/support.py

@@ -23,6 +23,7 @@ class VideoStreamTestCase(MarionetteTestCase):
         if "MOZ_UPLOAD_DIR" not in os.environ:
             os.environ["OBJ_PATH"] = "/tmp/"
         self.marionette.set_pref("media.autoplay.default", 1)
+        self.marionette.set_pref("privacy.trackingprotection.enabled", False)
 
     @contextmanager
     def using_proxy(self, video_id):
@@ -56,8 +57,6 @@ class VideoStreamTestCase(MarionetteTestCase):
         playback_file = os.path.join(playback_dir, "%s.playback" % video_id)
 
         config["playback_tool_args"] = [
-            "--set",
-            "stream_large_bodies=30",
             "--ssl-insecure",
             "--server-replay-nopop",
             "--set",
@@ -82,9 +81,29 @@ class VideoStreamTestCase(MarionetteTestCase):
     def youtube_video(self, video_id, **options):
         proxy = options.get("proxy", True)
         if proxy:
-            with self.using_proxy(video_id):
+            with self.using_proxy(video_id) as proxy:
+                options["upload_dir"] = proxy.upload_dir
                 with using_page(video_id, self.marionette, **options) as page:
                     yield page
         else:
             with using_page(video_id, self.marionette, **options) as page:
                 yield page
+
+    def assertVideoQuality(self, res):
+        self.assertTrue(res is not None, "We did not get back the results")
+        debug_info = res["mozRequestDebugInfo"]
+
+        # looking at mNumSamplesOutputTotal vs mNumSamplesSkippedTotal
+        decoded, skipped = debug_info["Video Frames Decoded"].split(" ", 1)
+        decoded = int(decoded)
+        skipped = int(skipped.split("=")[-1][:-1])
+        self.assertLess(skipped, decoded * 0.04)
+
+        # extracting in/out from the debugInfo
+        video_state = debug_info["Video State"]
+        video_in = int(video_state["in"])
+        video_out = int(video_state["out"])
+        # what's the ratio ? we want 99%+
+        if video_out != video_in:
+            in_out_ratio = float(video_out) / float(video_in) * 100
+            self.assertGreater(in_out_ratio, 99.0)

dom/media/test/marionette/yttest/uR0N3DrybGQ.manifest

@@ -0,0 +1,10 @@
+[
+  {
+    "size": 629013569,
+    "visibility": "public",
+    "digest": "213afa0e40411c26c86092a0803099a8c596b27cf789ed658ba0cf50dd8b404926dd784cd0236922aca22d3763edff666dd247c14bfe38359fb9d767f1869048",
+    "algorithm": "sha512",
+    "filename": "uR0N3DrybGQ.tar.gz",
+    "unpack": true
+  }
+]

dom/media/test/marionette/yttest/until_end_test.js

@@ -9,9 +9,11 @@ if (!video) {
 }
 
 video.addEventListener("ended", () => {
-    video.pause();
     %(video_playback_quality)s
     %(debug_info)s
+    // Pausing after we get the debug info so
+    // we can also look at in/out data in buffers
+    video.pause();
   }, {once: true}
 );
 

dom/media/test/marionette/yttest/video_playback_quality.js

@@ -1,7 +1 @@
-var vpq = video.getVideoPlaybackQuality();
-var result = {"currentTime": video.currentTime};
-result["creationTime"] = vpq.creationTime;
-result["corruptedVideoFrames"] = vpq.corruptedVideoFrames;
-result["droppedVideoFrames"] = vpq.droppedVideoFrames;
-result["totalVideoFrames"] = vpq.totalVideoFrames;
-result["defaultPlaybackRate"] = video.playbackRate;
+var result = {"getVideoPlaybackQuality": video.getVideoPlaybackQuality()};

dom/media/test/marionette/yttest/ytpage.py

@@ -6,6 +6,11 @@ Drives the browser during the playback test.
 """
 import contextlib
 import os
+import time
+import json
+import re
+
+from marionette_driver.by import By
 
 
 here = os.path.dirname(__file__)
@@ -23,6 +28,19 @@ for script in JS_MACROS:
     with open(js) as f:
         JS_MACROS[script] = f.read()
 
+SPLIT_FIELD = (
+    "Audio State",
+    "Audio Track Buffer Details",
+    "AudioSink",
+    "MDSM",
+    "Video State",
+    "Video Track Buffer Details",
+    "Dumping Audio Track",
+    "Dumping Video Track",
+    "MediaDecoder",
+    "VideoSink",
+)
+
 
 class YoutubePage:
     def __init__(self, video_id, marionette, **options):
@@ -53,14 +71,23 @@ class YoutubePage:
 
     def run_test(self):
         self.start_video()
+        # If we don't pause here for just a bit the media events
+        # are not intercepted.
+        time.sleep(5)
+        body = self.marionette.find_element(By.TAG_NAME, "html")
+        body.click()
         options = dict(JS_MACROS)
         options.update(self.options)
         if "duration" in options:
             script = DURATION_TEST % options
         else:
             script = UNTIL_END_TEST % options
-        self.marionette.set_pref("media.autoplay.default", 0)
-        return self.execute_async_script(script)
+        res = self.execute_async_script(script)
+        if res is None:
+            return res
+        res = self._parse_res(res)
+        self._dump_res(res)
+        return res
 
     def execute_async_script(self, script, context=None):
         if context is None:
@@ -68,6 +95,47 @@ class YoutubePage:
         with self.marionette.using_context(context):
             return self.marionette.execute_async_script(script, sandbox="system")
 
+    def _parse_res(self, res):
+        debug_info = {}
+        # The parsing won't be necessary once we have bug 1542674
+        for key, value in res["mozRequestDebugInfo"].items():
+            key, value = key.strip(), value.strip()
+            if key.startswith(SPLIT_FIELD):
+                value_dict = {}
+                for field in re.findall(r"\S+\(.+\)\s|\S+", value):
+                    field = field.strip()
+                    if field == "":
+                        continue
+                    if field.startswith("VideoQueue"):
+                        k = "VideoQueue"
+                        v = field[len("VideoQueue(") : -2]  # noqa: E203
+                        fields = {}
+                        v = v.split(" ")
+                        for h in v:
+                            f, vv = h.split("=")
+                            fields[f] = vv
+                        v = fields
+                    else:
+                        if "=" in field:
+                            k, v = field.split("=", 1)
+                        else:
+                            k, v = field.split(":", 1)
+                    value_dict[k] = v
+                value = value_dict
+            debug_info[key] = value
+        res["mozRequestDebugInfo"] = debug_info
+        return res
+
+    def _dump_res(self, res):
+        raw = json.dumps(res, indent=2, sort_keys=True)
+        print(raw)
+        if "upload_dir" in self.options:
+            fn = "%s-videoPlaybackQuality.json" % self.video_id
+            fn = os.path.join(self.options["upload_dir"], fn)
+            # dumping on disk
+            with open(fn, "w") as f:
+                f.write(raw)
+
     def close(self):
         if self.started:
             self.marionette.delete_session()