Bug 1454053 - re-record tp6 pages for raptor; r=igoldan

MozReview-Commit-ID: Esy7cWeADdy

--HG--
rename : testing/raptor/raptor/playback/mitmproxy-playback-set.manifest => testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6.manifest
extra : rebase_source : 99437134717bb9df35a7b70c493c3166319aad44

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -372,7 +372,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin):
             jsonschema.validate(data, schema)
         except Exception as e:
             self.exception("Error while validating PERFHERDER_DATA")
-            self.info(e)
+            self.info(str(e))
 
     def _artifact_perf_data(self, dest):
         src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor.json')
@@ -385,7 +385,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin):
             copyfile(src, dest)
         except Exception as e:
             self.critical("Error copying results %s to upload dir %s" % (src, dest))
-            self.info(e)
+            self.info(str(e))
 
     def run_tests(self, args=None, **kw):
         """run raptor tests"""

testing/raptor/raptor/manifest.py

@@ -18,8 +18,7 @@ LOG = get_proxy_logger(component="raptor-manifest")
 required_settings = ['apps', 'type', 'page_cycles', 'test_url', 'measure',
                      'unit', 'lower_is_better', 'alert_threshold']
 
-playback_settings = ['playback_binary_manifest', 'playback_binary_zip_mac',
-                     'playback_pageset_manifest', 'playback_pageset_zip_mac',
+playback_settings = ['playback_binary_manifest', 'playback_pageset_manifest',
                      'playback_recordings', 'python3_win_manifest']
 
 

testing/raptor/raptor/output.py

@@ -94,14 +94,15 @@ class Output(object):
                 elif 'webaudio' in test.measurements:
                     subtests, vals = self.parseWebaudioOutput(test)
                 suite['subtests'] = subtests
+
+                # if there is more than one subtest, calculate a summary result
+                if len(subtests) > 1:
+                    suite['value'] = self.construct_summary(vals, testname=test.name)
+
             else:
                 LOG.error("output.summarize received unsupported test results type")
                 return
 
-        # if there is more than one subtest, calculate a summary result
-        if len(subtests) > 1:
-            suite['value'] = self.construct_results(vals, testname=test.name)
-
         self.summarized_results = test_results
 
     def parseSpeedometerOutput(self, test):
@@ -341,7 +342,7 @@ class Output(object):
         score = 60 * 1000 / filter.geometric_mean(results) / correctionFactor
         return score
 
-    def construct_results(self, vals, testname):
+    def construct_summary(self, vals, testname):
         if testname.startswith('raptor-v8_7'):
             return self.v8_Metric(vals)
         elif testname.startswith('raptor-kraken'):

testing/raptor/raptor/playback/mitmproxy-playback-set.manifest

@@ -1,9 +0,0 @@
-[
-    {
-        "filename": "mitmproxy-recording-set-win10.zip",
-        "size": 9189938,
-        "digest": "e904917ed6bf1cef7201284385dc603a283e8e22f992876f17edcf0f1f20db95b609f0d8c7f593b4a0a6c20957dcb6a4d502c562ed74fb6cf4bc255c2f691f32",
-        "algorithm": "sha512",
-        "unpack": true
-    }
-]

testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6.manifest

@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "mitmproxy-recordings-raptor-tp6.zip",
+        "size": 8767174,
+        "digest": "3343f54b6c727a2061534872888da4e3aa647e81903c675dc318c717ed0c93f6ce4e2b98c66eb4128376cf8507590531283c95f3951259607edaaae28944d9a5",
+        "algorithm": "sha512",
+        "unpack": true
+    }
+]

testing/raptor/raptor/playback/mitmproxy.py

@@ -149,7 +149,7 @@ class Mitmproxy(Playback, Python3Virtualenv, TestingMixin, MercurialScript):
             transformed_manifest = transform_platform(_manifest, self.config['platform'])
             self._tooltool_fetch(transformed_manifest)
 
-        # we use one pageset for all platforms (pageset was recorded on win10)
+        # we use one pageset for all platforms
         LOG.info("downloading mitmproxy pageset")
         _manifest = os.path.join(here, self.config['playback_pageset_manifest'])
         transformed_manifest = transform_platform(_manifest, self.config['platform'])

testing/raptor/raptor/tests/raptor-tp6.ini

@@ -8,10 +8,8 @@
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
-playback_binary_zip_mac = mitmproxy-2.0.2-{platform}.tar.gz
 python3_win_manifest = python3{x64}.manifest
-playback_pageset_manifest = mitmproxy-playback-set.manifest
-playback_pageset_zip_mac = mitmproxy-recording-set-win10.zip
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
@@ -20,49 +18,50 @@ alert_threshold = 2.0
 [raptor-firefox-tp6-amazon]
 apps = firefox
 test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
-playback_recordings = mitmproxy-recording-amazon.mp
+playback_recordings = amazon.mp
 measure = fnbpaint
 
 [raptor-firefox-tp6-facebook]
 apps = firefox
 test_url = https://www.facebook.com
-playback_recordings = mitmproxy-recording-facebook.mp
+playback_recordings = facebook.mp
 measure = fnbpaint
 
 [raptor-firefox-tp6-google]
 apps = firefox
-test_url = https://www.google.com/#hl=en&q=barack+obama
-playback_recordings = mitmproxy-recording-google.mp
-measure = fnbpaint, hero
-hero = hero
+# note: use the full url as the first part (without '&cad=h') redirects
+# to the url + '&cad=h'; that redirection causes measure.js content
+# to be loaded into that page also; resulting in 2 fnbpaint values etc.
+test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
+playback_recordings = google-search.mp
+measure = fnbpaint
 
 [raptor-firefox-tp6-youtube]
 apps = firefox
 test_url = https://www.youtube.com
-playback_recordings = mitmproxy-recording-youtube.mp
+playback_recordings = youtube.mp
 measure = fnbpaint
 
 [raptor-chrome-tp6-amazon]
 apps = chrome
 test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
-playback_recordings = mitmproxy-recording-amazon.mp
+playback_recordings = amazon.mp
 measure = fcp
 
 [raptor-chrome-tp6-facebook]
 apps = chrome
 test_url = https://www.facebook.com
-playback_recordings = mitmproxy-recording-facebook.mp
+playback_recordings = facebook.mp
 measure = fcp
 
 [raptor-chrome-tp6-google]
 apps = chrome
 test_url = https://www.google.com/#hl=en&q=barack+obama
-playback_recordings = mitmproxy-recording-google.mp
-measure = fcp, hero
-hero = hero
+playback_recordings = google-search.mp
+measure = fcp
 
 [raptor-chrome-tp6-youtube]
 apps = chrome
 test_url = https://www.youtube.com
-playback_recordings = mitmproxy-recording-youtube.mp
+playback_recordings = youtube.mp
 measure = fcp

testing/raptor/test/test_playback.py

@@ -23,9 +23,7 @@ def test_get_playback(get_binary):
     config['obj_path'] = os.path.dirname(get_binary('firefox'))
     config['playback_tool'] = 'mitmproxy'
     config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest'
-    config['playback_binary_zip_mac'] = 'mitmproxy-2.0.2-osx.tar.gz'
     config['playback_pageset_manifest'] = 'mitmproxy-playback-set.manifest'
-    config['playback_pageset_zip_mac'] = 'mitmproxy-recording-set-win10.zip'
     config['playback_recordings'] = 'mitmproxy-recording-amazon.mp'
     config['binary'] = get_binary('firefox')
     playback = get_playback(config)

testing/raptor/webext/raptor/manifest.json

@@ -16,7 +16,6 @@
       "matches": ["*://*.amazon.com/*",
                   "*://*.facebook.com/*",
                   "*://*.google.com/*",
-                  "*://*.google.ca/*",
                   "*://*.youtube.com/*"],
       "js": ["measure.js"]
     },

testing/raptor/webext/raptor/measure.js

@@ -93,7 +93,7 @@ function measureHero() {
                          startMark = startMeasure,
                          endMark = heroFound);
         var perfResult = perfData.getEntriesByName(resultType);
-        var _result = perfResult[0].duration;
+        var _result = Math.round(perfResult[0].duration);
         var resultType = "hero:" + heroFound;
         sendResult(resultType, _result);
         perfData.clearMarks();