diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py index c5593bdab5cb..1664d2f8d54f 100644 --- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -372,7 +372,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin): jsonschema.validate(data, schema) except Exception as e: self.exception("Error while validating PERFHERDER_DATA") - self.info(e) + self.info(str(e)) def _artifact_perf_data(self, dest): src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor.json') @@ -385,7 +385,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin): copyfile(src, dest) except Exception as e: self.critical("Error copying results %s to upload dir %s" % (src, dest)) - self.info(e) + self.info(str(e)) def run_tests(self, args=None, **kw): """run raptor tests""" diff --git a/testing/raptor/raptor/manifest.py b/testing/raptor/raptor/manifest.py index 45a823c5b391..75caa89a7f7a 100644 --- a/testing/raptor/raptor/manifest.py +++ b/testing/raptor/raptor/manifest.py @@ -18,8 +18,7 @@ LOG = get_proxy_logger(component="raptor-manifest") required_settings = ['apps', 'type', 'page_cycles', 'test_url', 'measure', 'unit', 'lower_is_better', 'alert_threshold'] -playback_settings = ['playback_binary_manifest', 'playback_binary_zip_mac', - 'playback_pageset_manifest', 'playback_pageset_zip_mac', +playback_settings = ['playback_binary_manifest', 'playback_pageset_manifest', 'playback_recordings', 'python3_win_manifest'] diff --git a/testing/raptor/raptor/output.py b/testing/raptor/raptor/output.py index 39b7664a7ea6..d08c0bbb4ead 100644 --- a/testing/raptor/raptor/output.py +++ b/testing/raptor/raptor/output.py @@ -94,14 +94,15 @@ class Output(object): elif 'webaudio' in test.measurements: subtests, vals = self.parseWebaudioOutput(test) suite['subtests'] = subtests + + # if there is more than one subtest, calculate a summary result + if len(subtests) > 1: + suite['value'] = self.construct_summary(vals, testname=test.name) + else: LOG.error("output.summarize received unsupported test results type") return - # if there is more than one subtest, calculate a summary result - if len(subtests) > 1: - suite['value'] = self.construct_results(vals, testname=test.name) - self.summarized_results = test_results def parseSpeedometerOutput(self, test): @@ -341,7 +342,7 @@ class Output(object): score = 60 * 1000 / filter.geometric_mean(results) / correctionFactor return score - def construct_results(self, vals, testname): + def construct_summary(self, vals, testname): if testname.startswith('raptor-v8_7'): return self.v8_Metric(vals) elif testname.startswith('raptor-kraken'): diff --git a/testing/raptor/raptor/playback/mitmproxy-playback-set.manifest b/testing/raptor/raptor/playback/mitmproxy-playback-set.manifest deleted file mode 100644 index 60da07158146..000000000000 --- a/testing/raptor/raptor/playback/mitmproxy-playback-set.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "filename": "mitmproxy-recording-set-win10.zip", - "size": 9189938, - "digest": "e904917ed6bf1cef7201284385dc603a283e8e22f992876f17edcf0f1f20db95b609f0d8c7f593b4a0a6c20957dcb6a4d502c562ed74fb6cf4bc255c2f691f32", - "algorithm": "sha512", - "unpack": true - } -] \ No newline at end of file diff --git a/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6.manifest b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6.manifest new file mode 100644 index 000000000000..1773567234c6 --- /dev/null +++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitmproxy-recordings-raptor-tp6.zip", + "size": 8767174, + "digest": "3343f54b6c727a2061534872888da4e3aa647e81903c675dc318c717ed0c93f6ce4e2b98c66eb4128376cf8507590531283c95f3951259607edaaae28944d9a5", + "algorithm": "sha512", + "unpack": true + } +] \ No newline at end of file diff --git a/testing/raptor/raptor/playback/mitmproxy.py b/testing/raptor/raptor/playback/mitmproxy.py index 50f4eebe87b6..87884c1a21ef 100644 --- a/testing/raptor/raptor/playback/mitmproxy.py +++ b/testing/raptor/raptor/playback/mitmproxy.py @@ -149,7 +149,7 @@ class Mitmproxy(Playback, Python3Virtualenv, TestingMixin, MercurialScript): transformed_manifest = transform_platform(_manifest, self.config['platform']) self._tooltool_fetch(transformed_manifest) - # we use one pageset for all platforms (pageset was recorded on win10) + # we use one pageset for all platforms LOG.info("downloading mitmproxy pageset") _manifest = os.path.join(here, self.config['playback_pageset_manifest']) transformed_manifest = transform_platform(_manifest, self.config['platform']) diff --git a/testing/raptor/raptor/tests/raptor-tp6.ini b/testing/raptor/raptor/tests/raptor-tp6.ini index f6e4083095d2..6a2440e97373 100644 --- a/testing/raptor/raptor/tests/raptor-tp6.ini +++ b/testing/raptor/raptor/tests/raptor-tp6.ini @@ -8,10 +8,8 @@ type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest -playback_binary_zip_mac = mitmproxy-2.0.2-{platform}.tar.gz python3_win_manifest = python3{x64}.manifest -playback_pageset_manifest = mitmproxy-playback-set.manifest -playback_pageset_zip_mac = mitmproxy-recording-set-win10.zip +playback_pageset_manifest = mitmproxy-recordings-raptor-tp6.manifest page_cycles = 25 unit = ms lower_is_better = true @@ -20,49 +18,50 @@ alert_threshold = 2.0 [raptor-firefox-tp6-amazon] apps = firefox test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop -playback_recordings = mitmproxy-recording-amazon.mp +playback_recordings = amazon.mp measure = fnbpaint [raptor-firefox-tp6-facebook] apps = firefox test_url = https://www.facebook.com -playback_recordings = mitmproxy-recording-facebook.mp +playback_recordings = facebook.mp measure = fnbpaint [raptor-firefox-tp6-google] apps = firefox -test_url = https://www.google.com/#hl=en&q=barack+obama -playback_recordings = mitmproxy-recording-google.mp -measure = fnbpaint, hero -hero = hero +# note: use the full url as the first part (without '&cad=h') redirects +# to the url + '&cad=h'; that redirection causes measure.js content +# to be loaded into that page also; resulting in 2 fnbpaint values etc. +test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h +playback_recordings = google-search.mp +measure = fnbpaint [raptor-firefox-tp6-youtube] apps = firefox test_url = https://www.youtube.com -playback_recordings = mitmproxy-recording-youtube.mp +playback_recordings = youtube.mp measure = fnbpaint [raptor-chrome-tp6-amazon] apps = chrome test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop -playback_recordings = mitmproxy-recording-amazon.mp +playback_recordings = amazon.mp measure = fcp [raptor-chrome-tp6-facebook] apps = chrome test_url = https://www.facebook.com -playback_recordings = mitmproxy-recording-facebook.mp +playback_recordings = facebook.mp measure = fcp [raptor-chrome-tp6-google] apps = chrome test_url = https://www.google.com/#hl=en&q=barack+obama -playback_recordings = mitmproxy-recording-google.mp -measure = fcp, hero -hero = hero +playback_recordings = google-search.mp +measure = fcp [raptor-chrome-tp6-youtube] apps = chrome test_url = https://www.youtube.com -playback_recordings = mitmproxy-recording-youtube.mp +playback_recordings = youtube.mp measure = fcp diff --git a/testing/raptor/test/test_playback.py b/testing/raptor/test/test_playback.py index 86f2b4c8ec76..a581449f6a09 100644 --- a/testing/raptor/test/test_playback.py +++ b/testing/raptor/test/test_playback.py @@ -23,9 +23,7 @@ def test_get_playback(get_binary): config['obj_path'] = os.path.dirname(get_binary('firefox')) config['playback_tool'] = 'mitmproxy' config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest' - config['playback_binary_zip_mac'] = 'mitmproxy-2.0.2-osx.tar.gz' config['playback_pageset_manifest'] = 'mitmproxy-playback-set.manifest' - config['playback_pageset_zip_mac'] = 'mitmproxy-recording-set-win10.zip' config['playback_recordings'] = 'mitmproxy-recording-amazon.mp' config['binary'] = get_binary('firefox') playback = get_playback(config) diff --git a/testing/raptor/webext/raptor/manifest.json b/testing/raptor/webext/raptor/manifest.json index 3999d77ee92f..6b44cfd63ac1 100644 --- a/testing/raptor/webext/raptor/manifest.json +++ b/testing/raptor/webext/raptor/manifest.json @@ -16,7 +16,6 @@ "matches": ["*://*.amazon.com/*", "*://*.facebook.com/*", "*://*.google.com/*", - "*://*.google.ca/*", "*://*.youtube.com/*"], "js": ["measure.js"] }, diff --git a/testing/raptor/webext/raptor/measure.js b/testing/raptor/webext/raptor/measure.js index 632dcdcbe7c3..edd72df9c6c7 100644 --- a/testing/raptor/webext/raptor/measure.js +++ b/testing/raptor/webext/raptor/measure.js @@ -93,7 +93,7 @@ function measureHero() { startMark = startMeasure, endMark = heroFound); var perfResult = perfData.getEntriesByName(resultType); - var _result = perfResult[0].duration; + var _result = Math.round(perfResult[0].duration); var resultType = "hero:" + heroFound; sendResult(resultType, _result); perfData.clearMarks();