diff --git a/taskcluster/ci/test/raptor-chromium.yml b/taskcluster/ci/test/raptor-chromium.yml index 368d22d347d6..d5c048f87811 100644 --- a/taskcluster/ci/test/raptor-chromium.yml +++ b/taskcluster/ci/test/raptor-chromium.yml @@ -313,3 +313,14 @@ raptor-tp6-4-chromium-cold: extra-options: - --test=raptor-tp6-cold-4 - --app=chromium + +raptor-ares6-chromium: + description: "Raptor Ares6 on Chromium" + try-name: raptor-ares6-chromium + treeherder-symbol: Rap-Cr(ares6) + max-run-time: 2500 + tier: 3 + mozharness: + extra-options: + - --test=raptor-ares6 + - --app=chromium diff --git a/taskcluster/ci/test/raptor.yml b/taskcluster/ci/test/raptor.yml index 00e49ea4b5df..06bb5f66b449 100644 --- a/taskcluster/ci/test/raptor.yml +++ b/taskcluster/ci/test/raptor.yml @@ -279,7 +279,7 @@ raptor-tp6-binast-1-firefox: - --test=raptor-tp6-binast-1 raptor-tp6m-1-geckoview: - description: "Raptor tp6m-1 on Geckoview" + description: "Raptor tp6m-1 on GeckoView" try-name: raptor-tp6m-1-geckoview treeherder-symbol: Rap(tp6m-1) run-on-projects: ['mozilla-central'] @@ -357,7 +357,7 @@ raptor-tp6m-2-fenix: - --activity=org.mozilla.fenix.IntentReceiverActivity raptor-tp6m-2-geckoview: - description: "Raptor tp6m-2 on Geckoview" + description: "Raptor tp6m-2 on GeckoView" try-name: raptor-tp6m-2-geckoview treeherder-symbol: Rap(tp6m-2) target: geckoview_example.apk @@ -391,7 +391,7 @@ raptor-tp6m-2-refbrow: - --activity=org.mozilla.reference.browser.BrowserTestActivity raptor-tp6m-3-geckoview: - description: "Raptor tp6m-3 on Geckoview" + description: "Raptor tp6m-3 on GeckoView" try-name: raptor-tp6m-3-geckoview treeherder-symbol: Rap(tp6m-3) target: geckoview_example.apk @@ -446,7 +446,7 @@ raptor-tp6m-3-fenix: - --activity=org.mozilla.fenix.IntentReceiverActivity raptor-tp6m-4-geckoview: - description: "Raptor tp6m-4 on Geckoview" + description: "Raptor tp6m-4 on GeckoView" try-name: raptor-tp6m-4-geckoview treeherder-symbol: Rap(tp6m-4) target: geckoview_example.apk @@ -501,7 +501,7 @@ raptor-tp6m-4-fenix: - --activity=org.mozilla.fenix.IntentReceiverActivity raptor-tp6m-5-geckoview: - description: "Raptor tp6m-5 on Geckoview" + description: "Raptor tp6m-5 on GeckoView" try-name: raptor-tp6m-5-geckoview treeherder-symbol: Rap(tp6m-5) run-on-projects: ['mozilla-central'] @@ -557,7 +557,7 @@ raptor-tp6m-5-fenix: - --activity=org.mozilla.fenix.IntentReceiverActivity raptor-tp6m-6-geckoview: - description: "Raptor tp6m-6 on Geckoview" + description: "Raptor tp6m-6 on GeckoView" try-name: raptor-tp6m-6-geckoview treeherder-symbol: Rap(tp6m-6) run-on-projects: ['mozilla-central'] @@ -612,7 +612,7 @@ raptor-tp6m-6-fenix: - --activity=org.mozilla.fenix.IntentReceiverActivity raptor-tp6m-7-geckoview: - description: "Raptor tp6m-7 on Geckoview" + description: "Raptor tp6m-7 on GeckoView" try-name: raptor-tp6m-7-geckoview treeherder-symbol: Rap(tp6m-7) run-on-projects: ['mozilla-central'] @@ -668,7 +668,7 @@ raptor-tp6m-7-fenix: - --activity=org.mozilla.fenix.IntentReceiverActivity raptor-tp6m-8-geckoview: - description: "Raptor tp6m-8 on Geckoview" + description: "Raptor tp6m-8 on GeckoView" try-name: raptor-tp6m-8-geckoview treeherder-symbol: Rap(tp6m-8) run-on-projects: ['mozilla-central'] @@ -723,7 +723,7 @@ raptor-tp6m-8-fenix: - --activity=org.mozilla.fenix.IntentReceiverActivity raptor-tp6m-9-geckoview: - description: "Raptor tp6m-9 on Geckoview" + description: "Raptor tp6m-9 on GeckoView" try-name: raptor-tp6m-9-geckoview treeherder-symbol: Rap(tp6m-9) target: geckoview_example.apk @@ -779,7 +779,7 @@ raptor-tp6m-9-fenix: - --activity=org.mozilla.fenix.IntentReceiverActivity raptor-tp6m-10-geckoview: - description: "Raptor tp6m-10 on Geckoview" + description: "Raptor tp6m-10 on GeckoView" try-name: raptor-tp6m-10-geckoview treeherder-symbol: Rap(tp6m-10) target: geckoview_example.apk @@ -1558,7 +1558,7 @@ raptor-speedometer-firefox-profiling: - --gecko-profile raptor-speedometer-geckoview: - description: "Raptor Speedometer on Geckoview" + description: "Raptor Speedometer on GeckoView" try-name: raptor-speedometer-geckoview treeherder-symbol: Rap(sp) target: geckoview_example.apk @@ -1577,7 +1577,7 @@ raptor-speedometer-geckoview: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-speedometer-geckoview-power: - description: "Raptor Speedometer Power on Geckoview" + description: "Raptor Speedometer Power on GeckoView" try-name: raptor-speedometer-geckoview-power treeherder-symbol: Rap(sp-p) target: geckoview_example.apk @@ -1927,7 +1927,7 @@ raptor-unity-webgl-firefox-profiling: - unity-webgl raptor-unity-webgl-geckoview: - description: "Raptor Unity WebGL on Geckoview" + description: "Raptor Unity WebGL on GeckoView" try-name: raptor-unity-webgl-geckoview treeherder-symbol: Rap(ugl) target: geckoview_example.apk @@ -2297,7 +2297,7 @@ raptor-tp6-4-firefox-cold: - --test=raptor-tp6-cold-4 raptor-tp6m-1-geckoview-cold: - description: "Raptor tp6m-1 cold page-load on Geckoview Example" + description: "Raptor tp6m-1 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-1-geckoview-cold treeherder-symbol: Rap(tp6m-c-1) @@ -2319,7 +2319,7 @@ raptor-tp6m-1-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-2-geckoview-cold: - description: "Raptor tp6m-2 cold page-load on Geckoview Example" + description: "Raptor tp6m-2 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-2-geckoview-cold treeherder-symbol: Rap(tp6m-c-2) @@ -2341,7 +2341,7 @@ raptor-tp6m-2-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-3-geckoview-cold: - description: "Raptor tp6m-3 cold page-load on Geckoview Example" + description: "Raptor tp6m-3 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-3-geckoview-cold treeherder-symbol: Rap(tp6m-c-3) @@ -2363,7 +2363,7 @@ raptor-tp6m-3-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-4-geckoview-cold: - description: "Raptor tp6m-4 cold page-load on Geckoview Example" + description: "Raptor tp6m-4 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-4-geckoview-cold treeherder-symbol: Rap(tp6m-c-4) @@ -2385,7 +2385,7 @@ raptor-tp6m-4-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-5-geckoview-cold: - description: "Raptor tp6m-5 cold page-load on Geckoview Example" + description: "Raptor tp6m-5 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-5-geckoview-cold treeherder-symbol: Rap(tp6m-c-5) @@ -2407,7 +2407,7 @@ raptor-tp6m-5-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-6-geckoview-cold: - description: "Raptor tp6m-6 cold page-load on Geckoview Example" + description: "Raptor tp6m-6 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-6-geckoview-cold treeherder-symbol: Rap(tp6m-c-6) @@ -2429,7 +2429,7 @@ raptor-tp6m-6-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-7-geckoview-cold: - description: "Raptor tp6m-7 cold page-load on Geckoview Example" + description: "Raptor tp6m-7 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-7-geckoview-cold treeherder-symbol: Rap(tp6m-c-7) @@ -2448,7 +2448,7 @@ raptor-tp6m-7-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-8-geckoview-cold: - description: "Raptor tp6m-8 cold page-load on Geckoview Example" + description: "Raptor tp6m-8 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-8-geckoview-cold treeherder-symbol: Rap(tp6m-c-8) @@ -2467,7 +2467,7 @@ raptor-tp6m-8-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-9-geckoview-cold: - description: "Raptor tp6m-9 cold page-load on Geckoview Example" + description: "Raptor tp6m-9 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-9-geckoview-cold treeherder-symbol: Rap(tp6m-c-9) @@ -2486,7 +2486,7 @@ raptor-tp6m-9-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-10-geckoview-cold: - description: "Raptor tp6m-10 cold page-load on Geckoview Example" + description: "Raptor tp6m-10 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-10-geckoview-cold treeherder-symbol: Rap(tp6m-c-10) @@ -2505,7 +2505,7 @@ raptor-tp6m-10-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-11-geckoview-cold: - description: "Raptor tp6m-11 cold page-load on Geckoview Example" + description: "Raptor tp6m-11 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-11-geckoview-cold treeherder-symbol: Rap(tp6m-c-11) @@ -2524,7 +2524,7 @@ raptor-tp6m-11-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-12-geckoview-cold: - description: "Raptor tp6m-12 cold page-load on Geckoview Example" + description: "Raptor tp6m-12 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-12-geckoview-cold treeherder-symbol: Rap(tp6m-c-12) @@ -2543,7 +2543,7 @@ raptor-tp6m-12-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-13-geckoview-cold: - description: "Raptor tp6m-13 cold page-load on Geckoview Example" + description: "Raptor tp6m-13 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-13-geckoview-cold treeherder-symbol: Rap(tp6m-c-13) @@ -2562,7 +2562,7 @@ raptor-tp6m-13-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-14-geckoview-cold: - description: "Raptor tp6m-14 cold page-load on Geckoview Example" + description: "Raptor tp6m-14 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-14-geckoview-cold treeherder-symbol: Rap(tp6m-c-14) @@ -2581,7 +2581,7 @@ raptor-tp6m-14-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-15-geckoview-cold: - description: "Raptor tp6m-15 cold page-load on Geckoview Example" + description: "Raptor tp6m-15 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-15-geckoview-cold treeherder-symbol: Rap(tp6m-c-15) @@ -2600,7 +2600,7 @@ raptor-tp6m-15-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-16-geckoview-cold: - description: "Raptor tp6m-16 cold page-load on Geckoview Example" + description: "Raptor tp6m-16 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-16-geckoview-cold treeherder-symbol: Rap(tp6m-c-16) @@ -2619,7 +2619,7 @@ raptor-tp6m-16-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-17-geckoview-cold: - description: "Raptor tp6m-17 cold page-load on Geckoview Example" + description: "Raptor tp6m-17 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-17-geckoview-cold treeherder-symbol: Rap(tp6m-c-17) @@ -2638,7 +2638,7 @@ raptor-tp6m-17-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-18-geckoview-cold: - description: "Raptor tp6m-18 cold page-load on Geckoview Example" + description: "Raptor tp6m-18 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-18-geckoview-cold treeherder-symbol: Rap(tp6m-c-18) @@ -2657,7 +2657,7 @@ raptor-tp6m-18-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-19-geckoview-cold: - description: "Raptor tp6m-19 cold page-load on Geckoview Example" + description: "Raptor tp6m-19 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-19-geckoview-cold treeherder-symbol: Rap(tp6m-c-19) @@ -2676,7 +2676,7 @@ raptor-tp6m-19-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-20-geckoview-cold: - description: "Raptor tp6m-20 cold page-load on Geckoview Example" + description: "Raptor tp6m-20 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-20-geckoview-cold treeherder-symbol: Rap(tp6m-c-20) @@ -2695,7 +2695,7 @@ raptor-tp6m-20-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-21-geckoview-cold: - description: "Raptor tp6m-21 cold page-load on Geckoview Example" + description: "Raptor tp6m-21 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-21-geckoview-cold treeherder-symbol: Rap(tp6m-c-21) @@ -2714,7 +2714,7 @@ raptor-tp6m-21-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-22-geckoview-cold: - description: "Raptor tp6m-22 cold page-load on Geckoview Example" + description: "Raptor tp6m-22 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-22-geckoview-cold treeherder-symbol: Rap(tp6m-c-22) @@ -2733,7 +2733,7 @@ raptor-tp6m-22-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-23-geckoview-cold: - description: "Raptor tp6m-23 cold page-load on Geckoview Example" + description: "Raptor tp6m-23 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-23-geckoview-cold treeherder-symbol: Rap(tp6m-c-23) @@ -2752,7 +2752,7 @@ raptor-tp6m-23-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-24-geckoview-cold: - description: "Raptor tp6m-24 cold page-load on Geckoview Example" + description: "Raptor tp6m-24 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-24-geckoview-cold treeherder-symbol: Rap(tp6m-c-24) @@ -2771,7 +2771,7 @@ raptor-tp6m-24-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-25-geckoview-cold: - description: "Raptor tp6m-25 cold page-load on Geckoview Example" + description: "Raptor tp6m-25 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-25-geckoview-cold treeherder-symbol: Rap(tp6m-c-25) @@ -2790,7 +2790,7 @@ raptor-tp6m-25-geckoview-cold: - --activity=org.mozilla.geckoview_example.GeckoViewActivity raptor-tp6m-26-geckoview-cold: - description: "Raptor tp6m-26 cold page-load on Geckoview Example" + description: "Raptor tp6m-26 cold page-load on GeckoView Example" max-run-time: 2700 try-name: raptor-tp6m-26-geckoview-cold treeherder-symbol: Rap(tp6m-c-26) @@ -4249,3 +4249,26 @@ raptor-tp6m-26-refbrow-cold: - --app=refbrow - --binary-path=org.mozilla.reference.browser.raptor - --activity=org.mozilla.reference.browser.BrowserTestActivity + +raptor-ares6-firefox: + description: "Raptor Ares6 on Firefox" + variants: ["fission"] + try-name: raptor-ares6-firefox + treeherder-symbol: Rap(ares6) + max-run-time: 2500 + tier: 3 + mozharness: + extra-options: + - --test=raptor-ares6 + +raptor-ares6-firefox-profiling: + description: "Raptor Ares6 on Firefox with Gecko Profiling" + try-name: raptor-ares6-firefox-profiling + treeherder-symbol: Rap-Prof(ares6) + run-on-projects: ['mozilla-central', 'try'] + max-run-time: 2500 + tier: 3 + mozharness: + extra-options: + - --test=raptor-ares6 + - --gecko-profile diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml index 4db486e7e767..4f637f0b40f2 100644 --- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -103,6 +103,7 @@ raptor-firefox: - raptor-tp6-2-firefox-cold - raptor-tp6-3-firefox-cold - raptor-tp6-4-firefox-cold + - raptor-ares6-firefox raptor-profiling: - raptor-tp6-1-firefox-profiling @@ -122,6 +123,7 @@ raptor-profiling: - raptor-webaudio-firefox-profiling - raptor-sunspider-firefox-profiling - raptor-wasm-godot-firefox-profiling + - raptor-ares6-firefox-profiling raptor-chromium: - raptor-tp6-1-chromium @@ -145,6 +147,7 @@ raptor-chromium: - raptor-tp6-2-chromium-cold - raptor-tp6-3-chromium-cold - raptor-tp6-4-chromium-cold + - raptor-ares6-chromium # Fetch tasks are only supported on Linux for now, # so these need to be separate sets. diff --git a/testing/raptor/raptor/output.py b/testing/raptor/raptor/output.py index 58375c551863..58369506212e 100644 --- a/testing/raptor/raptor/output.py +++ b/testing/raptor/raptor/output.py @@ -133,6 +133,8 @@ class Output(object): elif test.type == "benchmark": if 'assorted-dom' in test.measurements: subtests, vals = self.parseAssortedDomOutput(test) + elif 'ares6' in test.measurements: + subtests, vals = self.parseAresSixOutput(test) elif 'motionmark' in test.measurements: subtests, vals = self.parseMotionmarkOutput(test) elif 'speedometer' in test.measurements: @@ -412,6 +414,109 @@ class Output(object): return subtests, vals + def parseAresSixOutput(self, test): + # https://browserbench.org/ARES-6/ + # Every pagecycle will perform the tests from the index page + # We have 4 main tests per index page: + # - Air, Basic, Babylon, ML + # - and from these 4 above, ares6 generates the Overall results + # Each test has 3 subtests (firstIteration, steadyState, averageWorstCase): + # - _steadyState + # - _firstIteration + # - _averageWorstCase + # Each index page will run 5 cycles, this is set in glue.js + # + # { + # 'expected_browser_cycles': 1, + # 'subtest_unit': 'ms', + # 'name': 'raptor-ares6-firefox', + # 'lower_is_better': False, + # 'browser_cycle': '1', + # 'subtest_lower_is_better': True, + # 'cold': False, + # 'browser': 'Firefox 69.0a1 20190531035909', + # 'type': 'benchmark', + # 'page': 'http://127.0.0.1:35369/ARES-6/index.html?raptor', + # 'unit': 'ms', + # 'alert_threshold': 2 + # 'measurements': { + # 'ares6': [[{ + # 'Babylon_firstIteration': [ + # 123.68, + # 168.21999999999997, + # 127.34000000000003, + # 113.56, + # 128.78, + # 169.44000000000003 + # ], + # 'Air_steadyState': [ + # 21.184723618090434, + # 22.906331658291457, + # 19.939396984924624, + # 20.572462311557775, + # 20.790452261306534, + # 18.378693467336696 + # ], + # etc. + # }]] + # } + # } + # + # Details on how /ARES6/index.html is showing the mean on subsequent test results: + # + # I selected just a small part from the metrics just to be easier to explain + # what is going on. + # + # After the raptor GeckoView test finishes, we have these results in the logs: + # + # Extracted from "INFO - raptor-control-server Info: received webext_results:" + # 'Air_firstIteration': [660.8000000000002, 626.4599999999999, 655.6199999999999, + # 635.9000000000001, 636.4000000000001] + # + # Extracted from "INFO - raptor-output Info: PERFHERDER_DATA:" + # {"name": "Air_firstIteration", "lowerIsBetter": true, "alertThreshold": 2.0, + # "replicates": [660.8, 626.46, 655.62, 635.9, 636.4], "value": 636.4, "unit": "ms"} + # + # On GeckoView's /ARES6/index.html this is what we see for Air - First Iteration: + # + # - on 1st test cycle : 660.80 (rounded from 660.8000000000002) + # + # - on 2nd test cycle : 643.63 , this is coming from + # (660.8000000000002 + 626.4599999999999) / 2 , + # then rounded up to a precision of 2 decimals + # + # - on 3rd test cycle : 647.63 this is coming from + # (660.8000000000002 + 626.4599999999999 + 655.6199999999999) / 3 , + # then rounded up to a precision of 2 decimals + # + # - and so on + # + + _subtests = {} + data = test.measurements['ares6'] + for page_cycle in data: + for sub, replicates in page_cycle[0].iteritems(): + # for each pagecycle, build a list of subtests and append all related replicates + if sub not in _subtests.keys(): + # subtest not added yet, first pagecycle, so add new one + _subtests[sub] = {'unit': test.subtest_unit, + 'alertThreshold': float(test.alert_threshold), + 'lowerIsBetter': test.subtest_lower_is_better, + 'name': sub, + 'replicates': []} + _subtests[sub]['replicates'].extend([round(x, 3) for x in replicates]) + + vals = [] + subtests = [] + names = _subtests.keys() + names.sort(reverse=True) + for name in names: + _subtests[name]['value'] = filters.mean(_subtests[name]['replicates']) + subtests.append(_subtests[name]) + vals.append([_subtests[name]['value'], name]) + + return subtests, vals + def parseWASMMiscOutput(self, test): ''' {u'wasm-misc': [ diff --git a/testing/raptor/raptor/raptor.ini b/testing/raptor/raptor/raptor.ini index 2601362a08d2..6dbbb86bf166 100644 --- a/testing/raptor/raptor/raptor.ini +++ b/testing/raptor/raptor/raptor.ini @@ -85,6 +85,7 @@ # raptor benchmark tests [include:tests/benchmarks/raptor-assorted-dom.ini] +[include:tests/benchmarks/raptor-ares6.ini] [include:tests/benchmarks/raptor-motionmark-animometer.ini] [include:tests/benchmarks/raptor-motionmark-htmlsuite.ini] [include:tests/benchmarks/raptor-speedometer.ini] diff --git a/testing/raptor/raptor/tests/benchmarks/raptor-ares6.ini b/testing/raptor/raptor/tests/benchmarks/raptor-ares6.ini new file mode 100644 index 000000000000..fa0a0ec763a5 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/raptor-ares6.ini @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# ARES-6 benchmark for firefox and chromium distributions + +[DEFAULT] +type = benchmark +test_url = http://:/ARES-6/index.html?raptor +page_cycles = 4 +page_timeout = 270000 +unit = ms +subtest_unit = ms +lower_is_better = true +subtest_lower_is_better = true +alert_threshold = 2.0 +gecko_profile_interval = 1 +gecko_profile_entries = 14000000 + +[raptor-ares6-firefox] +apps = firefox + +[raptor-ares6-chromium] +apps = chromium diff --git a/testing/raptor/webext/raptor/manifest.json b/testing/raptor/webext/raptor/manifest.json index a60e094aa5f5..445b3239a463 100644 --- a/testing/raptor/webext/raptor/manifest.json +++ b/testing/raptor/webext/raptor/manifest.json @@ -65,7 +65,8 @@ "*://*/wasm-misc/index.html*", "*://*/wasm-godot/index.html*", "*://*/assorted-dom/assorted/results.html*", - "*://*.mozaws.net/*" + "*://*.mozaws.net/*", + "*://*/ARES-6/index.html*" ], "js": ["benchmark.js"], "run_at": "document_end" diff --git a/third_party/webkit/PerformanceTests/ARES-6/driver.js b/third_party/webkit/PerformanceTests/ARES-6/driver.js index 633721ce3d38..0650b7d94c73 100644 --- a/third_party/webkit/PerformanceTests/ARES-6/driver.js +++ b/third_party/webkit/PerformanceTests/ARES-6/driver.js @@ -159,6 +159,8 @@ class Driver { this.readyTrigger(); if (typeof tpRecordTime !== "undefined") tpRecordTime(this._values.join(','), 0, this._names.join(',')); + + this.sendResultsToRaptor(); } else print("Success! Benchmark is now finished."); return; @@ -217,4 +219,36 @@ class Driver { } } + + sendResultsToRaptor() + { + // this contains all test names = [ "Air_firstIteration", "Air_averageWorstCase", ...] + var allNames = this._names; + // this contains all test metrics = [ 111, 83.5, 21.78894472361809, ...] + var allValues = this._values; + // this object will store name:[value1, value2, ...] pairs for the arrays above + var measuredValuesByFullName = {}; + for (var i = 0, len = allNames.length; i < len; i++) { + if (measuredValuesByFullName[allNames[i]] === undefined) { + measuredValuesByFullName[allNames[i]] = []; + } + } + + allNames.map(function(name, index) { + // now we save all the values for each test + // ex.: measuredValuesByFullName['Air_firstIteration'].push(111); + // measuredValuesByFullName['Air_firstIteration'].push(107); + measuredValuesByFullName[name].push(allValues[index]); + }); + + // delete the geomean array - this will be calculated by raptor + delete measuredValuesByFullName.geomean; + + if (location.search === '?raptor') { + var _data = ['raptor-benchmark', 'ares6', measuredValuesByFullName]; + console.log('ares6 source is about to post results to the raptor webext'); + window.postMessage(_data, '*'); + } + + } } diff --git a/third_party/webkit/PerformanceTests/ARES-6/glue.js b/third_party/webkit/PerformanceTests/ARES-6/glue.js index 8553200ed6e7..91871036ba7c 100644 --- a/third_party/webkit/PerformanceTests/ARES-6/glue.js +++ b/third_party/webkit/PerformanceTests/ARES-6/glue.js @@ -53,6 +53,6 @@ driver.addBenchmark(BabylonBenchmarkRunner); driver.addBenchmark(MLBenchmarkRunner); driver.readyTrigger(); -if (typeof location !== 'undefined' && location.search == '?gecko') { - driver.start(6); +if (typeof location !== 'undefined' && (location.search === '?gecko' || location.search === '?raptor')) { + driver.start(5); }