From c9827cf1b5f492ce1e80fb2204164ba67998d0c1 Mon Sep 17 00:00:00 2001 From: Joel Maher Date: Thu, 8 Feb 2018 14:44:28 -0500 Subject: [PATCH] Bug 1423267 - Add motionmark benchmark to try, mozilla-central. r=rwood --- taskcluster/ci/test/talos.yml | 17 +++++++ taskcluster/ci/test/test-sets.yml | 3 ++ testing/talos/talos.json | 3 ++ testing/talos/talos/config.py | 30 ++++++++++++ testing/talos/talos/test.py | 48 ++++++++++--------- .../tests/motionmark/animometer.manifest | 10 ++++ .../tests/motionmark/animometer.manifest.json | 14 ++++++ .../talos/tests/motionmark/htmlsuite.manifest | 11 +++++ .../tests/motionmark/htmlsuite.manifest.json | 16 +++++++ .../resources/debug-runner/animometer.js | 13 +++++ .../MotionMark/resources/extensions.js | 2 +- 11 files changed, 144 insertions(+), 23 deletions(-) create mode 100644 testing/talos/talos/tests/motionmark/animometer.manifest create mode 100644 testing/talos/talos/tests/motionmark/animometer.manifest.json create mode 100644 testing/talos/talos/tests/motionmark/htmlsuite.manifest create mode 100644 testing/talos/talos/tests/motionmark/htmlsuite.manifest.json diff --git a/taskcluster/ci/test/talos.yml b/taskcluster/ci/test/talos.yml index e062bd9e2abb..49cfa7c740a7 100644 --- a/taskcluster/ci/test/talos.yml +++ b/taskcluster/ci/test/talos.yml @@ -290,6 +290,23 @@ talos-h2: - --add-option - --webServer,localhost +talos-motionmark: + description: "Talos motionmark" + try-name: motionmark + treeherder-symbol: T(mm) + run-on-projects: ['mozilla-central', 'try'] + max-run-time: 3600 + tier: + by-test-platform: + windows10-64-ccov/.*: 3 + linux64-ccov/.*: 3 + default: 2 + mozharness: + extra-options: + - --suite=motionmark + - --add-option + - --webServer,localhost + talos-other: description: "Talos other" try-name: other diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml index 10d94cc2c65a..712e4aafa6d7 100644 --- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -70,6 +70,7 @@ talos: - talos-tp6 - talos-tp6-stylo-threads - talos-speedometer + - talos-motionmark - talos-h1 - talos-h2 @@ -117,6 +118,7 @@ linux-qr-talos: - talos-tp6 - talos-tp6-stylo-threads - talos-speedometer + - talos-motionmark linux-qr-tests: - cppunit @@ -250,6 +252,7 @@ macosx64-talos: - talos-tp6 - talos-tp6-stylo-threads - talos-speedometer + - talos-motionmark - talos-h1 macosx64-talos-profiling: diff --git a/testing/talos/talos.json b/testing/talos/talos.json index d07740bc31f0..021b4b499ffa 100644 --- a/testing/talos/talos.json +++ b/testing/talos/talos.json @@ -65,6 +65,9 @@ "talos_options": ["--geckoProfile"], "pagesets_name": "tp5n.zip" }, + "motionmark-e10s": { + "tests": ["motionmark_animometer", "motionmark_htmlsuite"] + }, "svgr-e10s": { "tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "tsvg_static"] }, diff --git a/testing/talos/talos/config.py b/testing/talos/talos/config.py index fd59166747ae..eb93f1c47359 100644 --- a/testing/talos/talos/config.py +++ b/testing/talos/talos/config.py @@ -4,10 +4,12 @@ from __future__ import absolute_import, print_function import copy +import json import os import sys import time +import mozinfo from mozlog.commandline import setup_logging from talos import utils, test from talos.cmdline import parse_args @@ -340,11 +342,39 @@ def build_manifest(config, manifestName): with open(manifestName, 'r') as fHandle: manifestLines = fHandle.readlines() + # look for configuration data - right now just MotionMark + tuning_data = {} + if os.path.isfile(manifestName + '.json'): + with open(manifestName + '.json', 'r') as f: + tuning_data = json.load(f) + # write modified manifest lines with open(manifestName + '.develop', 'w') as newHandle: for line in manifestLines: newline = line.replace('localhost', config['webserver']) newline = newline.replace('page_load_test', 'tests') + + if tuning_data: + suite = '' + test = '' + # parse suite/test from: suite-name=HTMLsuite&test-name=CompositedTransforms + parts = newline.split('&') + for part in parts: + key_val = part.split('=') + if len(key_val) != 2: + continue + + if key_val[0] == 'suite-name': + suite = key_val[1] + if key_val[0] == 'test-name': + test = key_val[1] + + if suite and test and tuning_data: + osver = mozinfo.os + if osver not in ['linux', 'win']: + osver = 'osx' + complexity = tuning_data[suite]['complexity'][test][osver] + newline = newline.replace('complexity=300', 'complexity=%s' % complexity) newHandle.write(newline) newManifestName = manifestName + '.develop' diff --git a/testing/talos/talos/test.py b/testing/talos/talos/test.py index 0fac2d9e1676..623485f8a283 100644 --- a/testing/talos/talos/test.py +++ b/testing/talos/talos/test.py @@ -787,34 +787,38 @@ class a11yr(PageloaderTest): alert_threshold = 5.0 +class WebkitBenchmark(PageloaderTest): + tpcycles = 1 + tppagecycles = 5 + tpmozafterpaint = False + tpchrome = False + format_pagename = False + lower_is_better = False + unit = 'score' + + @register_test() -class speedometer(PageloaderTest): - """ - Speedometer benchmark used by many browser vendors (from webkit) - """ +class speedometer(WebkitBenchmark): + # Speedometer benchmark used by many browser vendors (from webkit) tpmanifest = '${talos}/tests/speedometer/speedometer.manifest' - tpcycles = 1 - tppagecycles = 5 - tpmozafterpaint = False - tpchrome = False - format_pagename = False - lower_is_better = False - unit = 'score' @register_test() -class stylebench(PageloaderTest): - """ - StyleBench benchmark used by many browser vendors (from webkit) - """ +class stylebench(WebkitBenchmark): + # StyleBench benchmark used by many browser vendors (from webkit) tpmanifest = '${talos}/tests/stylebench/stylebench.manifest' - tpcycles = 1 - tppagecycles = 5 - tpmozafterpaint = False - tpchrome = False - format_pagename = False - lower_is_better = False - unit = 'score' + + +@register_test() +class motionmark_animometer(WebkitBenchmark): + # MotionMark benchmark used by many browser vendors (from webkit) + tpmanifest = '${talos}/tests/motionmark/animometer.manifest' + + +@register_test() +class motionmark_htmlsuite(WebkitBenchmark): + # MotionMark benchmark used by many browser vendors (from webkit) + tpmanifest = '${talos}/tests/motionmark/htmlsuite.manifest' @register_test() diff --git a/testing/talos/talos/tests/motionmark/animometer.manifest b/testing/talos/talos/tests/motionmark/animometer.manifest new file mode 100644 index 000000000000..cea4a2174855 --- /dev/null +++ b/testing/talos/talos/tests/motionmark/animometer.manifest @@ -0,0 +1,10 @@ +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Multiply&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=CanvasArcs&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Leaves&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Paths&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=CanvasLines&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Focus&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Images&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Design&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Suits&complexity=300 + diff --git a/testing/talos/talos/tests/motionmark/animometer.manifest.json b/testing/talos/talos/tests/motionmark/animometer.manifest.json new file mode 100644 index 000000000000..2ef6d7031953 --- /dev/null +++ b/testing/talos/talos/tests/motionmark/animometer.manifest.json @@ -0,0 +1,14 @@ +{"Animometer": { + "complexity": { + "Multiply": {"win": 391, "linux": 391, "osx": 193}, + "CanvasArcs": {"win": 1287, "linux": 1287, "osx": 575}, + "Leaves": {"win": 550, "linux": 550, "osx": 271}, + "Paths": {"win": 4070, "linux": 4070, "osx": 2024}, + "CanvasLines": {"win": 4692, "linux": 4692, "osx": 10932}, + "Focus": {"win": 44, "linux": 44, "osx": 32}, + "Images": {"win": 293, "linux": 293, "osx": 188}, + "Design": {"win": 60, "linux": 60, "osx": 17}, + "Suits": {"win": 210, "linux": 210, "osx": 145} + } + } +} \ No newline at end of file diff --git a/testing/talos/talos/tests/motionmark/htmlsuite.manifest b/testing/talos/talos/tests/motionmark/htmlsuite.manifest new file mode 100644 index 000000000000..b15b3d6bb80d --- /dev/null +++ b/testing/talos/talos/tests/motionmark/htmlsuite.manifest @@ -0,0 +1,11 @@ +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingcircles&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingclippedrects&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncinggradientcircles&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingblendcircles&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingfiltercircles&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingSVGimages&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingtaggedimages&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=Leaves20&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=Focus20&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=DOMparticlesSVGmasks&complexity=300 +% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CompositedTransforms&complexity=300 diff --git a/testing/talos/talos/tests/motionmark/htmlsuite.manifest.json b/testing/talos/talos/tests/motionmark/htmlsuite.manifest.json new file mode 100644 index 000000000000..0393036dfda2 --- /dev/null +++ b/testing/talos/talos/tests/motionmark/htmlsuite.manifest.json @@ -0,0 +1,16 @@ +{"HTMLsuite": { + "complexity": { + "CSSbouncingcircles": {"win": 322, "linux": 322, "osx": 218}, + "CSSbouncingclippedrects": {"win": 520, "linux": 520, "osx": 75}, + "CSSbouncinggradientcircles": {"win": 402, "linux": 402, "osx": 97}, + "CSSbouncingblendcircles": {"win": 171, "linux": 171, "osx": 254}, + "CSSbouncingfiltercircles": {"win": 189, "linux": 189, "osx": 189}, + "CSSbouncingSVGimages": {"win": 329, "linux": 329, "osx": 392}, + "CSSbouncingtaggedimages": {"win": 255, "linux": 255, "osx": 351}, + "Leaves20": {"win": 262, "linux": 262, "osx": 191}, + "Focus20": {"win": 15, "linux": 15, "osx": 18}, + "DOMparticlesSVGmasks": {"win": 390, "linux": 390, "osx": 54}, + "CompositedTransforms": {"win": 400, "linux": 400, "osx": 75} + } + } +} \ No newline at end of file diff --git a/third_party/webkit/PerformanceTests/MotionMark/resources/debug-runner/animometer.js b/third_party/webkit/PerformanceTests/MotionMark/resources/debug-runner/animometer.js index d4e9790ad08c..d6bd7434c031 100644 --- a/third_party/webkit/PerformanceTests/MotionMark/resources/debug-runner/animometer.js +++ b/third_party/webkit/PerformanceTests/MotionMark/resources/debug-runner/animometer.js @@ -628,6 +628,19 @@ Utilities.extendObject(window.benchmarkController, { } var score = dashboard.score; + var item = dashboard._results['iterationsResults'][0]; + var fullNames = new Array; + var values = new Array; + for (var suite in item['testsResults']) { + for (var subtest in item['testsResults'][suite.toString()]) { + fullNames.push(suite.toString() + "-" + subtest.toString().replace(/ /g, '_')); + values.push(item['testsResults'][suite.toString()][subtest.toString()]['controller']['average']); + } + } + if (typeof tpRecordTime !== "undefined") { + tpRecordTime(values.join(','), 0, fullNames.join(',')); + } + var confidence = ((dashboard.scoreLowerBound / score - 1) * 100).toFixed(2) + "% / +" + ((dashboard.scoreUpperBound / score - 1) * 100).toFixed(2) + "%"; sectionsManager.setSectionScore("results", score.toFixed(2), confidence); diff --git a/third_party/webkit/PerformanceTests/MotionMark/resources/extensions.js b/third_party/webkit/PerformanceTests/MotionMark/resources/extensions.js index 69fe374ee7eb..fb9d500877e3 100644 --- a/third_party/webkit/PerformanceTests/MotionMark/resources/extensions.js +++ b/third_party/webkit/PerformanceTests/MotionMark/resources/extensions.js @@ -371,7 +371,7 @@ UnitBezier = Utilities.createClass( sampleY: function(t) { - return ((this._a.y * t + this._b.y) * t + this._c.y) * t;kkkj + return ((this._a.y * t + this._b.y) * t + this._c.y) * t; }, sampleDerivativeX: function(t)