Bug 1423267 - Add motionmark benchmark to try, mozilla-central. r=rwood

2018-02-08 14:44:28 -05:00 · 2018-02-08 14:44:28 -05:00 · c9827cf1b5
--- a/taskcluster/ci/test/talos.yml
+++ b/taskcluster/ci/test/talos.yml
@ -290,6 +290,23 @@ talos-h2:
            - --add-option
            - --webServer,localhost

+talos-motionmark:
+    description: "Talos motionmark"
+    try-name: motionmark
+    treeherder-symbol: T(mm)
+    run-on-projects: ['mozilla-central', 'try']
+    max-run-time: 3600
+    tier:
+        by-test-platform:
+            windows10-64-ccov/.*: 3
+            linux64-ccov/.*: 3
+            default: 2
+    mozharness:
+        extra-options:
+            - --suite=motionmark
+            - --add-option
+            - --webServer,localhost
+
 talos-other:
    description: "Talos other"
    try-name: other
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@ -70,6 +70,7 @@ talos:
    - talos-tp6
    - talos-tp6-stylo-threads
    - talos-speedometer
+    - talos-motionmark
    - talos-h1
    - talos-h2

@ -117,6 +118,7 @@ linux-qr-talos:
    - talos-tp6
    - talos-tp6-stylo-threads
    - talos-speedometer
+    - talos-motionmark

 linux-qr-tests:
    - cppunit
@ -250,6 +252,7 @@ macosx64-talos:
    - talos-tp6
    - talos-tp6-stylo-threads
    - talos-speedometer
+    - talos-motionmark
    - talos-h1

 macosx64-talos-profiling:
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@ -65,6 +65,9 @@
            "talos_options": ["--geckoProfile"],
            "pagesets_name": "tp5n.zip"
        },
+        "motionmark-e10s": {
+            "tests": ["motionmark_animometer", "motionmark_htmlsuite"]
+        },
        "svgr-e10s": {
            "tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "tsvg_static"]
        },
--- a/testing/talos/talos/config.py
+++ b/testing/talos/talos/config.py
@ -4,10 +4,12 @@
 from __future__ import absolute_import, print_function

 import copy
+import json
 import os
 import sys
 import time

+import mozinfo
 from mozlog.commandline import setup_logging
 from talos import utils, test
 from talos.cmdline import parse_args
@ -340,11 +342,39 @@ def build_manifest(config, manifestName):
    with open(manifestName, 'r') as fHandle:
        manifestLines = fHandle.readlines()

+    # look for configuration data - right now just MotionMark
+    tuning_data = {}
+    if os.path.isfile(manifestName + '.json'):
+        with open(manifestName + '.json', 'r') as f:
+            tuning_data = json.load(f)
+
    # write modified manifest lines
    with open(manifestName + '.develop', 'w') as newHandle:
        for line in manifestLines:
            newline = line.replace('localhost', config['webserver'])
            newline = newline.replace('page_load_test', 'tests')
+
+            if tuning_data:
+                suite = ''
+                test = ''
+                # parse suite/test from: suite-name=HTMLsuite&test-name=CompositedTransforms
+                parts = newline.split('&')
+                for part in parts:
+                    key_val = part.split('=')
+                    if len(key_val) != 2:
+                        continue
+
+                    if key_val[0] == 'suite-name':
+                        suite = key_val[1]
+                    if key_val[0] == 'test-name':
+                        test = key_val[1]
+
+                if suite and test and tuning_data:
+                    osver = mozinfo.os
+                    if osver not in ['linux', 'win']:
+                        osver = 'osx'
+                    complexity = tuning_data[suite]['complexity'][test][osver]
+                    newline = newline.replace('complexity=300', 'complexity=%s' % complexity)
            newHandle.write(newline)

    newManifestName = manifestName + '.develop'
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@ -787,34 +787,38 @@ class a11yr(PageloaderTest):
    alert_threshold = 5.0


+class WebkitBenchmark(PageloaderTest):
+    tpcycles = 1
+    tppagecycles = 5
+    tpmozafterpaint = False
+    tpchrome = False
+    format_pagename = False
+    lower_is_better = False
+    unit = 'score'
+
+
@register_test()
-class speedometer(PageloaderTest):
-    """
-    Speedometer benchmark used by many browser vendors (from webkit)
-    """
+class speedometer(WebkitBenchmark):
+    # Speedometer benchmark used by many browser vendors (from webkit)
    tpmanifest = '${talos}/tests/speedometer/speedometer.manifest'
-    tpcycles = 1
-    tppagecycles = 5
-    tpmozafterpaint = False
-    tpchrome = False
-    format_pagename = False
-    lower_is_better = False
-    unit = 'score'


@register_test()
-class stylebench(PageloaderTest):
-    """
-    StyleBench benchmark used by many browser vendors (from webkit)
-    """
+class stylebench(WebkitBenchmark):
+    # StyleBench benchmark used by many browser vendors (from webkit)
    tpmanifest = '${talos}/tests/stylebench/stylebench.manifest'
-    tpcycles = 1
-    tppagecycles = 5
-    tpmozafterpaint = False
-    tpchrome = False
-    format_pagename = False
-    lower_is_better = False
-    unit = 'score'
+
+
+@register_test()
+class motionmark_animometer(WebkitBenchmark):
+    # MotionMark benchmark used by many browser vendors (from webkit)
+    tpmanifest = '${talos}/tests/motionmark/animometer.manifest'
+
+
+@register_test()
+class motionmark_htmlsuite(WebkitBenchmark):
+    # MotionMark benchmark used by many browser vendors (from webkit)
+    tpmanifest = '${talos}/tests/motionmark/htmlsuite.manifest'


@register_test()
--- a/testing/talos/talos/tests/motionmark/animometer.manifest
+++ b/testing/talos/talos/tests/motionmark/animometer.manifest
@ -0,0 +1,10 @@
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Multiply&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=CanvasArcs&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Leaves&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Paths&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=CanvasLines&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Focus&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Images&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Design&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&test-name=Suits&complexity=300
+
--- a/testing/talos/talos/tests/motionmark/animometer.manifest.json
+++ b/testing/talos/talos/tests/motionmark/animometer.manifest.json
@ -0,0 +1,14 @@
+{"Animometer": {
+  "complexity": {
+    "Multiply": {"win": 391, "linux": 391, "osx": 193}, 
+    "CanvasArcs": {"win": 1287, "linux": 1287, "osx": 575},
+    "Leaves": {"win": 550, "linux": 550, "osx": 271},
+    "Paths": {"win": 4070, "linux": 4070, "osx": 2024},
+    "CanvasLines": {"win": 4692, "linux": 4692, "osx": 10932},
+    "Focus": {"win": 44, "linux": 44, "osx": 32},
+    "Images": {"win": 293, "linux": 293, "osx": 188},
+    "Design": {"win": 60, "linux": 60, "osx": 17},
+    "Suits": {"win": 210, "linux": 210, "osx": 145}    
+    }
+  }
+}
--- a/testing/talos/talos/tests/motionmark/htmlsuite.manifest
+++ b/testing/talos/talos/tests/motionmark/htmlsuite.manifest
@ -0,0 +1,11 @@
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingcircles&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingclippedrects&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncinggradientcircles&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingblendcircles&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingfiltercircles&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingSVGimages&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CSSbouncingtaggedimages&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=Leaves20&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=Focus20&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=DOMparticlesSVGmasks&complexity=300
+% http://localhost/tests/webkit/PerformanceTests/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&test-name=CompositedTransforms&complexity=300
--- a/testing/talos/talos/tests/motionmark/htmlsuite.manifest.json
+++ b/testing/talos/talos/tests/motionmark/htmlsuite.manifest.json
@ -0,0 +1,16 @@
+{"HTMLsuite": {
+  "complexity": {
+    "CSSbouncingcircles": {"win": 322, "linux": 322, "osx": 218},
+    "CSSbouncingclippedrects": {"win": 520, "linux": 520, "osx": 75},
+    "CSSbouncinggradientcircles": {"win": 402, "linux": 402, "osx": 97},
+    "CSSbouncingblendcircles": {"win": 171, "linux": 171, "osx": 254},
+    "CSSbouncingfiltercircles": {"win": 189, "linux": 189, "osx": 189},
+    "CSSbouncingSVGimages": {"win": 329, "linux": 329, "osx": 392},
+    "CSSbouncingtaggedimages": {"win": 255, "linux": 255, "osx": 351},
+    "Leaves20": {"win": 262, "linux": 262, "osx": 191},
+    "Focus20": {"win": 15, "linux": 15, "osx": 18},
+    "DOMparticlesSVGmasks": {"win": 390, "linux": 390, "osx": 54},
+    "CompositedTransforms": {"win": 400, "linux": 400, "osx": 75}
+    }
+  }
+}
--- a/third_party/webkit/PerformanceTests/MotionMark/resources/debug-runner/animometer.js
+++ b/third_party/webkit/PerformanceTests/MotionMark/resources/debug-runner/animometer.js
@ -628,6 +628,19 @@ Utilities.extendObject(window.benchmarkController, {
        }

        var score = dashboard.score;
+        var item = dashboard._results['iterationsResults'][0];
+        var fullNames = new Array;
+        var values = new Array;
+        for (var suite in item['testsResults']) {
+            for (var subtest in item['testsResults'][suite.toString()]) {
+                fullNames.push(suite.toString() + "-" + subtest.toString().replace(/ /g, '_'));
+                values.push(item['testsResults'][suite.toString()][subtest.toString()]['controller']['average']);
+           }
+        }
+        if (typeof tpRecordTime !== "undefined") {
+            tpRecordTime(values.join(','), 0, fullNames.join(','));
+        }
+
        var confidence = ((dashboard.scoreLowerBound / score - 1) * 100).toFixed(2) +
            "% / +" + ((dashboard.scoreUpperBound / score - 1) * 100).toFixed(2) + "%";
        sectionsManager.setSectionScore("results", score.toFixed(2), confidence);
--- a/third_party/webkit/PerformanceTests/MotionMark/resources/extensions.js
+++ b/third_party/webkit/PerformanceTests/MotionMark/resources/extensions.js
@ -371,7 +371,7 @@ UnitBezier = Utilities.createClass(

    sampleY: function(t)
    {
-        return ((this._a.y * t + this._b.y) * t + this._c.y) * t;kkkj
+        return ((this._a.y * t + this._b.y) * t + this._c.y) * t;
    },

    sampleDerivativeX: function(t)