Bug 1493648 - Can we run the Godot Engine wasm benchmark in automation? r=jmaher

Reviewers: jmaher

Tags: #secure-revision

Bug #: 1493648

Differential Revision: https://phabricator.services.mozilla.com/D8957
This commit is contained in:
florin.strugariu 2018-10-17 10:40:57 +03:00
Родитель a8ec527c46
Коммит c7398f447c
12 изменённых файлов: 546 добавлений и 10 удалений

Просмотреть файл

@ -345,3 +345,26 @@ raptor-assorted-dom-chrome:
fetches:
fetch:
- assorted-dom
raptor-wasm-godot-firefox:
description: "Raptor Wasm GoDot on Firefox"
try-name: raptor-wasm-godot-firefox
treeherder-symbol: Rap(godot)
run-on-projects: ['try', 'mozilla-central']
tier: 2
max-run-time: 1500
mozharness:
extra-options:
- --test=raptor-wasm-godot
raptor-wasm-godot-chrome:
description: "Raptor Wasm GoDot on Chrome"
try-name: raptor-wasm-godot-chrome
treeherder-symbol: Rap-C(godot)
run-on-projects: ['try', 'mozilla-central']
tier: 2
max-run-time: 1500
mozharness:
extra-options:
- --test=raptor-wasm-godot
- --app=chrome

Просмотреть файл

@ -88,6 +88,7 @@ raptor-firefox:
- raptor-webaudio-firefox
- raptor-gdocs-firefox
- raptor-sunspider-firefox
- raptor-wasm-godot-firefox
raptor-chrome:
- raptor-tp6-chrome
@ -98,6 +99,7 @@ raptor-chrome:
- raptor-webaudio-chrome
- raptor-gdocs-chrome
- raptor-sunspider-chrome
- raptor-wasm-godot-firefox
# Fetch tasks are only supported on Linux for now,
# so these need to be separate sets.

Просмотреть файл

@ -103,6 +103,10 @@ def write_test_settings_json(test_details, oskey):
if test_details.get("alert_threshold", None) is not None:
test_settings['raptor-options']['alert_threshold'] = float(test_details['alert_threshold'])
if test_details.get("newtab_per_cycle", None) is not None:
test_settings['raptor-options']['newtab_per_cycle'] = \
bool(test_details['newtab_per_cycle'])
settings_file = os.path.join(tests_dir, test_details['name'] + '.json')
try:
with open(settings_file, 'w') as out_file:

Просмотреть файл

@ -117,6 +117,8 @@ class Output(object):
subtests, vals = self.parseAssortedDomOutput(test)
elif 'wasm-misc' in test.measurements:
subtests, vals = self.parseWASMMiscOutput(test)
elif 'wasm-godot' in test.measurements:
subtests, vals = self.parseWASMGoDotOutput(test)
suite['subtests'] = subtests
else:
@ -217,6 +219,45 @@ class Output(object):
return subtests, vals
def parseWASMGoDotOutput(self, test):
'''
{u'wasm-godot': [
{
"name": "wasm-instantiate",
"time": 349
},{
"name": "engine-instantiate",
"time": 1263
...
}]}
'''
_subtests = {}
data = test.measurements['wasm-godot']
print (data)
for page_cycle in data:
for item in page_cycle[0]:
# for each pagecycle, build a list of subtests and append all related replicates
sub = item['name']
if sub not in _subtests.keys():
# subtest not added yet, first pagecycle, so add new one
_subtests[sub] = {'unit': test.subtest_unit,
'alertThreshold': float(test.alert_threshold),
'lowerIsBetter': test.subtest_lower_is_better,
'name': sub,
'replicates': []}
_subtests[sub]['replicates'].append(item['time'])
vals = []
subtests = []
names = _subtests.keys()
names.sort(reverse=True)
for name in names:
_subtests[name]['value'] = filter.median(_subtests[name]['replicates'])
subtests.append(_subtests[name])
vals.append([_subtests[name]['value'], name])
return subtests, vals
def parseWebaudioOutput(self, test):
# each benchmark 'index' becomes a subtest; each pagecycle / iteration
# of the test has multiple values per index/subtest
@ -523,6 +564,14 @@ class Output(object):
results = [i for i, j in val_list if j == '__total__']
return filter.mean(results)
@classmethod
def wasm_godot_score(cls, val_list):
"""
wasm_godot_score: first-interactive mean
"""
results = [i for i, j in val_list if j == 'first-interactive']
return filter.mean(results)
@classmethod
def stylebench_score(cls, val_list):
"""
@ -601,6 +650,8 @@ class Output(object):
return self.assorted_dom_score(vals)
elif testname.startswith('raptor-wasm-misc'):
return self.wasm_misc_score(vals)
elif testname.startswith('raptor-wasm-godot'):
return self.wasm_godot_score(vals)
elif len(vals) > 1:
return round(filter.geometric_mean([i for i, j in vals]), 2)
else:

Просмотреть файл

@ -11,4 +11,5 @@
[include:tests/raptor-wasm-misc.ini]
[include:tests/raptor-wasm-misc-baseline.ini]
[include:tests/raptor-wasm-misc-ion.ini]
[include:tests/raptor-wasm-godot.ini]
[include:tests/raptor-assorted-dom.ini]

Просмотреть файл

@ -0,0 +1,21 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Wasm-godot benchmark for firefox and chrome
[DEFAULT]
type = benchmark
test_url = http://localhost:<port>/wasm-godot/index.html
page_cycles = 5
page_timeout = 120000
unit = ms
lower_is_better = true
alert_threshold = 2.0
newtab_per_cycle = true
[raptor-wasm-godot-firefox]
apps = firefox
[raptor-wasm-godot-chrome]
apps = chrome

Просмотреть файл

@ -27,6 +27,7 @@
"*://*/webaudio/*",
"*://*/unity-webgl/index.html*",
"*://*/wasm-misc/index.html*",
"*://*/wasm-godot/index.html*",
"*://*/assorted-dom/assorted/results.html*"],
"js": ["benchmark-relay.js"]
}

Просмотреть файл

@ -22,6 +22,9 @@ var postStartupDelay = 30000;
// delay (ms) between pageload cycles
var pageCycleDelay = 1000;
var newTabDelay = 1000;
var reuseTab = false;
var browserName;
var ext;
var testName = null;
@ -87,6 +90,10 @@ function getTestSettings() {
results.subtest_lower_is_better = settings.subtest_lower_is_better === true;
results.alert_threshold = settings.alert_threshold;
if (settings.newtab_per_cycle !== undefined) {
reuseTab = settings.newtab_per_cycle;
}
if (settings.page_timeout !== undefined) {
pageTimeout = settings.page_timeout;
}
@ -165,12 +172,16 @@ function getBrowserInfo() {
function testTabCreated(tab) {
testTabID = tab.id;
console.log("opened new empty tab " + testTabID);
nextCycle();
postToControlServer("status", "opened new empty tab " + testTabID);
}
function testTabRemoved(tab) {
postToControlServer("status", "Removed tab " + testTabID);
testTabID = 0;
}
async function testTabUpdated(tab) {
console.log("test tab updated");
postToControlServer("status", "test tab updated " + testTabID);
// wait for pageload test result from content
await waitForResult();
// move on to next cycle (or test complete)
@ -232,12 +243,25 @@ function nextCycle() {
} else if (testType == "benchmark") {
isBenchmarkPending = true;
}
// update the test page - browse to our test URL
ext.tabs.update(testTabID, {url: testURL}, testTabUpdated);
}, pageCycleDelay);
} else {
verifyResults();
}
if (reuseTab && testTabID != 0) {
// close previous test tab
ext.tabs.remove(testTabID);
postToControlServer("status", "closing Tab " + testTabID);
// open new tab
ext.tabs.create({url: "about:blank"});
postToControlServer("status", "Open new tab");
}
setTimeout(function() {
postToControlServer("status", "update tab " + testTabID);
// update the test page - browse to our test URL
ext.tabs.update(testTabID, {url: testURL}, testTabUpdated);
}, newTabDelay);
}, pageCycleDelay);
} else {
verifyResults();
}
}
function timeoutAlarmListener() {
@ -402,8 +426,13 @@ function runner() {
}
// results listener
ext.runtime.onMessage.addListener(resultListener);
// tab creation listener
ext.tabs.onCreated.addListener(testTabCreated);
// tab remove listener
ext.tabs.onRemoved.addListener(testTabRemoved);
// timeout alarm listener
ext.alarms.onAlarm.addListener(timeoutAlarmListener);
@ -412,11 +441,15 @@ function runner() {
var text = "* pausing " + postStartupDelay / 1000 + " seconds to let browser settle... *";
postToControlServer("status", text);
// setTimeout(function() { nextCycle(); }, postStartupDelay);
// on geckoview you can't create a new tab; only using existing tab - set it blank first
if (config.browser == "geckoview") {
setTimeout(function() { nextCycle(); }, postStartupDelay);
} else {
setTimeout(function() { ext.tabs.create({url: "about:blank"}); }, postStartupDelay);
setTimeout(function() {
ext.tabs.create({url: "about:blank"});
nextCycle();
}, postStartupDelay);
}
});
});

4
third_party/webkit/PerformanceTests/wasm-godot/godot.js поставляемый Normal file

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

Двоичные данные
third_party/webkit/PerformanceTests/wasm-godot/godot.pck поставляемый Normal file

Двоичный файл не отображается.

Двоичные данные
third_party/webkit/PerformanceTests/wasm-godot/godot.wasm поставляемый Normal file

Двоичный файл не отображается.

396
third_party/webkit/PerformanceTests/wasm-godot/index.html поставляемый Normal file
Просмотреть файл

@ -0,0 +1,396 @@
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="" xml:lang="">
<!-- Mirrored from godot.eska.me/pub/wasm-benchmark/ by HTTrack Website Copier/3.x [XR&CO'2014], Tue, 16 Oct 2018 12:26:59 GMT -->
<head>
<meta charset="utf-8" />
<title></title>
<style type="text/css">
body {
margin: 0;
border: 0 none;
padding: 0;
text-align: center;
background-color: #222226;
font-family: 'Noto Sans', Arial, sans-serif;
}
/* Godot Engine default theme style
* ================================ */
.godot {
color: #e0e0e0;
background-color: #3b3943;
background-image: linear-gradient(to bottom, #403e48, #35333c);
border: 1px solid #45434e;
box-shadow: 0 0 1px 1px #2f2d35;
}
button.godot {
font-family: 'Droid Sans', Arial, sans-serif; /* override user agent style */
padding: 1px 5px;
background-color: #37353f;
background-image: linear-gradient(to bottom, #413e49, #3a3842);
border: 1px solid #514f5d;
border-radius: 1px;
box-shadow: 0 0 1px 1px #2a2930;
}
button.godot:hover {
color: #f0f0f0;
background-color: #44414e;
background-image: linear-gradient(to bottom, #494652, #423f4c);
border: 1px solid #5a5667;
box-shadow: 0 0 1px 1px #26252b;
}
button.godot:active {
color: #fff;
background-color: #3e3b46;
background-image: linear-gradient(to bottom, #36343d, #413e49);
border: 1px solid #4f4c59;
box-shadow: 0 0 1px 1px #26252b;
}
button.godot:disabled {
color: rgba(230, 230, 230, 0.2);
background-color: #3d3d3d;
background-image: linear-gradient(to bottom, #434343, #393939);
border: 1px solid #474747;
box-shadow: 0 0 1px 1px #2d2b33;
}
/* Canvas / wrapper
* ================ */
#container {
display: inline-block; /* scale with canvas */
vertical-align: top; /* prevent extra height */
position: relative; /* root for absolutely positioned overlay */
margin: 0;
border: 0 none;
padding: 0;
background-color: #0c0c0c;
}
#canvas {
display: block;
margin: 0 auto;
color: white;
}
#canvas:focus {
outline: none;
}
/* Status display
* ============== */
#status {
position: absolute;
left: 0;
top: 0;
right: 0;
bottom: 0;
display: flex;
justify-content: center;
align-items: center;
/* don't consume click events - make children visible explicitly */
visibility: hidden;
}
#status-progress {
width: 244px;
height: 7px;
background-color: #38363A;
border: 1px solid #444246;
padding: 1px;
box-shadow: 0 0 2px 1px #1B1C22;
border-radius: 2px;
visibility: visible;
}
#status-progress-inner {
height: 100%;
width: 0;
box-sizing: border-box;
transition: width 0.5s linear;
background-color: #202020;
border: 1px solid #222223;
box-shadow: 0 0 1px 1px #27282E;
border-radius: 3px;
}
#status-indeterminate {
visibility: visible;
position: relative;
}
#status-indeterminate > div {
width: 3px;
height: 0;
border-style: solid;
border-width: 6px 2px 0 2px;
border-color: #2b2b2b transparent transparent transparent;
transform-origin: center 14px;
position: absolute;
}
#status-indeterminate > div:nth-child(1) { transform: rotate( 22.5deg); }
#status-indeterminate > div:nth-child(2) { transform: rotate( 67.5deg); }
#status-indeterminate > div:nth-child(3) { transform: rotate(112.5deg); }
#status-indeterminate > div:nth-child(4) { transform: rotate(157.5deg); }
#status-indeterminate > div:nth-child(5) { transform: rotate(202.5deg); }
#status-indeterminate > div:nth-child(6) { transform: rotate(247.5deg); }
#status-indeterminate > div:nth-child(7) { transform: rotate(292.5deg); }
#status-indeterminate > div:nth-child(8) { transform: rotate(337.5deg); }
#status-notice {
margin: 0 100px;
line-height: 1.3;
visibility: visible;
padding: 4px 6px;
visibility: visible;
}
/* Debug output
* ============ */
#output-panel {
max-width: 600px;
font-size: small;
margin: 6px auto 0;
padding: 0 4px 4px;
text-align: left;
line-height: 2.2;
}
#output-header {
display: flex;
justify-content: space-between;
align-items: center;
}
#output-container {
padding: 6px;
background-color: #2c2a32;
box-shadow: inset 0 0 1px 1px #232127;
color: #bbb;
}
#output-scroll {
line-height: 1;
height: 8em;
overflow-y: scroll;
white-space: pre-wrap;
font-size: small;
font-family: "Lucida Console", Monaco, monospace;
}
</style>
</head>
<body>
<div id="container">
<canvas id="canvas" oncontextmenu="event.preventDefault();" width="800" height="480">
HTML5 canvas appears to be unsupported in the current browser.<br />
Please try updating or use a different browser.
</canvas>
<div id="status">
<div id='status-progress' style='display: none;' oncontextmenu="event.preventDefault();"><div id ='status-progress-inner'></div></div>
<div id='status-indeterminate' style='display: none;' oncontextmenu="event.preventDefault();">
<div></div>
<div></div>
<div></div>
<div></div>
<div></div>
<div></div>
<div></div>
<div></div>
</div>
<div id="status-notice" class="godot" style='display: none;'></div>
</div>
</div>
<div id="output-panel" class="godot">
<div id="output-header">
Benchmark (all times in msec):
</div>
<div id="output-container"><div id="output-scroll"></div></div>
</div>
<div style='margin: 0 auto; max-width: 700px; text-align: left; color:#ddd;'>
<p><em>Interactive</em> is defined as the end of the first frame faster than 55 frames per second.</p>
<p><em>CPU time</em> excludes dead time caused by <code>requestAnimationFrame</code>.</p>
</div>
<script type="text/javascript" src="godot.js"></script>
<script type="text/javascript">//<![CDATA[
var game = new Engine;
(function() {
const BASENAME = 'godot';
const INDETERMINATE_STATUS_STEP_MS = 100;
var container = document.getElementById('container');
var canvas = document.getElementById('canvas');
var statusProgress = document.getElementById('status-progress');
var statusProgressInner = document.getElementById('status-progress-inner');
var statusIndeterminate = document.getElementById('status-indeterminate');
var statusNotice = document.getElementById('status-notice');
var initializing = true;
var statusMode = 'hidden';
var indeterminiateStatusAnimationId = 0;
var results = [];
setStatusMode('indeterminate');
game.setCanvas(canvas);
function addResult(name, value) {
results.push({
name: name,
time: value
});
}
function setStatusMode(mode) {
if (statusMode === mode || !initializing)
return;
[statusProgress, statusIndeterminate, statusNotice].forEach(elem => {
elem.style.display = 'none';
});
if (indeterminiateStatusAnimationId !== 0) {
cancelAnimationFrame(indeterminiateStatusAnimationId);
indeterminiateStatusAnimationId = 0;
}
switch (mode) {
case 'progress':
statusProgress.style.display = 'block';
break;
case 'indeterminate':
statusIndeterminate.style.display = 'block';
indeterminiateStatusAnimationId = requestAnimationFrame(animateStatusIndeterminate);
break;
case 'notice':
statusNotice.style.display = 'block';
break;
case 'hidden':
break;
default:
throw new Error("Invalid status mode");
}
statusMode = mode;
}
function animateStatusIndeterminate(ms) {
var i = Math.floor(ms / INDETERMINATE_STATUS_STEP_MS % 8);
if (statusIndeterminate.children[i].style.borderTopColor == '') {
Array.prototype.slice.call(statusIndeterminate.children).forEach(child => {
child.style.borderTopColor = '';
});
statusIndeterminate.children[i].style.borderTopColor = '#dfdfdf';
}
requestAnimationFrame(animateStatusIndeterminate);
}
function setStatusNotice(text) {
while (statusNotice.lastChild) {
statusNotice.removeChild(statusNotice.lastChild);
}
var lines = text.split('\n');
lines.forEach((line, index) => {
statusNotice.appendChild(document.createTextNode(line));
statusNotice.appendChild(document.createElement('br'));
});
};
game.setProgressFunc((current, total) => {
if (total > 0) {
statusProgressInner.style.width = current/total * 100 + '%';
setStatusMode('progress');
if (current === total) {
// wait for progress bar animation
setTimeout(() => {
setStatusMode('indeterminate');
}, 500);
}
} else {
setStatusMode('indeterminate');
}
});
Benchmark.print = function() {
var result = idPrefix => this[idPrefix + '-Finish'] - this[idPrefix + '-Start'];
this.timeWaitingForAssets = Math.max(0, this['[godot.pck]-load-Finish'] - this['game-instantiated']);
this.emscriptenRuntimeInstantiation = Benchmark['game-instantiated'] - Benchmark['engine-loaded'] - result('wasm-instantiate');
var cpuTime = this['game-instantiated'] - this['engine-loaded'];
cpuTime += this['main-loop-ready'] - this['game-loaded'];
this.loops.forEach(duration => {
cpuTime += duration.cpuTime;
});
addResult('wasm-instantiate', result('wasm-instantiate'));
addResult('engine-instantiate', ((this['main-loop-ready'] - this['game-loaded']) + this.loops[0].cpuTime));
addResult('first-frame', (this['first-frame'] - this['engine-loaded'] - this.timeWaitingForAssets));
addResult('first-interactive', (this['game-interactive'] - this['engine-loaded'] - this.timeWaitingForAssets));
addResult('cpuTime', cpuTime);
text = "WebAssembly download: " + result('[godot.wasm]-load')
+ "\nWebAssembly instantiation: " + result('wasm-instantiate')
+ "\nEngine initialization: " + ((this['main-loop-ready'] - this['game-loaded']) + this.loops[0].cpuTime)
+ "\n"
+ "\nFrom download finish to first frame: " + (this['first-frame'] - this['engine-loaded'] - this.timeWaitingForAssets)
+ "\nFrom download finish to interactive: " + (this['game-interactive'] - this['engine-loaded'] - this.timeWaitingForAssets)
+ "\nCPU time to interactive: " + cpuTime
;
var outputScroll = document.getElementById("output-scroll");
var msg = document.createElement("div");
msg.textContent = text;
var scrollToBottom = outputScroll.scrollHeight - (outputScroll.clientHeight + outputScroll.scrollTop) < 10;
outputScroll.appendChild(msg);
if (scrollToBottom) {
outputScroll.scrollTop = outputScroll.scrollHeight;
}
for (let r of results) {
console.log(r.name, ':', r.time);
}
console.log('Done!');
console.log(results);
_data = ['raptor-benchmark', 'wasm-godot', results];
window.postMessage(_data, '*');
};
game.start(BASENAME + '.pck').then(() => {
setStatusMode('hidden');
initializing = false;
}, err => {
setStatusNotice(err.message || "Error during start-up");
setStatusMode('notice');
initializing = false;
});
})();
//]]></script>
</body>
<!-- Mirrored from godot.eska.me/pub/wasm-benchmark/ by HTTrack Website Copier/3.x [XR&CO'2014], Tue, 16 Oct 2018 12:26:59 GMT -->
</html>