From 9e788488b31021fc2234486dcdc2c12f5cace780 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 22 Nov 2016 17:50:51 +0100 Subject: [PATCH 1/5] A few import cleanups; Reorders imports this way: - system namespace imports - system member imports - imports from local files --- slave/benchmarks.py | 6 +++--- slave/benchmarks_remote.py | 11 ++++++----- slave/download.py | 25 +++++++++++++------------ slave/execute.py | 15 ++++++++------- slave/executors.py | 10 +++++----- slave/server.py | 22 +++++++++++----------- slave/submitter.py | 2 +- 7 files changed, 47 insertions(+), 44 deletions(-) diff --git a/slave/benchmarks.py b/slave/benchmarks.py index ddea7c7..a670fd6 100644 --- a/slave/benchmarks.py +++ b/slave/benchmarks.py @@ -3,12 +3,12 @@ def getBenchmark(benchmark): section, name = benchmark.split(".") if section == "local": import benchmarks_local - return benchmarks_local.getBenchmark(name); + return benchmarks_local.getBenchmark(name) elif section == "remote": import benchmarks_remote - return benchmarks_remote.getBenchmark(name); + return benchmarks_remote.getBenchmark(name) elif section == "shell": import benchmarks_shell - return benchmarks_shell.getBenchmark(name); + return benchmarks_shell.getBenchmark(name) else: raise Exception("Unknown benchmark type") diff --git a/slave/benchmarks_remote.py b/slave/benchmarks_remote.py index 12e9642..aed1470 100644 --- a/slave/benchmarks_remote.py +++ b/slave/benchmarks_remote.py @@ -1,11 +1,12 @@ -import subprocess -import socket -import os -import time import json +import os +import socket +import subprocess import sys +import time sys.path.insert(1, '../driver') + import utils class Benchmark: @@ -157,7 +158,7 @@ def getBenchmark(name): raise Exception("Unknown benchmark") # Test if server is running and start server if needed. -s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = s.connect_ex(("localhost", 8000)) s.close() if result > 0: diff --git a/slave/download.py b/slave/download.py index 1b5eef6..95c2ca7 100644 --- a/slave/download.py +++ b/slave/download.py @@ -1,18 +1,19 @@ import json -import urllib2 -import urllib -import re import os +import platform +import re import shutil import socket -import utils -import platform -import url_creator - import tarfile +import urllib +import urllib2 import zipfile + socket.setdefaulttimeout(120) +import url_creator +import utils + DEBUG = True class DownloadTools(object): @@ -58,13 +59,13 @@ class Downloader(object): def valid(self): return self.getfilename() != None - def setOutputFolder(self, folder): + def set_output_folder(self, folder): if not folder.endswith("/"): folder += "/" self.folder = folder def download(self): - self.createOutputFolder() + self.create_output_folder() filename = self.getfilename() assert filename @@ -76,7 +77,7 @@ class Downloader(object): json.dump(info, fp) fp.close() - def createOutputFolder(self): + def create_output_folder(self): if os.path.isdir(self.folder): shutil.rmtree(self.folder) os.makedirs(self.folder) @@ -295,7 +296,7 @@ if __name__ == "__main__": elif options.repo: downloader = DownloadTools.forRepo(options.repo, options.cset) else: - raise Exception("You'll need to specify atleast an url or repo") + raise Exception("You'll need to specify at least an url or repo") - downloader.setOutputFolder(options.output) + downloader.set_output_folder(options.output) downloader.download() diff --git a/slave/execute.py b/slave/execute.py index a6a2cd8..57df651 100644 --- a/slave/execute.py +++ b/slave/execute.py @@ -1,15 +1,16 @@ -import benchmarks -import configs -import executors -import engineInfo -import submitter import json +import sys import traceback -import sys +from optparse import OptionParser + +import benchmarks +import configs +import engineInfo +import executors +import submitter import utils -from optparse import OptionParser parser = OptionParser(usage="usage: %prog url [options]") parser.add_option("-b", "--benchmark", action="append", dest="benchmarks", diff --git a/slave/executors.py b/slave/executors.py index 1a15156..d2d1d15 100644 --- a/slave/executors.py +++ b/slave/executors.py @@ -1,10 +1,10 @@ -import runners import time import os import sys import json import utils +import runners class ShellExecutor(object): def __init__(self, engineInfo): @@ -235,13 +235,13 @@ class ServoExecutor(BrowserExecutor): def getExecutor(engineInfo): if engineInfo["shell"]: return ShellExecutor(engineInfo) - if engineInfo["engine_type"] == "firefox" and not engineInfo["shell"]: + if engineInfo["engine_type"] == "firefox": return FirefoxExecutor(engineInfo) - if engineInfo["engine_type"] == "chrome" and not engineInfo["shell"]: + if engineInfo["engine_type"] == "chrome": return ChromeExecutor(engineInfo) - if engineInfo["engine_type"] == "webkit" and not engineInfo["shell"]: + if engineInfo["engine_type"] == "webkit": return WebKitExecutor(engineInfo) - if engineInfo["engine_type"] == "edge" and not engineInfo["shell"]: + if engineInfo["engine_type"] == "edge": return EdgeExecutor(engineInfo) if engineInfo["engine_type"] == "servo": return ServoExecutor(engineInfo) diff --git a/slave/server.py b/slave/server.py index 3e69dac..2ecbdb8 100644 --- a/slave/server.py +++ b/slave/server.py @@ -1,16 +1,18 @@ -import sys import BaseHTTPServer -from SimpleHTTPServer import SimpleHTTPRequestHandler -import urlparse -import os -import json -import urllib -import httplib -from SocketServer import ThreadingMixIn import hashlib +import httplib +import json +import os import pickle -import utils import signal +import sys +import urllib +import urlparse + +from SimpleHTTPServer import SimpleHTTPRequestHandler +from SocketServer import ThreadingMixIn + +import utils class FakeHandler(SimpleHTTPRequestHandler): @@ -322,8 +324,6 @@ Protocol = "HTTP/1.0" Port = 8000 ServerAddress = ('', Port) -import os -import utils path = os.path.abspath(os.path.join(os.path.dirname(__file__),"..")) with utils.FolderChanger(path): HandlerClass.protocol_version = Protocol diff --git a/slave/submitter.py b/slave/submitter.py index e5d358e..161fdf1 100644 --- a/slave/submitter.py +++ b/slave/submitter.py @@ -32,7 +32,7 @@ class Submitter(object): def mode(self, engine_type, config): name = engine_type + "," + config if name in self.rules: - return self.rules[engine_type + "," + config] + return self.rules[name] else: return name From 02273e2dc7fd48ca5e29c19322e39c894a49efc6 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 22 Nov 2016 19:19:59 +0100 Subject: [PATCH 2/5] Refactor finding benchmarks by name; --- slave/benchmarks.py | 1 + slave/benchmarks_remote.py | 92 +++++++++++++++++++++++++++----------- 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/slave/benchmarks.py b/slave/benchmarks.py index a670fd6..06faf97 100644 --- a/slave/benchmarks.py +++ b/slave/benchmarks.py @@ -12,3 +12,4 @@ def getBenchmark(benchmark): return benchmarks_shell.getBenchmark(name) else: raise Exception("Unknown benchmark type") + diff --git a/slave/benchmarks_remote.py b/slave/benchmarks_remote.py index aed1470..4714c3d 100644 --- a/slave/benchmarks_remote.py +++ b/slave/benchmarks_remote.py @@ -11,18 +11,22 @@ import utils class Benchmark: """ timeout is in minutes """ - def __init__(self, suite, version, timeout=2): - self.suite = suite - self.version = suite+" "+version + def __init__(self, version, timeout=2, suite=None): + self.suite = suite if suite is not None else self.name() + self.version = self.suite + " " + version self.url = 'http://' + self.suite + ".localhost:8000" self.timeout = timeout def processResults(self, results): return results + @staticmethod + def name(self): + raise Exception("NYI") + class Octane(Benchmark): def __init__(self): - Benchmark.__init__(self, "octane", "2.0.1") + Benchmark.__init__(self, "2.0.1") def processResults(self, results): ret = [] @@ -33,9 +37,13 @@ class Octane(Benchmark): ret.append({'name': key, 'time': results[key]}) return ret + @staticmethod + def name(): + return "octane" + class Dromaeo(Benchmark): def __init__(self): - Benchmark.__init__(self, "dromaeo", "1.0", 17) + Benchmark.__init__(self, "1.0", 17) self.url = 'http://' + self.suite + ".localhost:8000/?recommended" def processResults(self, results): @@ -47,9 +55,13 @@ class Dromaeo(Benchmark): ret.append({'name': key, 'time': results[key]}) return ret + @staticmethod + def name(): + return "dromaeo" + class Massive(Benchmark): def __init__(self): - Benchmark.__init__(self, "massive", "1.2", 9) + Benchmark.__init__(self, "1.2", 9) def processResults(self, results): ret = [] @@ -62,9 +74,13 @@ class Massive(Benchmark): ret.append({'name': item["benchmark"], 'time': item["result"]}) return ret + @staticmethod + def name(): + return "massive" + class JetStream(Benchmark): def __init__(self): - Benchmark.__init__(self, "jetstream", "1.0", 5) + Benchmark.__init__(self, "1.0", 5) def processResults(self, results): ret = [] @@ -75,13 +91,21 @@ class JetStream(Benchmark): ret.append({'name': item, 'time': results[item]["statistics"]["mean"]}) return ret + @staticmethod + def name(): + return "jetstream" + class Speedometer(Benchmark): def __init__(self): - Benchmark.__init__(self, "speedometer", "1.0", 4) + Benchmark.__init__(self, "1.0", 4) + + @staticmethod + def name(): + return "speedometer" class Kraken(Benchmark): def __init__(self): - Benchmark.__init__(self, "kraken", "1.1") + Benchmark.__init__(self, "1.1") def processResults(self, results): ret = [] @@ -101,9 +125,13 @@ class Kraken(Benchmark): ret.append({'name': "__total__", 'time': total }) return ret + @staticmethod + def name(): + return "kraken" + class SunSpider(Benchmark): def __init__(self): - Benchmark.__init__(self, "ss", "1.0.2", 1) + Benchmark.__init__(self, "1.0.2", 1, suite="ss") self.url = "http://sunspider.localhost:8000/" def processResults(self, results): @@ -124,9 +152,13 @@ class SunSpider(Benchmark): ret.append({'name': "__total__", 'time': total }) return ret + @staticmethod + def name(): + return "sunspider" + class Browsermark(Benchmark): def __init__(self): - Benchmark.__init__(self, "browsermark", "2.1", 5) + Benchmark.__init__(self, "2.1", 5) self.url = "http://browsermark.local:8082/" def processResults(self, results): @@ -138,23 +170,29 @@ class Browsermark(Benchmark): ret.append({'name': item[0], 'time': item[1]}) return ret + @staticmethod + def name(): + return "browsermark" + +KnownBenchmarks = [ + Octane, + Dromaeo, + Massive, + JetStream, + Speedometer, + Kraken, + SunSpider, + Browsermark +] + +# TODO use this when showing execute.py's help. +def get_all_known_benchmark_names(): + return [b.name() for b in KnownBenchmarks] + def getBenchmark(name): - if name == "octane": - return Octane() - if name == "dromaeo": - return Dromaeo() - if name == "massive": - return Massive() - if name == "jetstream": - return JetStream() - if name == "speedometer": - return Speedometer() - if name == "kraken": - return Kraken() - if name == "sunspider": - return SunSpider() - if name == "browsermark": - return Browsermark() + for b in KnownBenchmarks: + if name == b.name(): + return b() raise Exception("Unknown benchmark") # Test if server is running and start server if needed. From a08673283bcdc5d18f598905257e43b15c1fcb7f Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 22 Nov 2016 19:33:23 +0100 Subject: [PATCH 3/5] Add a remote wasm benchmark; --- slave/benchmarks_remote.py | 12 +++++++++++- slave/configs.py | 1 + slave/execute.py | 2 +- slave/executors.py | 2 +- slave/server.py | 1 - 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/slave/benchmarks_remote.py b/slave/benchmarks_remote.py index 4714c3d..de50741 100644 --- a/slave/benchmarks_remote.py +++ b/slave/benchmarks_remote.py @@ -174,6 +174,15 @@ class Browsermark(Benchmark): def name(): return "browsermark" +class WasmMisc(Benchmark): + def __init__(self): + Benchmark.__init__(self, "0.1") + self.url = "http://wasm.local" + + @staticmethod + def name(): + return "wasm" + KnownBenchmarks = [ Octane, Dromaeo, @@ -182,7 +191,8 @@ KnownBenchmarks = [ Speedometer, Kraken, SunSpider, - Browsermark + Browsermark, + WasmMisc, ] # TODO use this when showing execute.py's help. diff --git a/slave/configs.py b/slave/configs.py index 7b1af8e..d3545f6 100644 --- a/slave/configs.py +++ b/slave/configs.py @@ -19,6 +19,7 @@ class Default(object): if engine == "firefox": self.env_["JSGC_DISABLE_POISONING"] = "1" self.profile_ += "user_pref(\"dom.max_script_run_time\", 0);\n" + self.profile_ += "user_pref(\"javascript.options.wasm\", true);\n" elif engine == "chrome": pass elif engine == "webkit": diff --git a/slave/execute.py b/slave/execute.py index 57df651..62d92fa 100644 --- a/slave/execute.py +++ b/slave/execute.py @@ -14,7 +14,7 @@ import utils parser = OptionParser(usage="usage: %prog url [options]") parser.add_option("-b", "--benchmark", action="append", dest="benchmarks", - help="Benchmark to run (the local ones are deprecated): remote.octane, remote.dromaeo, remote.massive, remote.jetstream, remote.speedometer, remote.kraken, remote.sunspider, remote.browsermark, shell.octane, shell.sunspider, shell.kraken, shell.assorted, shell.asmjsapps, shell.asmjsmicro, shell.shumway, shell.dart, local.octane, local.sunspider, local.kraken, local.weglsamples, local.assorteddom") + help="Benchmark to run (the local ones are deprecated): remote.octane, remote.dromaeo, remote.massive, remote.jetstream, remote.speedometer, remote.kraken, remote.sunspider, remote.browsermark, remote.wasm, shell.octane, shell.sunspider, shell.kraken, shell.assorted, shell.asmjsapps, shell.asmjsmicro, shell.shumway, shell.dart, local.octane, local.sunspider, local.kraken, local.weglsamples, local.assorteddom") parser.add_option("-s", "--submitter", dest="submitter", type="string", default="print", help="Submitter class ('remote' or 'print')") diff --git a/slave/executors.py b/slave/executors.py index d2d1d15..24be6d1 100644 --- a/slave/executors.py +++ b/slave/executors.py @@ -130,7 +130,7 @@ class FirefoxExecutor(BrowserExecutor): self.resetResults() # start browser - process = runner.start(binary, args + ["--profile", runner.getdir("profile")], env) + process = runner.start(binary, args + ["--no-remote", "--profile", runner.getdir("profile")], env) # wait for results self.waitForResults(benchmark.timeout) diff --git a/slave/server.py b/slave/server.py index 2ecbdb8..bc2ea62 100644 --- a/slave/server.py +++ b/slave/server.py @@ -284,7 +284,6 @@ class FakeHandler(SimpleHTTPRequestHandler): if path == "/benchmarks/misc-desktop/hosted/assorted/driver.html": return data.replace('location = "results.html?" + encodeURI(outputString);', 'location.href = "http://localhost:8000/submit?results=" + encodeURI(outputString);'); - if host == "localhost": if path == "/benchmarks/webaudio/webaudio-bench.js": return data.replace('xhr.open("POST", "/results", true);', 'xhr.open("POST", "/submit", true);'); From 1ca9f82c89e5cc00183d152621f3e62a38667aeb Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Wed, 23 Nov 2016 16:58:35 +0100 Subject: [PATCH 4/5] Create a new config for wasm and wasm-baseline; --- .gitignore | 2 ++ slave/benchmarks_remote.py | 2 +- slave/configs.py | 23 +++++++++++++++++++---- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 92ce860..52eba09 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ benchmarks/asmjs-apps/zlib/minigzipsh driver/awfy.config slave/awfy.config +slave/results +slave/profile **/*-results/ output/ diff --git a/slave/benchmarks_remote.py b/slave/benchmarks_remote.py index de50741..e25c1f7 100644 --- a/slave/benchmarks_remote.py +++ b/slave/benchmarks_remote.py @@ -176,7 +176,7 @@ class Browsermark(Benchmark): class WasmMisc(Benchmark): def __init__(self): - Benchmark.__init__(self, "0.1") + Benchmark.__init__(self, "0.2") self.url = "http://wasm.local" @staticmethod diff --git a/slave/configs.py b/slave/configs.py index d3545f6..e0b9bd1 100644 --- a/slave/configs.py +++ b/slave/configs.py @@ -19,7 +19,6 @@ class Default(object): if engine == "firefox": self.env_["JSGC_DISABLE_POISONING"] = "1" self.profile_ += "user_pref(\"dom.max_script_run_time\", 0);\n" - self.profile_ += "user_pref(\"javascript.options.wasm\", true);\n" elif engine == "chrome": pass elif engine == "webkit": @@ -35,7 +34,7 @@ class Default(object): def omit(self): return self.omit_ - + def args(self): return self.args_ @@ -46,6 +45,18 @@ class Default(object): # Currently only for firefox profile js file. return self.profile_ +class Wasm(Default): + def __init__(self, engine, shell): + super(Wasm, self).__init__(engine, shell) + if engine == "firefox": + self.profile_ += "user_pref(\"javascript.options.wasm\", true);\n" + +class WasmBaseline(Wasm): + def __init__(self, engine, shell): + super(WasmBaseline, self).__init__(engine, shell) + if engine == "firefox": + self.profile_ += "user_pref(\"javascript.options.wasm_baselinejit\", true);\n" + class UnboxedObjects(Default): def __init__(self, engine, shell): super(UnboxedObjects, self).__init__(engine, shell) @@ -71,7 +82,7 @@ class TurboFan(Default): self.args_.append("--turbo"); else: self.omit_ = True - + class TurboIgnition(Default): def __init__(self, engine, shell): super(TurboIgnition, self).__init__(engine, shell) @@ -80,7 +91,7 @@ class TurboIgnition(Default): self.args_.append("--ignition-staging"); else: self.omit_ = True - + class Ignition(Default): def __init__(self, engine, shell): super(Ignition, self).__init__(engine, shell) @@ -148,6 +159,10 @@ class E10S(Default): def getConfig(name, info): if name == "default": return Default(info["engine_type"], info["shell"]) + if name == "wasm": + return Wasm(info["engine_type"], info["shell"]) + if name == "wasm-baseline": + return WasmBaseline(info["engine_type"], info["shell"]) if name == "unboxedobjects": return UnboxedObjects(info["engine_type"], info["shell"]) if name == "testbedregalloc": From f3e3d1d22995a4f8c0d218554ec47356ef42beb6 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Wed, 23 Nov 2016 18:43:36 +0100 Subject: [PATCH 5/5] chromium: add support for linux x64, wasm and use latest builds; --- slave/configs.py | 2 ++ slave/download.py | 8 ++++---- slave/url_creator.py | 9 ++++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/slave/configs.py b/slave/configs.py index e0b9bd1..d8e49f0 100644 --- a/slave/configs.py +++ b/slave/configs.py @@ -50,6 +50,8 @@ class Wasm(Default): super(Wasm, self).__init__(engine, shell) if engine == "firefox": self.profile_ += "user_pref(\"javascript.options.wasm\", true);\n" + elif engine == "chrome": + self.args_ += ['--js-flags=--expose_wasm'] class WasmBaseline(Wasm): def __init__(self, engine, shell): diff --git a/slave/download.py b/slave/download.py index 95c2ca7..9310944 100644 --- a/slave/download.py +++ b/slave/download.py @@ -219,11 +219,11 @@ class GoogleAPISDownloader(Downloader): def getfilename(self): platform = self.url.split("/")[-3] - if platform == "Linux": + if platform.startswith("Linux"): return "chrome-linux.zip" elif platform == "Mac": return "chrome-mac.zip" - elif platform.startswith("Win"): # Yeah chrome puts win64 in win32 folder + elif platform.startswith("Win"): # Chrome puts win64 in win32 folder. return "chrome-win32.zip" elif platform == "Android": return "chrome-android.zip" @@ -286,9 +286,9 @@ if __name__ == "__main__": parser.add_option("-u", "--url", dest="url", help="Specify a specific url to download.", default=None) parser.add_option("--repo", dest="repo", - help="Specify a repo to download. Currently only mozilla-inbound supported.", default=None) + help="Specify a repo to download. Currently supports: mozilla-inbound (cset?) | chrome", default=None) parser.add_option("-r", dest="cset", - help="Specify the revision to download. Default to 'latest'", default='latest') + help="Specify the revision to download. Defaults to 'latest'", default='latest') (options, args) = parser.parse_args() if options.url: diff --git a/slave/url_creator.py b/slave/url_creator.py index 0f52788..3c41d69 100644 --- a/slave/url_creator.py +++ b/slave/url_creator.py @@ -19,19 +19,22 @@ class ChromeUrlCreator(UrlCreator): def _url_base(self): platform = self._platform() - return "http://commondatastorage.googleapis.com/chromium-browser-continuous/"+platform+"/" + return "http://commondatastorage.googleapis.com/chromium-browser-snapshots/"+platform+"/" def _platform(self): arch, _ = platform.architecture() arch = arch[0:2] if platform.system() == "Linux": - return "Linux" + if arch == '64': + return "Linux_x64" + if arch == '32': + return "Linux" if platform.system() == "Darwin": return "Mac" if platform.system() == "Windows" or platform.system().startswith("CYGWIN"): if arch == '32': return "Win" - elif arch == '64': + if arch == '64': return "Win_x64" raise Exception("Unknown platform: " + platform.system())