From 4a169cb6d0ec820a0cc97e81fe7976996830d792 Mon Sep 17 00:00:00 2001 From: Ted Mielczarek Date: Fri, 30 Aug 2013 13:41:53 -0400 Subject: [PATCH] Significantly refactor harness. Start a single httpd to run all HTML tests, loading them from a JSON manifest. Added webharness/ to contain the in-browser harness code. The web harness loads the JSON manifest, connects to a signalling server using socket.io, and waits for the other client to be connected to the signalling server before running tests. Imported q.js as a Promise implementation to keep me sane while writing all this asynchronous logic. --- sample.html | 14 --- steeplechase/runsteeplechase.py | 122 +++++++++++---------- webharness/harness.js | 183 ++++++++++++++++++++++++++++++++ webharness/index.html | 10 ++ webharness/manifest.json | 3 + webharness/q.min.js | 1 + webharness/test.js | 22 ++++ webharness/tests/sample.html | 25 +++++ 8 files changed, 310 insertions(+), 70 deletions(-) delete mode 100644 sample.html create mode 100644 webharness/harness.js create mode 100644 webharness/index.html create mode 100644 webharness/manifest.json create mode 100644 webharness/q.min.js create mode 100644 webharness/test.js create mode 100644 webharness/tests/sample.html diff --git a/sample.html b/sample.html deleted file mode 100644 index abd6b44..0000000 --- a/sample.html +++ /dev/null @@ -1,14 +0,0 @@ - - - -sample test - - - -

Hello world

- - diff --git a/steeplechase/runsteeplechase.py b/steeplechase/runsteeplechase.py index 0713a17..c48bc12 100644 --- a/steeplechase/runsteeplechase.py +++ b/steeplechase/runsteeplechase.py @@ -16,6 +16,7 @@ import moznetwork import os import sys import threading +import uuid class Options(OptionParser): def __init__(self, **kwargs): @@ -39,6 +40,13 @@ class Options(OptionParser): self.add_option("--host2", action="store", type="string", dest="host2", help="first remote host to run tests on") + self.add_option("--signalling-server", + action="store", type="string", dest="signalling_server", + help="signalling server URL to use for tests"); + self.add_option("--noSetup", + action="store_false", dest="setup", + default="True", + help="do not copy files to device"); self.set_usage(usage) @@ -51,6 +59,7 @@ class RunThread(threading.Thread): dm, cmd, env, cond, results = self.args try: output = dm.shellCheckOutput(cmd, env=env) + print "%s\n%s\n%s" % ("=" * 20, output, "=" * 20) finally: #TODO: actual result cond.acquire() @@ -59,17 +68,13 @@ class RunThread(threading.Thread): cond.release() del self.args -class HTMLTest(object): - def __init__(self, test_file, remote_info, options): - self.test_file = os.path.abspath(test_file) +class HTMLTests(object): + def __init__(self, httpd, remote_info, options): self.remote_info = remote_info self.options = options - #XXX: start httpd in main, not here - self.httpd = MozHttpd(host=moznetwork.get_ip(), - docroot=os.path.dirname(self.test_file)) + self.httpd = httpd def run(self): - self.httpd.start(block=False) locations = ServerLocations() locations.add_host(host=self.httpd.host, port=self.httpd.port, @@ -84,16 +89,22 @@ class HTMLTest(object): prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) + prefs["steeplechase.signalling_server"] = self.options.signalling_server + prefs["steeplechase.signalling_room"] = str(uuid.uuid4()) specialpowers_path = self.options.specialpowers - with mozfile.TemporaryDirectory() as profile_path: - # Create and push profile - print "Writing profile..." - profile = FirefoxProfile(profile=profile_path, - preferences=prefs, - addons=[specialpowers_path], - locations=locations) - for info in self.remote_info: + threads = [] + results = [] + cond = threading.Condition() + for info in self.remote_info: + with mozfile.TemporaryDirectory() as profile_path: + # Create and push profile + print "Writing profile..." + prefs["steeplechase.is_initiator"] = info['is_initiator'] + profile = FirefoxProfile(profile=profile_path, + preferences=prefs, + addons=[specialpowers_path], + locations=locations) print "Pushing profile..." remote_profile_path = os.path.join(info['test_root'], "profile") info['dm'].mkDir(remote_profile_path) @@ -103,36 +114,34 @@ class HTMLTest(object): env = {} env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["XPCOM_DEBUG_BREAK"] = "warn" - env["DISPLAY"] = ":1" + env["DISPLAY"] = ":0" - threads = [] - results = [] - cond = threading.Condition() - for info in self.remote_info: - cmd = [info['remote_app_path'], "-no-remote", - "-profile", info['remote_profile_path'], - self.httpd.get_url("/" + os.path.basename(self.test_file))] - print "cmd: %s" % (cmd, ) - t = RunThread(args=(info['dm'], cmd, env, cond, results)) - t.start() - threads.append(t) - print "Waiting for results..." - while threads: - cond.acquire() - while not results: - cond.wait() - res = results.pop(0) - cond.release() - print "Got result: %d" % res[1] - threads.remove(res[0]) - print "Done!" - self.httpd.stop() + cmd = [info['remote_app_path'], "-no-remote", + "-profile", info['remote_profile_path'], + self.httpd.get_url("/index.html")] + print "cmd: %s" % (cmd, ) + t = RunThread(args=(info['dm'], cmd, env, cond, results)) + threads.append(t) + + for t in threads: + t.start() + + print "Waiting for results..." + while threads: + cond.acquire() + while not results: + cond.wait() + res = results.pop(0) + cond.release() + print "Got result: %d" % res[1] + threads.remove(res[0]) + print "Done!" return True def main(args): parser = Options() options, args = parser.parse_args() - if not args or not options.binary or not options.specialpowers or not options.host1 or not options.host2: + if not options.binary or not options.specialpowers or not options.host1 or not options.host2 or not options.signalling_server: parser.print_usage() return 2 @@ -140,7 +149,7 @@ def main(args): parser.error("Binary %s does not exist" % options.binary) return 2 if not os.path.isdir(options.specialpowers): - parser.error("SpecialPowers direcotry %s does not exist" % options.specialpowers) + parser.error("SpecialPowers directory %s does not exist" % options.specialpowers) return 2 if options.prefs and not os.path.isfile(options.prefs): parser.error("Prefs file %s does not exist" % options.prefs) @@ -150,32 +159,33 @@ def main(args): log.setLevel(mozlog.DEBUG) dm1 = DeviceManagerSUT(options.host1) dm2 = DeviceManagerSUT(options.host2) - remote_info = [{'dm': dm1}, {'dm': dm2}] + remote_info = [{'dm': dm1, 'is_initiator': True}, + {'dm': dm2, 'is_initiator': False}] # first, push app for info in remote_info: dm = info['dm'] test_root = dm.getDeviceRoot() + "/steeplechase" - if dm.dirExists(test_root): - dm.removeDir(test_root) - dm.mkDir(test_root) + if options.setup: + if dm.dirExists(test_root): + dm.removeDir(test_root) + dm.mkDir(test_root) info['test_root'] = test_root app_path = options.binary remote_app_dir = test_root + "/app" - dm.mkDir(remote_app_dir) - dm.pushDir(os.path.dirname(app_path), remote_app_dir) + if options.setup: + dm.mkDir(remote_app_dir) + dm.pushDir(os.path.dirname(app_path), remote_app_dir) info['remote_app_path'] = remote_app_dir + "/" + os.path.basename(app_path) result = True - for arg in args: - test = None - if arg.endswith(".html"): - test = HTMLTest(arg, remote_info, options) - else: - #TODO: support C++ tests - log.error("Unknown test type: %s", arg) - continue - result = result and test.run() - + #TODO: only start httpd if we have HTML tests + httpd = MozHttpd(host=moznetwork.get_ip(), + docroot=os.path.join(os.path.dirname(__file__), "..", "webharness")) + httpd.start(block=False) + #TODO: support test manifests + test = HTMLTests(httpd, remote_info, options) + result = test.run() + httpd.stop() return 0 if result else 1 if __name__ == '__main__': diff --git a/webharness/harness.js b/webharness/harness.js new file mode 100644 index 0000000..1393b0a --- /dev/null +++ b/webharness/harness.js @@ -0,0 +1,183 @@ +var tests = []; +var current_test = -1; +var current_window = null; +var socket; +var socket_messages = []; +var socket_message_deferreds = []; +var is_initiator = SpecialPowers.getBoolPref("steeplechase.is_initiator"); + +function fetch_manifest() { + var deferred = Q.defer(); + // Load test manifest + var req = new XMLHttpRequest(); + req.open("GET", "/manifest.json", true); + req.responseType = "json"; + req.overrideMimeType("application/json"); + req.onload = function() { + if (req.readyState == 4) { + if (req.status == 200) { + deferred.resolve(req.response); + } else { + deferred.reject(new Error("Error fetching test manifest")); + } + } + }; + req.onerror = function() { + deferred.reject(new Error("Error fetching test manifest")); + }; + req.send(null); + return deferred.promise; +} + +function load_script(script) { + var deferred = Q.defer(); + var s = document.createElement("script"); + s.src = script; + s.onload = function() { + deferred.resolve(s); + }; + s.onerror = function() { + deferred.reject(new Error("Error loading socket.io.js")); + }; + document.head.appendChild(s); + return deferred.promise; +} + +/* + * Receive a single message from |socket|. If + * there is a deferred (from wait_for_message) + * waiting on it, resolve that deferred. Otherwise + * queue the message for a future waiter. + */ +function socket_message(data) { + var message = JSON.parse(data); + if (socket_message_deferreds.length > 0) { + var d = socket_message_deferreds.shift(); + d.resolve(message); + } else { + socket_messages.push(message); + } +} + +/* + * Return a promise for the next available message + * to come in on |socket|. If there is a queued + * message, resolves the promise immediately, otherwise + * waits for socket_message to receive one. + */ +function wait_for_message() { + var deferred = Q.defer(); + if (socket_messages.length > 0) { + deferred.resolve(socket_messages.shift()); + } else { + socket_message_deferreds.push(deferred); + } + return deferred.promise; +} + +/* + * Send an object as a message on |socket|. + */ +function send_message(data) { + socket.send(JSON.stringify(data)); +} + +function connect_socket() { + var server = SpecialPowers.getCharPref("steeplechase.signalling_server"); + var room = SpecialPowers.getCharPref("steeplechase.signalling_room"); + var script = server + "socket.io/socket.io.js"; + return load_script(script).then(function() { + var deferred = Q.defer(); + socket = io.connect(server + "?room=" + room); + socket.on("connect", function() { + socket.on("message", socket_message); + deferred.resolve(socket); + }); + socket.on("error", function() { + deferred.reject(new Error("socket.io error")); + }); + socket.on("connect_failed", function() { + deferred.reject(new Error("socket failed to connect")); + }); + return deferred; + }).then(function () { + var deferred = Q.defer(); + socket.once("numclients", function(data) { + if (data.clients == 2) { + // Other side is already there. + deferred.resolve(socket); + } else if (data.clients > 2) { + deferred.reject(new Error("Too many clients connected")); + } else { + // Just us, wait for the other side. + socket.once("client_joined", function() { + deferred.resolve(socket); + }); + } + }); + return deferred.promise; + }); +} + +Q.all([fetch_manifest(), + connect_socket()]).then(run_tests, + harness_error); + +function run_tests(results) { + var manifest = results[0]; + // Manifest looks like: + // {'tests': [{'path': '...'}, ...]} + tests = manifest.tests; + run_next_test(); +} + +function run_next_test() { + ++current_test; + if (current_test >= tests.length) { + finish(); + return; + } + + current_window = window.open("/tests/" + tests[current_test].path); + current_window.addEventListener("load", function() { + dump("loaded\n"); + send_message({"action": "test_loaded", "test": tests[current_test].path}); + // Wait for other side to have loaded this test. + wait_for_message().then(function (m) { + if (m.action != "test_loaded") { + //XXX: should this be fatal? + harness_error(new Error("Looking for test_loaded, got: " + JSON.stringify(m))); + return; + } + if (m.test != tests[current_test].path) { + harness_error(new Error("Wrong test loaded on other side: " + m.test)); + return; + } + current_window.test(is_initiator); + }); + }); + //TODO: timeout handling +} + +function harness_error(error) { + log_result(false, error.message, "harness"); + dump(error.stack +"\n"); + finish(); +} + +// Called by tests via test.js. +function test_finished() { + run_next_test(); +} + +function finish() { + SpecialPowers.quit(); +} + +function log_result(result, message, test) { + var output = {'action': result ? "test_pass" : "test_unexpected_fail", + 'diagnostic': message, + 'time': Date.now(), + 'source_file': test || tests[current_test].path}; + dump(JSON.stringify(output) + "\n"); +} diff --git a/webharness/index.html b/webharness/index.html new file mode 100644 index 0000000..7a50268 --- /dev/null +++ b/webharness/index.html @@ -0,0 +1,10 @@ + + + +Steeplechase harness + + + + + + diff --git a/webharness/manifest.json b/webharness/manifest.json new file mode 100644 index 0000000..25c0a60 --- /dev/null +++ b/webharness/manifest.json @@ -0,0 +1,3 @@ +{"tests": [ + {"path": "sample.html"} +]} \ No newline at end of file diff --git a/webharness/q.min.js b/webharness/q.min.js new file mode 100644 index 0000000..148cf5c --- /dev/null +++ b/webharness/q.min.js @@ -0,0 +1 @@ +!function(a){if("function"==typeof bootstrap)bootstrap("promise",a);else if("object"==typeof exports)module.exports=a();else if("function"==typeof define&&define.amd)define(a);else if("undefined"!=typeof ses){if(!ses.ok())return;ses.makeQ=a}else Q=a()}(function(){"use strict";function a(a){var b=Function.call;return function(){return b.apply(a,arguments)}}function b(a){return a===Object(a)}function c(a){return"[object StopIteration]"===sb(a)||a instanceof hb}function d(a,b){if(eb&&b.stack&&"object"==typeof a&&null!==a&&a.stack&&-1===a.stack.indexOf(ub)){for(var c=[],d=b;d;d=d.source)d.stack&&c.unshift(d.stack);c.unshift(a.stack);var f=c.join("\n"+ub+"\n");a.stack=e(f)}}function e(a){for(var b=a.split("\n"),c=[],d=0;d=ib&&Ab>=d}function i(){if(eb)try{throw new Error}catch(a){var b=a.stack.split("\n"),c=b[0].indexOf("@")>0?b[1]:b[2],d=g(c);if(!d)return;return gb=d[0],d[1]}}function j(a,b,c){return function(){return"undefined"!=typeof console&&"function"==typeof console.warn&&console.warn(b+" is deprecated, use "+c+" instead.",new Error("").stack),a.apply(a,arguments)}}function k(a){return B(a)}function l(){function a(a){b=a,f.source=a,mb(c,function(b,c){kb(function(){a.promiseDispatch.apply(a,c)})},void 0),c=void 0,d=void 0}var b,c=[],d=[],e=pb(l.prototype),f=pb(n.prototype);if(f.promiseDispatch=function(a,e,f){var g=lb(arguments);c?(c.push(g),"when"===e&&f[1]&&d.push(f[1])):kb(function(){b.promiseDispatch.apply(b,g)})},f.valueOf=j(function(){if(c)return f;var a=o(b);return p(a)&&(b=a),a},"valueOf","inspect"),f.inspect=function(){return b?b.inspect():{state:"pending"}},k.longStackSupport&&eb)try{throw new Error}catch(g){f.stack=g.stack.substring(g.stack.indexOf("\n")+1)}return e.promise=f,e.resolve=function(c){b||a(B(c))},e.fulfill=function(c){b||a(A(c))},e.reject=function(c){b||a(z(c))},e.notify=function(a){b||mb(d,function(b,c){kb(function(){c(a)})},void 0)},e}function m(a){if("function"!=typeof a)throw new TypeError("resolver must be a function.");var b=l();return O(a,b.resolve,b.reject,b.notify).fail(b.reject),b.promise}function n(a,b,c){void 0===b&&(b=function(a){return z(new Error("Promise does not support operation: "+a))}),void 0===c&&(c=function(){return{state:"unknown"}});var d=pb(n.prototype);if(d.promiseDispatch=function(c,e,f){var g;try{g=a[e]?a[e].apply(d,f):b.call(d,e,f)}catch(h){g=z(h)}c&&c(g)},d.inspect=c,c){var e=c();"rejected"===e.state&&(d.exception=e.reason),d.valueOf=j(function(){var a=c();return"pending"===a.state||"rejected"===a.state?d:a.value})}return d}function o(a){if(p(a)){var b=a.inspect();if("fulfilled"===b.state)return b.value}return a}function p(a){return b(a)&&"function"==typeof a.promiseDispatch&&"function"==typeof a.inspect}function q(a){return b(a)&&"function"==typeof a.then}function r(a){return p(a)&&"pending"===a.inspect().state}function s(a){return!p(a)||"fulfilled"===a.inspect().state}function t(a){return p(a)&&"rejected"===a.inspect().state}function u(){xb||"undefined"==typeof window||window.Touch||!window.console||console.warn("[Q] Unhandled rejection reasons (should be empty):",vb),xb=!0}function v(){for(var a=0;a=d)throw new TypeError}for(;d>c;c++)c in this&&(b=a(b,this[c],c));return b}),nb=a(Array.prototype.indexOf||function(a){for(var b=0;b2?a.resolve(lb(arguments,1)):a.resolve(c)}},k.promise=m,k.makePromise=n,n.prototype.then=function(a,b,c){function e(b){try{return"function"==typeof a?a(b):b}catch(c){return z(c)}}function f(a){if("function"==typeof b){d(a,h);try{return b(a)}catch(c){return z(c)}}return z(a)}function g(a){return"function"==typeof c?c(a):a}var h=this,i=l(),j=!1;return kb(function(){h.promiseDispatch(function(a){j||(j=!0,i.resolve(e(a)))},"when",[function(a){j||(j=!0,i.resolve(f(a)))}])}),h.promiseDispatch(void 0,"when",[void 0,function(a){var b,c=!1;try{b=g(a)}catch(d){if(c=!0,!k.onerror)throw d;k.onerror(d)}c||i.notify(b)}]),i.promise},n.prototype.thenResolve=function(a){return E(this,function(){return a})},n.prototype.thenReject=function(a){return E(this,function(){throw a})},mb(["isFulfilled","isRejected","isPending","dispatch","when","spread","get","set","del","delete","post","send","mapply","invoke","mcall","keys","fapply","fcall","fbind","all","allResolved","timeout","delay","catch","finally","fail","fin","progress","done","nfcall","nfapply","nfbind","denodeify","nbind","npost","nsend","nmapply","ninvoke","nmcall","nodeify"],function(a,b){n.prototype[b]=function(){return k[b].apply(k,[this].concat(lb(arguments)))}},void 0),n.prototype.toSource=function(){return this.toString()},n.prototype.toString=function(){return"[object Promise]"},k.nearer=o,k.isPromise=p,k.isPromiseAlike=q,k.isPending=r,k.isFulfilled=s,k.isRejected=t;var vb=[],wb=[],xb=!1,yb=!0;k.resetUnhandledRejections=w,k.getUnhandledReasons=function(){return vb.slice()},k.stopUnhandledRejectionTracking=function(){w(),"undefined"!=typeof process&&process.on&&process.removeListener("exit",v),yb=!1},w(),k.reject=z,k.fulfill=A,k.resolve=B,k.master=D,k.when=E,k.spread=F,k.async=G,k.spawn=H,k["return"]=I,k.promised=J,k.dispatch=K,k.dispatcher=L,k.get=L("get"),k.set=L("set"),k["delete"]=k.del=L("delete");var zb=k.post=L("post");k.mapply=zb,k.send=M,k.invoke=M,k.mcall=M,k.fapply=N,k["try"]=O,k.fcall=O,k.fbind=P,k.keys=L("keys"),k.all=Q,k.allResolved=j(R,"allResolved","allSettled"),k.allSettled=S,k["catch"]=k.fail=T,k.progress=U,k["finally"]=k.fin=V,k.done=W,k.timeout=X,k.delay=Y,k.nfapply=Z,k.nfcall=$,k.nfbind=_,k.denodeify=k.nfbind,k.nbind=ab,k.npost=bb,k.nmapply=bb,k.nsend=cb,k.ninvoke=k.nsend,k.nmcall=k.nsend,k.nodeify=db;var Ab=i();return k}); \ No newline at end of file diff --git a/webharness/test.js b/webharness/test.js new file mode 100644 index 0000000..8cde135 --- /dev/null +++ b/webharness/test.js @@ -0,0 +1,22 @@ +var log = window.opener.log_result; +var pass_count = 0; +var fail_count = 0; + +function ok(condition, message) { + log(!!condition, message); +} + +function is(a, b, message) { + ok(a == b, message); +} + +function isnot(a, b, message) { + ok(a != b, message); +} + +function finish() { + window.opener.test_finished(); +} + +var wait_for_message = window.opener.wait_for_message; +var send_message = window.opener.send_message; diff --git a/webharness/tests/sample.html b/webharness/tests/sample.html new file mode 100644 index 0000000..3a8e735 --- /dev/null +++ b/webharness/tests/sample.html @@ -0,0 +1,25 @@ + + + +sample test + + + + +

Hello world

+ +