Significantly refactor harness.

Start a single httpd to run all HTML tests, loading them from a JSON manifest. Added webharness/ to contain the in-browser harness code. The web harness loads the JSON manifest, connects to a signalling server using socket.io, and waits for the other client to be connected to the signalling server before running tests. Imported q.js as a Promise implementation to keep me sane while writing all this asynchronous logic.
2013-08-30 13:41:53 -04:00 · 2013-08-30 13:41:53 -04:00 · 4a169cb6d0
--- a/sample.html
+++ b/sample.html
@ -1,14 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<title>sample test</title>
-<script>
-addEventListener("load", function() {
-  SpecialPowers.quit();
-});
-</script>
-</head>
-<body>
-<h1>Hello world</h1>
-</body>
-</html>
--- a/steeplechase/runsteeplechase.py
+++ b/steeplechase/runsteeplechase.py
@ -16,6 +16,7 @@ import moznetwork
 import os
 import sys
 import threading
+import uuid

 class Options(OptionParser):
    def __init__(self, **kwargs):
@ -39,6 +40,13 @@ class Options(OptionParser):
        self.add_option("--host2",
                        action="store", type="string", dest="host2",
                        help="first remote host to run tests on")
+        self.add_option("--signalling-server",
+                        action="store", type="string", dest="signalling_server",
+                        help="signalling server URL to use for tests");
+        self.add_option("--noSetup",
+                        action="store_false", dest="setup",
+                        default="True",
+                        help="do not copy files to device");

        self.set_usage(usage)

@ -51,6 +59,7 @@ class RunThread(threading.Thread):
        dm, cmd, env, cond, results = self.args
        try:
            output = dm.shellCheckOutput(cmd, env=env)
+            print "%s\n%s\n%s" % ("=" * 20, output, "=" * 20)
        finally:
            #TODO: actual result
            cond.acquire()
@ -59,17 +68,13 @@ class RunThread(threading.Thread):
            cond.release()
            del self.args

-class HTMLTest(object):
-    def __init__(self, test_file, remote_info, options):
-        self.test_file = os.path.abspath(test_file)
+class HTMLTests(object):
+    def __init__(self, httpd, remote_info, options):
        self.remote_info = remote_info
        self.options = options
-        #XXX: start httpd in main, not here
-        self.httpd = MozHttpd(host=moznetwork.get_ip(),
-                              docroot=os.path.dirname(self.test_file))
+        self.httpd = httpd

    def run(self):
-        self.httpd.start(block=False)
        locations = ServerLocations()
        locations.add_host(host=self.httpd.host,
                           port=self.httpd.port,
@ -84,16 +89,22 @@ class HTMLTest(object):
        prefs = json.loads(json.dumps(prefs) % interpolation)
        for pref in prefs:
          prefs[pref] = Preferences.cast(prefs[pref])
+        prefs["steeplechase.signalling_server"] = self.options.signalling_server
+        prefs["steeplechase.signalling_room"] = str(uuid.uuid4())

        specialpowers_path = self.options.specialpowers
-        with mozfile.TemporaryDirectory() as profile_path:
-            # Create and push profile
-            print "Writing profile..."
-            profile = FirefoxProfile(profile=profile_path,
-                                     preferences=prefs,
-                                     addons=[specialpowers_path],
-                                     locations=locations)
-            for info in self.remote_info:
+        threads = []
+        results = []
+        cond = threading.Condition()
+        for info in self.remote_info:
+            with mozfile.TemporaryDirectory() as profile_path:
+                # Create and push profile
+                print "Writing profile..."
+                prefs["steeplechase.is_initiator"] = info['is_initiator']
+                profile = FirefoxProfile(profile=profile_path,
+                                         preferences=prefs,
+                                         addons=[specialpowers_path],
+                                         locations=locations)
                print "Pushing profile..."
                remote_profile_path = os.path.join(info['test_root'], "profile")
                info['dm'].mkDir(remote_profile_path)
@ -103,36 +114,34 @@ class HTMLTest(object):
            env = {}
            env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
            env["XPCOM_DEBUG_BREAK"] = "warn"
-            env["DISPLAY"] = ":1"
+            env["DISPLAY"] = ":0"

-            threads = []
-            results = []
-            cond = threading.Condition()
-            for info in self.remote_info:
-                cmd = [info['remote_app_path'], "-no-remote",
-                       "-profile", info['remote_profile_path'],
-                       self.httpd.get_url("/" + os.path.basename(self.test_file))]
-                print "cmd: %s" % (cmd, )
-                t = RunThread(args=(info['dm'], cmd, env, cond, results))
-                t.start()
-                threads.append(t)
-            print "Waiting for results..."
-            while threads:
-                cond.acquire()
-                while not results:
-                    cond.wait()
-                res = results.pop(0)
-                cond.release()
-                print "Got result: %d" % res[1]
-                threads.remove(res[0])
-            print "Done!"
-            self.httpd.stop()
+            cmd = [info['remote_app_path'], "-no-remote",
+                   "-profile", info['remote_profile_path'],
+                   self.httpd.get_url("/index.html")]
+            print "cmd: %s" % (cmd, )
+            t = RunThread(args=(info['dm'], cmd, env, cond, results))
+            threads.append(t)
+
+        for t in threads:
+            t.start()
+
+        print "Waiting for results..."
+        while threads:
+            cond.acquire()
+            while not results:
+                cond.wait()
+            res = results.pop(0)
+            cond.release()
+            print "Got result: %d" % res[1]
+            threads.remove(res[0])
+        print "Done!"
        return True

 def main(args):
    parser = Options()
    options, args = parser.parse_args()
-    if not args or not options.binary or not options.specialpowers or not options.host1 or not options.host2:
+    if not options.binary or not options.specialpowers or not options.host1 or not options.host2 or not options.signalling_server:
        parser.print_usage()
        return 2

@ -140,7 +149,7 @@ def main(args):
        parser.error("Binary %s does not exist" % options.binary)
        return 2
    if not os.path.isdir(options.specialpowers):
-        parser.error("SpecialPowers direcotry %s does not exist" % options.specialpowers)
+        parser.error("SpecialPowers directory %s does not exist" % options.specialpowers)
        return 2
    if options.prefs and not os.path.isfile(options.prefs):
        parser.error("Prefs file %s does not exist" % options.prefs)
@ -150,32 +159,33 @@ def main(args):
    log.setLevel(mozlog.DEBUG)
    dm1 = DeviceManagerSUT(options.host1)
    dm2 = DeviceManagerSUT(options.host2)
-    remote_info = [{'dm': dm1}, {'dm': dm2}]
+    remote_info = [{'dm': dm1, 'is_initiator': True},
+                   {'dm': dm2, 'is_initiator': False}]
    # first, push app
    for info in remote_info:
        dm = info['dm']
        test_root = dm.getDeviceRoot() + "/steeplechase"
-        if dm.dirExists(test_root):
-            dm.removeDir(test_root)
-        dm.mkDir(test_root)
+        if options.setup:
+            if dm.dirExists(test_root):
+                dm.removeDir(test_root)
+            dm.mkDir(test_root)
        info['test_root'] = test_root
        app_path = options.binary
        remote_app_dir = test_root + "/app"
-        dm.mkDir(remote_app_dir)
-        dm.pushDir(os.path.dirname(app_path), remote_app_dir)
+        if options.setup:
+            dm.mkDir(remote_app_dir)
+            dm.pushDir(os.path.dirname(app_path), remote_app_dir)
        info['remote_app_path'] = remote_app_dir + "/" + os.path.basename(app_path)

    result = True
-    for arg in args:
-        test = None
-        if arg.endswith(".html"):
-            test = HTMLTest(arg, remote_info, options)
-        else:
-            #TODO: support C++ tests
-            log.error("Unknown test type: %s", arg)
-            continue
-        result = result and test.run()
-
+    #TODO: only start httpd if we have HTML tests
+    httpd = MozHttpd(host=moznetwork.get_ip(),
+                     docroot=os.path.join(os.path.dirname(__file__), "..", "webharness"))
+    httpd.start(block=False)
+    #TODO: support test manifests
+    test = HTMLTests(httpd, remote_info, options)
+    result = test.run()
+    httpd.stop()
    return 0 if result else 1

 if __name__ == '__main__':
--- a/webharness/harness.js
+++ b/webharness/harness.js
@ -0,0 +1,183 @@
+var tests = [];
+var current_test = -1;
+var current_window = null;
+var socket;
+var socket_messages = [];
+var socket_message_deferreds = [];
+var is_initiator = SpecialPowers.getBoolPref("steeplechase.is_initiator");
+
+function fetch_manifest() {
+  var deferred = Q.defer();
+  // Load test manifest
+  var req = new XMLHttpRequest();
+  req.open("GET", "/manifest.json", true);
+  req.responseType = "json";
+  req.overrideMimeType("application/json");
+  req.onload = function() {
+    if (req.readyState == 4) {
+      if (req.status == 200) {
+        deferred.resolve(req.response);
+      } else {
+        deferred.reject(new Error("Error fetching test manifest"));
+      }
+    }
+  };
+  req.onerror = function() {
+    deferred.reject(new Error("Error fetching test manifest"));
+  };
+  req.send(null);
+  return deferred.promise;
+}
+
+function load_script(script) {
+  var deferred = Q.defer();
+  var s = document.createElement("script");
+  s.src = script;
+  s.onload = function() {
+    deferred.resolve(s);
+  };
+  s.onerror = function() {
+    deferred.reject(new Error("Error loading socket.io.js"));
+  };
+  document.head.appendChild(s);
+  return deferred.promise;
+}
+
+/*
+ * Receive a single message from |socket|. If
+ * there is a deferred (from wait_for_message)
+ * waiting on it, resolve that deferred. Otherwise
+ * queue the message for a future waiter.
+ */
+function socket_message(data) {
+  var message = JSON.parse(data);
+  if (socket_message_deferreds.length > 0) {
+    var d = socket_message_deferreds.shift();
+    d.resolve(message);
+  } else {
+    socket_messages.push(message);
+  }
+}
+
+/*
+ * Return a promise for the next available message
+ * to come in on |socket|. If there is a queued
+ * message, resolves the promise immediately, otherwise
+ * waits for socket_message to receive one.
+ */
+function wait_for_message() {
+  var deferred = Q.defer();
+  if (socket_messages.length > 0) {
+    deferred.resolve(socket_messages.shift());
+  } else {
+    socket_message_deferreds.push(deferred);
+  }
+  return deferred.promise;
+}
+
+/*
+ * Send an object as a message on |socket|.
+ */
+function send_message(data) {
+  socket.send(JSON.stringify(data));
+}
+
+function connect_socket() {
+  var server = SpecialPowers.getCharPref("steeplechase.signalling_server");
+  var room = SpecialPowers.getCharPref("steeplechase.signalling_room");
+  var script = server + "socket.io/socket.io.js";
+  return load_script(script).then(function() {
+    var deferred = Q.defer();
+    socket = io.connect(server + "?room=" + room);
+    socket.on("connect", function() {
+      socket.on("message", socket_message);
+      deferred.resolve(socket);
+    });
+    socket.on("error", function() {
+      deferred.reject(new Error("socket.io error"));
+    });
+    socket.on("connect_failed", function() {
+      deferred.reject(new Error("socket failed to connect"));
+    });
+    return deferred;
+  }).then(function () {
+    var deferred = Q.defer();
+    socket.once("numclients", function(data) {
+      if (data.clients == 2) {
+        // Other side is already there.
+        deferred.resolve(socket);
+      } else if (data.clients > 2) {
+        deferred.reject(new Error("Too many clients connected"));
+      } else {
+        // Just us, wait for the other side.
+        socket.once("client_joined", function() {
+          deferred.resolve(socket);
+        });
+      }
+    });
+    return deferred.promise;
+  });
+}
+
+Q.all([fetch_manifest(),
+       connect_socket()]).then(run_tests,
+                               harness_error);
+
+function run_tests(results) {
+  var manifest = results[0];
+  // Manifest looks like:
+  // {'tests': [{'path': '...'}, ...]}
+  tests = manifest.tests;
+  run_next_test();
+}
+
+function run_next_test() {
+  ++current_test;
+  if (current_test >= tests.length) {
+    finish();
+    return;
+  }
+
+  current_window = window.open("/tests/" + tests[current_test].path);
+  current_window.addEventListener("load", function() {
+    dump("loaded\n");
+    send_message({"action": "test_loaded", "test": tests[current_test].path});
+    // Wait for other side to have loaded this test.
+    wait_for_message().then(function (m) {
+      if (m.action != "test_loaded") {
+        //XXX: should this be fatal?
+        harness_error(new Error("Looking for test_loaded, got: " + JSON.stringify(m)));
+        return;
+      }
+      if (m.test != tests[current_test].path) {
+        harness_error(new Error("Wrong test loaded on other side: " + m.test));
+        return;
+      }
+      current_window.test(is_initiator);
+    });
+  });
+  //TODO: timeout handling
+}
+
+function harness_error(error) {
+  log_result(false, error.message, "harness");
+  dump(error.stack +"\n");
+  finish();
+}
+
+// Called by tests via test.js.
+function test_finished() {
+  run_next_test();
+}
+
+function finish() {
+  SpecialPowers.quit();
+}
+
+function log_result(result, message, test) {
+  var output = {'action': result ? "test_pass" : "test_unexpected_fail",
+                'diagnostic': message,
+                'time': Date.now(),
+                'source_file': test || tests[current_test].path};
+  dump(JSON.stringify(output) + "\n");
+}
--- a/webharness/index.html
+++ b/webharness/index.html
@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>Steeplechase harness</title>
+<script src="q.min.js"></script>
+<script src="harness.js"></script>
+</head>
+<body>
+</body>
+</html>
--- a/webharness/manifest.json
+++ b/webharness/manifest.json
@ -0,0 +1,3 @@
+{"tests": [
+          {"path": "sample.html"}
+]}
--- a/webharness/q.min.js
+++ b/webharness/q.min.js
--- a/webharness/test.js
+++ b/webharness/test.js
@ -0,0 +1,22 @@
+var log = window.opener.log_result;
+var pass_count = 0;
+var fail_count = 0;
+
+function ok(condition, message) {
+  log(!!condition, message);
+}
+
+function is(a, b, message) {
+  ok(a == b, message);
+}
+
+function isnot(a, b, message) {
+  ok(a != b, message);
+}
+
+function finish() {
+  window.opener.test_finished();
+}
+
+var wait_for_message = window.opener.wait_for_message;
+var send_message = window.opener.send_message;
--- a/webharness/tests/sample.html
+++ b/webharness/tests/sample.html
@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>sample test</title>
+<script src="/test.js"></script>
+<script>
+function test(is_initiator) {
+  ok(true, "sample test started");
+  if (is_initiator) {
+    send_message({"data": "sample_data"});
+    ok(true, "sent message");
+    finish();
+  } else {
+    wait_for_message().then(function (message) {
+      is(message.data, "sample data", "got right message");
+      finish();
+    });
+  }
+}
+</script>
+</head>
+<body>
+<h1>Hello world</h1>
+</body>
+</html>