Bug 375469: new test framework to run tests in the browser window scope, r=sayrer

2007-07-09 09:24:15 -07:00 · 2007-07-09 09:24:15 -07:00 · 46e8bf86e3
--- a/testing/mochitest/Makefile.in
+++ b/testing/mochitest/Makefile.in
@ -56,6 +56,9 @@ _SERV_FILES = 	runtests.pl \
 		gen_template.pl \
 		server.js \
 		harness.xul \
+		browser-test-overlay.xul \
+		browser-test.js \
+		browser-harness.xul \
 		redirect.html \
 		$(topsrcdir)/netwerk/test/httpserver/httpd.js \
 		$(NULL)	
--- a/testing/mochitest/browser-harness.xul
+++ b/testing/mochitest/browser-harness.xul
@ -0,0 +1,200 @@
+<?xml version="1.0"?>
+<?xml-stylesheet href="chrome://global/skin" type="text/css"?>
+<!-- ***** BEGIN LICENSE BLOCK *****
+   - Version: MPL 1.1/GPL 2.0/LGPL 2.1
+   -
+   - The contents of this file are subject to the Mozilla Public License Version
+   - 1.1 (the "License"); you may not use this file except in compliance with
+   - the License. You may obtain a copy of the License at
+   - http://www.mozilla.org/MPL/
+   -
+   - Software distributed under the License is distributed on an "AS IS" basis,
+   - WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+   - for the specific language governing rights and limitations under the
+   - License.
+   -
+   - The Original Code is Browser Test Harness.
+   -
+   - The Initial Developer of the Original Code is
+   - Mozilla Corporation.
+   -
+   - Portions created by the Initial Developer are Copyright (C) 2007
+   - the Initial Developer. All Rights Reserved.
+   -
+   - Contributor(s):
+   -      Gavin Sharp <gavin@gavinsharp.com> (original author)
+   -
+   - Alternatively, the contents of this file may be used under the terms of
+   - either the GNU General Public License Version 2 or later (the "GPL"), or
+   - the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+   - in which case the provisions of the GPL or the LGPL are applicable instead
+   - of those above. If you wish to allow use of your version of this file only
+   - under the terms of either the GPL or the LGPL, and not to allow others to
+   - use your version of this file under the terms of the MPL, indicate your
+   - decision by deleting the provisions above and replace them with the notice
+   - and other provisions required by the LGPL or the GPL. If you do not delete
+   - the provisions above, a recipient may use your version of this file under
+   - the terms of any one of the MPL, the GPL or the LGPL.
+   -
+   - ***** END LICENSE BLOCK ***** -->
+
+<window id="browserTestHarness"
+        xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"
+        onload="TestStart();"
+        title="Browser chrome tests">
+  <script src="chrome://mochikit/content/tests/SimpleTest/MozillaFileLogger.js"/>
+  <script src="chrome://mochikit/content/tests/SimpleTest/quit.js"/>
+  <script type="application/javascript;version=1.7"><![CDATA[
+    var gConfig;
+    function TestStart() {
+      gConfig = readConfig();
+
+      if (gConfig.autoRun)
+        setTimeout(runAllTests, 0);
+    }
+
+    function readConfig() {
+      var fileLocator = Cc["@mozilla.org/file/directory_service;1"].
+                        getService(Ci.nsIProperties);
+      var configFile = fileLocator.get("ProfD", Ci.nsIFile);
+      configFile.append("testConfig.js");
+
+      if (!configFile.exists())
+        return;
+
+      var fileInStream = Cc["@mozilla.org/network/file-input-stream;1"].
+                         createInstance(Ci.nsIFileInputStream);
+      var sstream = Cc["@mozilla.org/scriptableinputstream;1"].
+                    createInstance(Ci.nsIScriptableInputStream);
+      fileInStream.init(configFile, -1, 0, 0);
+      sstream.init(fileInStream);
+
+      var config = "";
+      var str = sstream.read(4096);
+      while (str.length > 0) {
+        config += str;
+        str = sstream.read(4096);
+      }
+
+      sstream.close();
+      fileInStream.close();
+
+      return eval(config);
+    }
+
+    function getChromeDir() {
+      const Cc = Components.classes; const Ci = Components.interfaces;
+
+      /** Find our chrome dir **/
+      var ios = Cc["@mozilla.org/network/io-service;1"].
+                  getService(Ci.nsIIOService);
+      var chromeURI = ios.newURI("chrome://mochikit/content/",
+                                 null, null);
+      var resolvedURI = Cc["@mozilla.org/chrome/chrome-registry;1"].
+                        getService(Ci.nsIChromeRegistry).
+                        convertChromeURL(chromeURI);
+      var fileHandler = Cc["@mozilla.org/network/protocol;1?name=file"].
+                        getService(Ci.nsIFileProtocolHandler);
+      var chromeDir = fileHandler.getFileFromURLSpec(resolvedURI.spec);
+
+      return chromeDir.parent.QueryInterface(Ci.nsILocalFile);
+    }
+
+    function browserTestFile(aTestFile) {
+      this.path = aTestFile;
+      this.exception = null;
+      this.timedOut = false;
+      this.tests = [];
+      this.scope = null;
+    }
+    browserTestFile.prototype = {
+      get allPassed() {
+        return !this.tests.some(function (t) !t.pass);
+      },
+      get log() {
+        return this.tests.map(function (t) t.msg).join("\n");
+      }
+    };
+
+    // Returns an array of chrome:// URLs to all the test files
+    function listTests() {
+      const Cc = Components.classes; const Ci = Components.interfaces;
+
+      var testsDir = getChromeDir();
+      testsDir.appendRelativePath("browser");
+
+      /** load server.js in so we can share template functions **/
+      var scriptLoader = Cc["@mozilla.org/moz/jssubscript-loader;1"].
+                         getService(Ci.mozIJSSubScriptLoader);
+      var srvScope = {};
+      scriptLoader.loadSubScript("chrome://mochikit/content/server.js", srvScope);
+
+      var [links, count] = srvScope.list("chrome://mochikit/content/browser/",
+                                         testsDir, true);
+      var fileNames = [];
+      srvScope.arrayOfTestFiles(links, fileNames, /browser_.+\.js$/);
+
+      return fileNames.map(function (f) new browserTestFile(f));;
+    }
+
+    function setStatus(aStatusString) {
+      document.getElementById("status").value = aStatusString;
+    }
+
+    function runAllTests() {
+      var windowMediator = Cc['@mozilla.org/appshell/window-mediator;1'].
+                             getService(Ci.nsIWindowMediator);
+      var testWin = windowMediator.getMostRecentWindow("navigator:browser");
+
+      setStatus("Running...");
+      testWin.focus();
+
+      var Tester = new testWin.Tester(listTests(), testsFinished);
+      Tester.start();
+    }
+
+    function getLogFromTests(aTests) {
+      return aTests.map(function (f) {
+                          var output = f.path + "\n";
+                          if (f.log)
+                            output += f.log + "\n";
+                          if (f.exception)
+                            output += "\tFAIL - Exception thrown: " + f.exception + "\n";
+                          if (f.timedOut)
+                            output += "\tFAIL - Timed out\n";
+                          return output;
+                        }).join("");
+    }
+
+    function testsFinished(aTests) {
+      // Focus our window, to display the results
+      window.focus();
+
+      var start = "*** Start BrowserChrome Test Results ***\n";
+      var end = "\n*** End BrowserChrome Test Results ***\n";
+      var output = start + getLogFromTests(aTests) + end;
+
+      // Output to stdout
+      dump(output);
+
+      // Output to file
+      if (gConfig.logPath) {
+        MozillaFileLogger.init(gConfig.logPath);
+        MozillaFileLogger._foStream.write(output, output.length);
+        MozillaFileLogger.close();
+      }
+
+      if (gConfig.closeWhenDone) {
+        goQuitApplication();
+        return;
+      }
+
+      // UI
+      document.getElementById("results").value = output;
+      setStatus("Done.");
+    }
+  ]]></script>
+  <button onclick="runAllTests();" label="Run All Tests"/>
+  <label id="status"/>
+  <textbox flex="1" multiline="true" id="results"/>
+</window>
--- a/testing/mochitest/browser-test-overlay.xul
+++ b/testing/mochitest/browser-test-overlay.xul
@ -0,0 +1,44 @@
+<?xml version="1.0"?>
+<?xml-stylesheet href="chrome://global/skin" type="text/css"?>
+<!-- ***** BEGIN LICENSE BLOCK *****
+   - Version: MPL 1.1/GPL 2.0/LGPL 2.1
+   -
+   - The contents of this file are subject to the Mozilla Public License Version
+   - 1.1 (the "License"); you may not use this file except in compliance with
+   - the License. You may obtain a copy of the License at
+   - http://www.mozilla.org/MPL/
+   -
+   - Software distributed under the License is distributed on an "AS IS" basis,
+   - WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+   - for the specific language governing rights and limitations under the
+   - License.
+   -
+   - The Original Code is Browser Test Harness.
+   -
+   - The Initial Developer of the Original Code is
+   - Mozilla Corporation.
+   -
+   - Portions created by the Initial Developer are Copyright (C) 2007
+   - the Initial Developer. All Rights Reserved.
+   -
+   - Contributor(s):
+   -      Gavin Sharp <gavin@gavinsharp.com> (original author)
+   -
+   - Alternatively, the contents of this file may be used under the terms of
+   - either the GNU General Public License Version 2 or later (the "GPL"), or
+   - the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+   - in which case the provisions of the GPL or the LGPL are applicable instead
+   - of those above. If you wish to allow use of your version of this file only
+   - under the terms of either the GPL or the LGPL, and not to allow others to
+   - use your version of this file under the terms of the MPL, indicate your
+   - decision by deleting the provisions above and replace them with the notice
+   - and other provisions required by the LGPL or the GPL. If you do not delete
+   - the provisions above, a recipient may use your version of this file under
+   - the terms of any one of the MPL, the GPL or the LGPL.
+   -
+   - ***** END LICENSE BLOCK ***** -->
+
+<overlay id="browserTestOverlay"
+         xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
+  <script type="application/javascript" src="chrome://mochikit/content/browser-test.js"/>
+</overlay>
--- a/testing/mochitest/browser-test.js
+++ b/testing/mochitest/browser-test.js
@ -0,0 +1,179 @@
+window.addEventListener("load", testOnLoad, false);
+
+function testOnLoad() {
+  const Cc = Components.classes;
+  const Ci = Components.interfaces;
+
+  // Make sure to launch the test harness for the first opened window only
+  var prefs = Cc["@mozilla.org/preferences-service;1"].
+              getService(Ci.nsIPrefBranch);
+  if (prefs.prefHasUserValue("testing.browserTestHarness.running"))
+    return;
+
+  prefs.setBoolPref("testing.browserTestHarness.running", true);
+
+  var ww = Cc["@mozilla.org/embedcomp/window-watcher;1"].
+           getService(Ci.nsIWindowWatcher);
+  var sstring = Cc["@mozilla.org/supports-string;1"].
+                createInstance(Ci.nsISupportsString);
+  sstring.data = location.search;
+  ww.openWindow(window, "chrome://mochikit/content/browser-harness.xul", "browserTest",
+                "chrome,centerscreen,dialog,resizable,titlebar,toolbar=no,width=800,height=600", sstring);
+}
+
+function Tester(aTests, aCallback) {
+  this.tests = aTests;
+  this.callback = aCallback;
+}
+Tester.prototype = {
+  checker: null,
+  currentTestIndex: -1,
+  get currentTest() {
+    return this.tests[this.currentTestIndex];
+  },
+  get done() {
+    return this.currentTestIndex == this.tests.length - 1;
+  },
+  step: function Tester_step() {
+    this.currentTestIndex++;
+  },
+  
+  start: function Tester_start() {
+    this.execTest();
+  },
+
+  finish: function Tester_finish() {
+    // Tests complete, notify the callback and return
+    this.callback(this.tests);
+    this.callback = null;
+    this.tests = null;
+  },
+
+  execTest: function Tester_execTest() {
+    if (this.done) {
+      this.finish();
+      return;
+    }
+
+    // Move to the next test (or first test).
+    this.step();
+
+    // Load the tests into a testscope
+    this.currentTest.scope = new testScope(this.currentTest.tests);
+
+    var scriptLoader = Cc["@mozilla.org/moz/jssubscript-loader;1"].
+                       getService(Ci.mozIJSSubScriptLoader);
+    scriptLoader.loadSubScript(this.currentTest.path, this.currentTest.scope);
+
+    // Run the test
+    var exception = null;
+    try {
+      this.currentTest.scope.test();
+    } catch (ex) {
+      this.currentTest.exception = ex;
+    }
+
+    // If the test ran synchronously, set the result and move to the next test,
+    // otherwise start a poller to monitor it's progress.
+    if (this.currentTest.scope.done) {
+      this.currentTest.result = this.currentTest.scope.result;
+      this.execTest();
+    } else {
+      var self = this;
+      this.checker = new resultPoller(this.currentTest, function () { self.execTest(); });
+      this.checker.start();
+    }
+  }
+};
+
+function testResult(aCondition, aName, aDiag, aIsTodo) {
+  aName = aName || "";
+
+  this.pass = !!aCondition;
+  if (this.pass)
+    this.msg = "\tPASS - " + aName;
+  else {
+    this.msg = "\tFAIL - ";
+    if (aIsTodo)
+      this.msg += "TODO Worked? - ";
+    this.msg += aName;
+    if (aDiag)
+      this.msg += " - " + aDiag;
+  }
+}
+
+function testScope(aTests) {
+  var scriptLoader = Cc["@mozilla.org/moz/jssubscript-loader;1"].
+                     getService(Ci.mozIJSSubScriptLoader);
+  scriptLoader.loadSubScript("chrome://mochikit/content/tests/SimpleTest/EventUtils.js", this.EventUtils);
+
+  this.tests = aTests;
+}
+testScope.prototype = {
+  ok: function test_ok(condition, name, diag) {
+    this.tests.push(new testResult(condition, name, diag, false));
+  },
+  
+  is: function test_is(a, b, name) {
+    this.ok(a == b, name, "Got " + a + ", expected " + b);
+  },
+  
+  isnot: function test_isnot(a, b, name) {
+    this.ok(a != b, name, "Didn't expect " + a + ", but got it");
+  },
+
+  todo: function test_todo(condition, name, diag) {
+    this.tests.push(new testResult(!condition, name, diag, true));
+  },
+
+  done: true,
+  waitForExplicitFinish: function test_WFEF() {
+    this.done = false;
+  },
+  finish: function test_finish() {
+    this.done = true;
+  },
+
+  EventUtils: {}
+};
+
+// Check whether the test has completed every 3 seconds
+const CHECK_INTERVAL = 3000;
+// Test timeout (seconds)
+const TIMEOUT_SECONDS = 15;
+
+const MAX_LOOP_COUNT = (TIMEOUT_SECONDS * 1000) / CHECK_INTERVAL;
+
+function resultPoller(aTest, aCallback) {
+  this.test = aTest;
+  this.callback = aCallback;
+}
+resultPoller.prototype = {
+  loopCount: 0,
+  interval: 0,
+
+  start: function resultPoller_start() {
+    var self = this;
+    function checkDone() {
+      self.loopCount++;
+  
+      if (self.loopCount > MAX_LOOP_COUNT) {
+        self.test.timedOut = true;
+        self.test.scope.done = true;
+      }
+
+      if (self.test.scope.done) {
+        // Set the result
+        self.test.result = self.test.scope.result;
+
+        clearInterval(self.interval);
+
+        // Notify the callback
+        self.callback();
+        self.callback = null; 
+        self.test = null;
+      }
+    }
+    this.interval = setInterval(checkDone, CHECK_INTERVAL);
+  }
+};
--- a/testing/mochitest/runtests.pl.in
+++ b/testing/mochitest/runtests.pl.in
@ -123,15 +123,17 @@ my $unixish = (!($is_win32) && !($is_mac));

 sub main {
  my ($close_when_done, $appoverride, $log_path, $autorun,
-      $console_level, $file_level, $help, $do_chrome, $test_path); 
+      $console_level, $file_level, $help, $do_chrome, $test_path,
+      $do_browser_chrome);
  GetOptions("close-when-done!"=> \$close_when_done,
             "appname:s"=> \$appoverride,
             "log-file:s" => \$log_path,
             "autorun!" => \$autorun,
             "console-level:s" => \$console_level,
             "file-level:s" => \$file_level,
-	     "chrome!" => \$do_chrome,
-	     "test-path:s" => \$test_path,
+             "chrome!" => \$do_chrome,
+             "test-path:s" => \$test_path,
+             "browser-chrome!" => \$do_browser_chrome,
             "help!" => \$help);

  # if the switches include --help, exit and print directions
@ -152,42 +154,54 @@ sub main {
    die $error_message;
  }

-  my $manifest = initializeProfile($app);
-  my $serverPid = startServer($close_when_done);
-
-  # If we're lucky, the server has fully started by now, and all paths are
-  # ready, etc.  However, xpcshell cold start times suck, at least for debug
-  # builds.  We'll try to connect to the server for 30 seconds or until we
-  # succeed, whichever is first.  If we succeed, then we continue with
-  # execution.  If we fail, we try to kill the server and exit with an error.
-  wait_for_server_startup($serverPid, SERVER_STARTUP_TIMEOUT);
+  my $manifest = initializeProfile($app, $do_browser_chrome);
+  my $serverPid;
+  if (!$do_browser_chrome) {
+    $serverPid = startServer($close_when_done);
+  
+    # If we're lucky, the server has fully started by now, and all paths are
+    # ready, etc.  However, xpcshell cold start times suck, at least for debug
+    # builds.  We'll try to connect to the server for 30 seconds or until we
+    # succeed, whichever is first.  If we succeed, then we continue with
+    # execution.  If we fail, we try to kill the server and exit with an error.
+    wait_for_server_startup($serverPid, SERVER_STARTUP_TIMEOUT);
+  }

  my $url;
  if ($do_chrome) {
   $url = CHROMETESTS_URL . ($test_path ? $test_path : "") . "?";
+  } elsif ($do_browser_chrome) {
+   # Tests will run from an overlay, no need to load any URL
+   $url = "about:blank";
  } else {
   $url = TESTS_URL . ($test_path ? $test_path : "") . "?";
  }

-  if ($autorun) {
-    $url .= "&autorun=1";
-  }
-  if ($close_when_done) {
-    $url .= "&closeWhenDone=1";
-  }
-  if ($log_path) {
-    $url .= "&logFile=$log_path";
-  }
-  if ($file_level) {
-    $url .= "&fileLevel=$file_level";
-  }
-  if ($console_level) {
-    $url .= "&consoleLevel=$console_level";
+  if ($do_browser_chrome) {
+    generate_test_config($autorun, $close_when_done, $log_path);
+  } else {
+    if ($autorun) {
+      $url .= "&autorun=1";
+    }
+    if ($close_when_done) {
+      $url .= "&closeWhenDone=1";
+    }
+    if ($log_path) {
+      $url .= "&logFile=$log_path";
+    }
+    if ($file_level) {
+      $url .= "&fileLevel=$file_level";
+    }
+    if ($console_level) {
+      $url .= "&consoleLevel=$console_level";
+    }
  }
  
  my $test_start = runTests($url);

-  shutdownServer($serverPid);
+  if (!$do_browser_chrome) {
+    shutdownServer($serverPid);
+  }

  # print test run times
  my $test_finish = localtime();
@ -208,11 +222,13 @@ sub usage_and_exit {
  print "Usage instructons for runtests.pl.\n";
  print "If --log-file is specified, --file-level must be specified as well.\n";
  print "If --chrome is specified, chrome tests will be run instead of web content tests";
+  print "If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests";
  print "\n\n";
  print "Syntax:\n";
  print "  runtests.pl \\\n";
  print "   [--autorun] \\\n";
  print "   [--chrome] \\\n";
+  print "   [--browser-chrome] \\\n";
  print "   [--close-when-done] \\\n";
  print "   [--appname=/path/to/app] \\\n";
  print "   [--log-file=/path/to/logfile] \\\n";
@ -318,8 +334,29 @@ sub startServer {
 # TEST SETUP #
 ##############

+sub generate_test_config {
+  my ($autorun, $close_when_done, $log_path) = @_;
+  $autorun = $autorun || 0;
+  $close_when_done = $close_when_done || 0;
+  $log_path = $log_path || "";
+  $log_path =~ s/\\/\\\\/;
+
+  my $config_content = <<CONFIGEND;
+({
+  autoRun: $autorun,
+  closeWhenDone: $close_when_done,
+  logPath: "$log_path"
+})
+CONFIGEND
+
+  open(CONFIGOUTFILE, ">$profile_dir/testConfig.js") ||
+    die("Could not open testConfig.js file $!");
+  print CONFIGOUTFILE ($config_content);
+  close(CONFIGOUTFILE);
+}
+
 sub initializeProfile {
-  my ($app_path) = @_;
+  my ($app_path, $do_browser_tests) = @_;
  my $pref_content = <<PREFEND;
 user_pref("browser.dom.window.dump.enabled", true);
 user_pref("capability.principal.codebase.p1.granted", "UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite UniversalPreferencesRead UniversalPreferencesWrite UniversalFileRead");
@ -378,7 +415,10 @@ CHROMEEND
  my $manifest = $directories . "chrome/mochikit.manifest";
  open(MANIFEST, ">$manifest") ||
    die("Could not open manifest file $!");
-  print MANIFEST ("content mochikit $chrometest_dir");
+  print MANIFEST ("content mochikit $chrometest_dir\n");
+  if ($do_browser_tests) {
+    print MANIFEST ("overlay chrome://browser/content/browser.xul chrome://mochikit/content/browser-test-overlay.xul\n");
+  }
  close(MANIFEST);

  return $manifest;
--- a/testing/mochitest/server.js
+++ b/testing/mochitest/server.js
@ -276,11 +276,14 @@ function list(requestPath, directory, recurse)
 * is a test case to be executed in the harness, or just
 * a supporting file.
 */
-function isTest(filename)
+function isTest(filename, pattern)
 {
-  return  (filename.indexOf("test_") > -1 &&
-           filename.indexOf(".js") == -1 &&
-           filename.indexOf(".css") == -1);
+  if (pattern)
+    return pattern.test(filename);
+
+  return filename.indexOf("test_") > -1 &&
+         filename.indexOf(".js") == -1 &&
+         filename.indexOf(".css") == -1;
 }

 /**
@ -339,24 +342,23 @@ function linksToTableRows(links)
  return response;
 }

+function arrayOfTestFiles(linkArray, fileArray, testPattern) {
+  for (var [link, value] in linkArray) {
+    if (value instanceof Object) {
+      arrayOfTestFiles(value, fileArray, testPattern);
+    } else if (isTest(link, testPattern)) {
+      fileArray.push(link)
+    }
+  }
+}
 /**
 * Produce a flat array of test file paths to be executed in the harness.
 */
 function jsonArrayOfTestFiles(links)
 {
  var testFiles = [];
-  function arrayOfTestFiles(linkArray) {
-    for (var [link, value] in linkArray) {
-      if (value instanceof Object) {
-        arrayOfTestFiles(value);
-      } else {
-        testFiles.push(link)
-      }
-    }
-  }
-  arrayOfTestFiles(links);
-  var testFiles = ['"' + file + '"' for each(file in testFiles)
-                   if (isTest(file))];
+  arrayOfTestFiles(links, testFiles);
+  testFiles = ['"' + file + '"' for each(file in testFiles)];
  return "[" + testFiles.join(",\n") + "]";
 }