Bug 1190474 - Allow test-specific timeouts to be specified in cppunittest.ini, r=chmanchester

--HG--
extra : commitid : KHoHs6bXJln

testing/config/mozharness/b2g_emulator_config.py

@@ -12,7 +12,6 @@ config = {
                 "--addEnv",
                 "LD_LIBRARY_PATH=/vendor/lib:/system/lib:/system/b2g",
                 "--with-b2g-emulator=%(b2gpath)s",
-                "--skip-manifest=b2g_cppunittest_manifest.txt",
                 "."
             ],
             "run_filename": "remotecppunittests.py",

testing/mach_commands.py

@@ -299,9 +299,10 @@ class MachCommands(MachCommandBase):
 
         if len(params['test_files']) == 0:
             testdir = os.path.join(self.distdir, 'cppunittests')
-            tests = cppunittests.extract_unittests_from_args([testdir], mozinfo.info)
+            manifest = os.path.join(self.topsrcdir, 'testing', 'cppunittest.ini')
+            tests = cppunittests.extract_unittests_from_args([testdir], mozinfo.info, manifest)
         else:
-            tests = cppunittests.extract_unittests_from_args(params['test_files'], mozinfo.info)
+            tests = cppunittests.extract_unittests_from_args(params['test_files'], mozinfo.info, None)
 
         # See if we have crash symbols
         symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')

testing/remotecppunittests.py

@@ -114,7 +114,8 @@ class RemoteCPPUnitTests(cppunittests.CPPUnitTests):
 
         return env
 
-    def run_one_test(self, prog, env, symbols_path=None, interactive=False):
+    def run_one_test(self, prog, env, symbols_path=None, interactive=False,
+                     timeout_factor=1):
         """
         Run a single C++ unit test program remotely.
 
@@ -123,6 +124,7 @@ class RemoteCPPUnitTests(cppunittests.CPPUnitTests):
         * env: The environment to use for running the program.
         * symbols_path: A path to a directory containing Breakpad-formatted
                         symbol files for producing stack traces on crash.
+        * timeout_factor: An optional test-specific timeout multiplier.
 
         Return True if the program exits with a zero status, False otherwise.
         """
@@ -130,8 +132,9 @@ class RemoteCPPUnitTests(cppunittests.CPPUnitTests):
         remote_bin = posixpath.join(self.remote_bin_dir, basename)
         self.log.test_start(basename)
         buf = StringIO.StringIO()
+        test_timeout = cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
         returncode = self.device.shell([remote_bin], buf, env=env, cwd=self.remote_home_dir,
-                                       timeout=cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT)
+                                       timeout=test_timeout)
         self.log.process_output(basename, "\n%s" % buf.getvalue(),
                                 command=[remote_bin])
         with mozfile.TemporaryDirectory() as tempdir:
@@ -254,8 +257,10 @@ def main():
 
     options.xre_path = os.path.abspath(options.xre_path)
     cppunittests.update_mozinfo()
-    progs = cppunittests.extract_unittests_from_args(args, mozinfo.info)
-    tester = RemoteCPPUnitTests(dm, options, progs)
+    progs = cppunittests.extract_unittests_from_args(args,
+                                                     mozinfo.info,
+                                                     options.manifest_path)
+    tester = RemoteCPPUnitTests(dm, options, [item[0] for item in progs])
     try:
         result = tester.run_tests(progs, options.xre_path, options.symbols_path)
     except Exception, e:

testing/runcppunittests.py

@@ -24,7 +24,8 @@ class CPPUnitTests(object):
     # Time (seconds) in which process will be killed if it produces no output.
     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
 
-    def run_one_test(self, prog, env, symbols_path=None, interactive=False):
+    def run_one_test(self, prog, env, symbols_path=None, interactive=False,
+                     timeout_factor=1):
         """
         Run a single C++ unit test program.
 
@@ -33,6 +34,7 @@ class CPPUnitTests(object):
         * env: The environment to use for running the program.
         * symbols_path: A path to a directory containing Breakpad-formatted
                         symbol files for producing stack traces on crash.
+        * timeout_factor: An optional test-specific timeout multiplier.
 
         Return True if the program exits with a zero status, False otherwise.
         """
@@ -53,7 +55,8 @@ class CPPUnitTests(object):
                                                  processOutputLine=lambda _: None)
             #TODO: After bug 811320 is fixed, don't let .run() kill the process,
             # instead use a timeout in .wait() and then kill to get a stack.
-            proc.run(timeout=CPPUnitTests.TEST_PROC_TIMEOUT,
+            test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
+            proc.run(timeout=test_timeout,
                      outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT)
             proc.wait()
             if proc.output:
@@ -133,7 +136,7 @@ class CPPUnitTests(object):
         Run a set of C++ unit test programs.
 
         Arguments:
-        * programs: An iterable containing paths to test programs.
+        * programs: An iterable containing (test path, test timeout factor) tuples
         * xre_path: A path to a directory containing a XUL Runtime Environment.
         * symbols_path: A path to a directory containing Breakpad-formatted
                         symbol files for producing stack traces on crash.
@@ -148,7 +151,10 @@ class CPPUnitTests(object):
         pass_count = 0
         fail_count = 0
         for prog in programs:
-            single_result = self.run_one_test(prog, env, symbols_path, interactive)
+            test_path = prog[0]
+            timeout_factor = prog[1]
+            single_result = self.run_one_test(test_path, env, symbols_path,
+                                              interactive, timeout_factor)
             if single_result:
                 pass_count += 1
             else:
@@ -172,33 +178,41 @@ class CPPUnittestOptions(OptionParser):
                         action = "store", type = "string", dest = "symbols_path",
                         default = None,
                         help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
-        self.add_option("--skip-manifest",
-                        action = "store", type = "string", dest = "manifest_file",
+        self.add_option("--manifest-path",
+                        action = "store", type = "string", dest = "manifest_path",
                         default = None,
-                        help = "absolute path to a manifest file")
+                        help = "path to test manifest, if different from the path to test binaries")
 
-def extract_unittests_from_args(args, environ):
+def extract_unittests_from_args(args, environ, manifest_path):
     """Extract unittests from args, expanding directories as needed"""
     mp = manifestparser.TestManifest(strict=True)
     tests = []
-    for p in args:
-        if os.path.isdir(p):
-            try:
-                mp.read(os.path.join(p, 'cppunittest.ini'))
-            except IOError:
-                tests.extend([os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)])
-        else:
-            tests.append(os.path.abspath(p))
+    binary_path = None
+
+    if manifest_path:
+        mp.read(manifest_path)
+        binary_path = os.path.abspath(args[0])
+    else:
+        for p in args:
+            if os.path.isdir(p):
+                try:
+                    mp.read(os.path.join(p, 'cppunittest.ini'))
+                except IOError:
+                    tests.extend([(os.path.abspath(os.path.join(p, x)), 1) for x in os.listdir(p)])
+            else:
+                tests.append((os.path.abspath(p), 1))
 
     # we skip the existence check here because not all tests are built
     # for all platforms (and it will fail on Windows anyway)
-    if mozinfo.isWin:
-        tests.extend([test['path'] + '.exe' for test in mp.active_tests(exists=False, disabled=False, **environ)])
+    active_tests = mp.active_tests(exists=False, disabled=False, **environ)
+    suffix = '.exe' if mozinfo.isWin else ''
+    if binary_path:
+        tests.extend([(os.path.join(binary_path, test['relpath'] + suffix), int(test.get('requesttimeoutfactor', 1))) for test in active_tests])
     else:
-        tests.extend([test['path'] for test in mp.active_tests(exists=False, disabled=False, **environ)])
+        tests.extend([(test['path'] + suffix, int(test.get('requesttimeoutfactor', 1))) for test in active_tests])
 
     # skip non-existing tests
-    tests = [test for test in tests if os.path.isfile(test)]
+    tests = [test for test in tests if os.path.isfile(test[0])]
 
     return tests
 
@@ -223,12 +237,15 @@ def main():
     if not options.xre_path:
         print >>sys.stderr, """Error: --xre-path is required"""
         sys.exit(1)
+    if options.manifest_path and len(args) > 1:
+        print >>sys.stderr, "Error: multiple arguments not supported with --test-manifest"
+        sys.exit(1)
 
     log = mozlog.commandline.setup_logging("cppunittests", options,
                                            {"tbpl": sys.stdout})
 
     update_mozinfo()
-    progs = extract_unittests_from_args(args, mozinfo.info)
+    progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
     options.xre_path = os.path.abspath(options.xre_path)
     if mozinfo.isMac:
         options.xre_path = os.path.join(os.path.dirname(options.xre_path), 'Resources')