Bug 1614626 - Move xpcshell selftests to before running actual xpcshell tests. r=gbrown

This allows to test them in the exact same environment as the tests are
going to run, which turns out to have revealed a few issues that would
only appear once xpcshell tests fail, impeding on debugging those
failures.

Differential Revision: https://phabricator.services.mozilla.com/D84781

python/mozbuild/mozbuild/action/test_archive.py

@@ -634,6 +634,7 @@ ARCHIVE_FILES = {
                 'dns-packet/**',
                 'remotexpcshelltests.py',
                 'runxpcshelltests.py',
+                'selftest.py',
                 'xpcshellcommandline.py',
             ],
             'dest': 'xpcshell',

testing/mozharness/configs/unittests/linux_unittest.py

@@ -118,6 +118,7 @@ config = {
         },
         "xpcshell": {
             "options": [
+                "--self-test",
                 "--symbols-path=%(symbols_path)s",
                 "--test-plugin-path=%(test_plugin_path)s",
                 "--log-raw=%(raw_log_file)s",

testing/mozharness/configs/unittests/mac_unittest.py

@@ -95,6 +95,7 @@ config = {
         },
         "xpcshell": {
             "options": [
+                "--self-test",
                 "--symbols-path=%(symbols_path)s",
                 "--test-plugin-path=%(test_plugin_path)s",
                 "--log-raw=%(raw_log_file)s",

testing/mozharness/configs/unittests/win_unittest.py

@@ -115,6 +115,7 @@ config = {
         },
         "xpcshell": {
             "options": [
+                "--self-test",
                 "--symbols-path=%(symbols_path)s",
                 "--test-plugin-path=%(test_plugin_path)s",
                 "--log-raw=%(raw_log_file)s",

testing/testsuite-targets.mk

@@ -259,12 +259,9 @@ stage-extensions: make-stage-dir
 	@$(foreach ext,$(TEST_EXTENSIONS), cp -RL $(DIST)/xpi-stage/$(ext) $(PKG_STAGE)/extensions;)
 
 
+ifeq (,$(MOZ_ASAN)$(MOZ_TSAN)$(FUZZING_INTERFACES))  # No tests run here for asan/tsan/fuzzing builds.
 check::
 	$(eval cores=$(shell $(PYTHON3) -c 'import multiprocessing; print(multiprocessing.cpu_count())'))
-	@echo "Starting 'mach python-test' with -j$(cores)"
-	@$(topsrcdir)/mach --log-no-times python-test -j$(cores) --subsuite default
-	@echo "Finished 'mach python-test' successfully"
-ifeq (,$(MOZ_ASAN)$(MOZ_TSAN)$(FUZZING_INTERFACES))  # No tests run here for asan/tsan/fuzzing builds.
 	@echo "Starting 'mach python-test' with --python $(PYTHON3) -j$(cores)"
 	@$(topsrcdir)/mach --log-no-times python-test --python python3 -j$(cores) --subsuite default
 	@echo "Finished 'mach python-test' with py3 successfully"

testing/xpcshell/moz.build

@@ -6,10 +6,6 @@
 
 TEST_DIRS += ['example', 'moz-http2']
 
-PYTHON_UNITTEST_MANIFESTS += [
-    'python.ini',
-]
-
 TESTING_JS_MODULES += [
     'dbg-actors.js',
 ]

testing/xpcshell/python.ini

@@ -1,2 +0,0 @@
-[selftest.py]
-skip-if = python == 3

testing/xpcshell/runxpcshelltests.py

@@ -1418,6 +1418,22 @@ class XPCShellTests(object):
 
         return True
 
+    def runSelfTest(self):
+        import selftest
+        import unittest
+        this = self
+
+        class XPCShellTestsTests(selftest.XPCShellTestsTests):
+            def __init__(self, name):
+                unittest.TestCase.__init__(self, name)
+                self.testing_modules = this.testingModulesDir
+                self.xpcshellBin = this.xpcshell
+                self.utility_path = this.utility_path
+                self.symbols_path = this.symbolsPath
+
+        suite = unittest.TestLoader().loadTestsFromTestCase(XPCShellTestsTests)
+        return unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful()
+
     def runTests(self, options, testClass=XPCShellTestThread, mobileArgs=None):
         """
           Run xpcshell tests.
@@ -1491,7 +1507,7 @@ class XPCShellTests(object):
         self.verboseIfFails = options.get('verboseIfFails')
         self.keepGoing = options.get('keepGoing')
         self.logfiles = options.get('logfiles')
-        self.totalChunks = options.get('totalChunks')
+        self.totalChunks = options.get('totalChunks', 1)
         self.thisChunk = options.get('thisChunk')
         self.profileName = options.get('profileName') or "xpcshell"
         self.mozInfo = options.get('mozInfo')
@@ -1518,6 +1534,10 @@ class XPCShellTests(object):
         if not self.updateMozinfo(prefs, options):
             return False
 
+        if options.get('self_test'):
+            if not self.runSelfTest():
+                return False
+
         if "tsan" in self.mozInfo and self.mozInfo["tsan"] and not options.get('threadCount'):
             # TSan requires significantly more memory, so reduce the amount of parallel
             # tests we run to avoid OOMs and timeouts.

testing/xpcshell/selftest.py

@@ -7,7 +7,6 @@
 from __future__ import absolute_import
 
 import mozinfo
-import mozunit
 import os
 import pprint
 import re
@@ -17,26 +16,10 @@ import sys
 import tempfile
 import unittest
 
-from buildconfig import substs
-from StringIO import StringIO
 from mozlog import structured
-from mozbuild.base import MozbuildObject
-os.environ.pop('MOZ_OBJDIR', None)
-build_obj = MozbuildObject.from_environment()
 
 from runxpcshelltests import XPCShellTests
 
-mozinfo.find_and_update_from_json()
-
-objdir = build_obj.topobjdir.encode("utf-8")
-
-if mozinfo.isMac:
-    xpcshellBin = os.path.join(objdir, "dist", substs['MOZ_MACBUNDLE_NAME'],
-                               "Contents", "MacOS", "xpcshell")
-else:
-    xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
-    if sys.platform == "win32":
-        xpcshellBin += ".exe"
 
 TEST_PASS_STRING = "TEST-PASS"
 TEST_FAIL_STRING = "TEST-UNEXPECTED-FAIL"
@@ -491,19 +474,37 @@ class XPCShellTestsTests(unittest.TestCase):
     """
     Yes, these are unit tests for a unit test harness.
     """
-    def setUp(self):
-        self.log = StringIO()
-        self.tempdir = tempfile.mkdtemp()
+    def __init__(self, name):
+        super(XPCShellTestsTests, self).__init__(name)
+        from buildconfig import substs
+        from mozbuild.base import MozbuildObject
+        os.environ.pop('MOZ_OBJDIR', None)
+        self.build_obj = MozbuildObject.from_environment()
+
+        objdir = self.build_obj.topobjdir.encode("utf-8")
+        self.testing_modules = os.path.join(objdir, '_tests', 'modules')
+
+        if mozinfo.isMac:
+            self.xpcshellBin = os.path.join(objdir, "dist", substs['MOZ_MACBUNDLE_NAME'],
+                                            "Contents", "MacOS", "xpcshell")
+        else:
+            self.xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
+        if sys.platform == "win32":
+            self.xpcshellBin += ".exe"
         self.utility_path = os.path.join(objdir, 'dist', 'bin')
+        self.symbols_path = None
+        candidate_path = os.path.join(self.build_obj.distdir, 'crashreporter-symbols')
+        if (os.path.isdir(candidate_path)):
+            self.symbols_path = candidate_path
+
+    def setUp(self):
+        self.log = six.StringIO()
+        self.tempdir = tempfile.mkdtemp()
         logger = structured.commandline.setup_logging("selftest%s" % id(self),
                                                       {},
                                                       {"tbpl": self.log})
         self.x = XPCShellTests(logger)
         self.x.harness_timeout = 30 if not mozinfo.info["ccov"] else 60
-        self.symbols_path = None
-        candidate_path = os.path.join(build_obj.distdir, 'crashreporter-symbols')
-        if (os.path.isdir(candidate_path)):
-            self.symbols_path = candidate_path
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
@@ -552,7 +553,7 @@ prefs =
         returns |expected|.
         """
         kwargs = {}
-        kwargs['xpcshell'] = xpcshellBin
+        kwargs['xpcshell'] = self.xpcshellBin
         kwargs['symbolsPath'] = self.symbols_path
         kwargs['manifest'] = self.manifest
         kwargs['mozInfo'] = mozinfo.info
@@ -560,7 +561,7 @@ prefs =
         kwargs['verbose'] = verbose
         kwargs['headless'] = headless
         kwargs['sequential'] = True
-        kwargs['testingModulesDir'] = os.path.join(objdir, '_tests', 'modules')
+        kwargs['testingModulesDir'] = self.testing_modules
         kwargs['utility_path'] = self.utility_path
         self.assertEquals(expected,
                           self.x.runTests(kwargs),
@@ -1019,7 +1020,7 @@ add_test({
         self.assertEquals(1, self.x.testCount)
         self.assertEquals(0, self.x.passCount)
         self.assertEquals(1, self.x.failCount)
-        if substs.get('MOZ_CRASHREPORTER'):
+        if mozinfo.info.get('crashreporter'):
             self.assertInLog("\nPROCESS-CRASH")
 
     def testLogCorrectFileName(self):
@@ -1182,7 +1183,7 @@ add_test({
             self.assertTestResult(True)
         except Exception as ex:
             raised = True
-            self.assertEquals(ex.message[0:9], "head file")
+            self.assertEquals(str(ex)[0:9], "head file")
 
         self.assertTrue(raised)
 
@@ -1495,4 +1496,6 @@ add_test({
 
 
 if __name__ == "__main__":
+    import mozunit
+    mozinfo.find_and_update_from_json()
     mozunit.main()

testing/xpcshell/xpcshellcommandline.py

@@ -153,6 +153,10 @@ def add_common_arguments(parser):
                         dest="headless",
                         help="Enable headless mode by default for tests which don't specify "
                              "whether to use headless mode")
+    parser.add_argument("--self-test",
+                        action="store_true", default=False,
+                        dest="self_test",
+                        help="Run self tests")
 
 
 def add_remote_arguments(parser):