Bug 1301197 - Add xpcshell code coverage to linux64-jsdcov and the 'coverage' mozinfo flag. r=jmaher

This patch makes it possible to collect code coverage for xpcshell tests using the linux64-jsdcov build. It also enables the use of a 'coverage' flag to disable tests when they are instrumented with the js debugger for code coverage. Lastly, it uses the 'coverage' flag to disable certain tests.

MozReview-Commit-ID: 97VFkJmlwQn

--HG--
extra : rebase_source : 26c841f5a68f927889c0903e701bfde4b7ca84ac

devtools/server/tests/unit/xpcshell.ini

@@ -91,6 +91,7 @@ support-files =
 [test_promises_actor_attach.js]
 [test_promises_actor_exist.js]
 [test_promises_actor_list_promises.js]
+skip-if = coverage # bug 1336670
 [test_promises_actor_onnewpromise.js]
 [test_promises_actor_onpromisesettled.js]
 [test_promises_client_getdependentpromises.js]

dom/indexedDB/test/unit/xpcshell-parent-process.ini

@@ -65,5 +65,8 @@ skip-if = true
 skip-if = os == "android" && processor == "x86"
 [test_view_put_get_values.js]
 [test_wasm_getAll.js]
+skip-if = coverage # bug 1336727
 [test_wasm_put_get_values.js]
+skip-if = coverage # bug 1336727
 [test_wasm_recompile.js]
+skip-if = coverage # bug 1336727

security/manager/ssl/tests/unit/xpcshell-smartcards.ini

@@ -9,6 +9,7 @@ support-files =
 [test_pkcs11_module.js]
 [test_pkcs11_no_events_after_removal.js]
 [test_pkcs11_safe_mode.js]
+skip-if = coverage # bug 1336728
 [test_pkcs11_slot.js]
 [test_pkcs11_token.js]
 [test_pkcs11_tokenDB.js]

taskcluster/ci/test/test-sets.yml

@@ -83,6 +83,7 @@ ccov-code-coverage-tests:
 jsdcov-code-coverage-tests:
     - mochitest-browser-chrome
     - mochitest-devtools-chrome
+    - xpcshell
 
 ##
 # Test sets still being greened up in various ways

taskcluster/ci/test/tests.yml

@@ -1225,8 +1225,15 @@ web-platform-tests-wdspec:
 
 xpcshell:
     description: "xpcshell test run"
-    suite: xpcshell
+    suite:
+        by-test-platform:
+            linux64-jsdcov/opt: xpcshell-coverage
+            default: xpcshell
     treeherder-symbol: tc-X()
+    run-on-projects:
+        by-test-platform:
+            linux64-jsdcov/opt: []
+            default: ['all']
     chunks:
         by-test-platform:
             linux64/debug: 10
@@ -1270,4 +1277,8 @@ xpcshell:
                             - unittests/linux_unittest.py
                             - remove_executables.py
                 extra-options:
+                    by-test-platform:
+                        linux64-jsdcov/opt:
+                            - --xpcshell-suite=xpcshell-coverage
+                        default:
                             - --xpcshell-suite=xpcshell

testing/mozharness/configs/unittests/linux_unittest.py

@@ -245,7 +245,8 @@ config = {
         },
         "xpcshell-coverage": {
             "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
-                        "--manifest=tests/xpcshell/tests/xpcshell.ini"],
+                        "--manifest=tests/xpcshell/tests/xpcshell.ini",
+                        "--sequential"],
             "tests": []
         },
     },

testing/xpcshell/head.js

@@ -552,9 +552,6 @@ function _execute_test() {
     // has already been logged so there is no need to log it again. It's
     // possible that this will mask an NS_ERROR_ABORT that happens after a
     // do_check failure though.
-    if (coverageCollector != null) {
-      coverageCollector.recordTestCoverage(_TEST_FILE[0]);
-    }
 
     if (!_quit || e != Components.results.NS_ERROR_ABORT) {
       let extra = {};
@@ -572,11 +569,11 @@ function _execute_test() {
       }
       _testLogger.error(message, extra);
     }
-  }
-
+  } finally {
     if (coverageCollector != null) {
       coverageCollector.finalize();
     }
+  }
 
   // Execute all of our cleanup functions.
   let reportCleanupError = function(ex) {

testing/xpcshell/runxpcshelltests.py

@@ -1213,6 +1213,10 @@ class XPCShellTests(object):
 
         mozinfo.update(self.mozInfo)
 
+        # Add a flag to mozinfo to indicate that code coverage is enabled.
+        if self.jscovdir:
+            mozinfo.update({"coverage": True})
+
         self.stack_fixer_function = None
         if self.utility_path and os.path.exists(self.utility_path):
             self.stack_fixer_function = get_stack_fixer_function(self.utility_path, self.symbolsPath)

toolkit/components/jsdownloads/test/unit/xpcshell.ini

@@ -15,4 +15,5 @@ support-files =
 [test_Downloads.js]
 [test_DownloadStore.js]
 [test_PrivateTemp.js]
-skip-if = os != 'linux'
+# coverage flag is for bug 1336730
+skip-if = (os != 'linux' || coverage)

toolkit/components/osfile/tests/xpcshell/xpcshell.ini

@@ -47,4 +47,5 @@ requesttimeoutfactor = 4
 [test_reset.js]
 [test_shutdown.js]
 [test_telemetry.js]
+skip-if = coverage # bug 1336672
 [test_unique.js]

toolkit/components/startup/tests/unit/xpcshell.ini

@@ -3,3 +3,4 @@ head = head_startup.js
 skip-if = toolkit == 'android'
 
 [test_startup_crash.js]
+skip-if = coverage # bug 1336673

tools/profiler/tests/xpcshell.ini

@@ -15,3 +15,4 @@ skip-if = !debug
 [test_enterjit_osr_enabling.js]
 skip-if = !debug
 [test_asm.js]
+skip-if = coverage # bug 1336674