Bug 1367763 - Run grcov after code coverage collection on linux64-ccov. r=jmaher,marco

This patch adds grcov to linux64-ccov. Grcov is then used after code coverage collection to process the .gcno and .gcda files into a .info (LCOV-style) file. It also uses the 'linked-files-map.json' to rewrite symbolic links to the path that they point to. The .info file that is produced is called 'grcov_lcov_output_stdout' and is stored in the zip file 'code-coverage-grcov.zip'.

MozReview-Commit-ID: 5WhD289N6Fs

--HG--
extra : rebase_source : 9ed60bd96f5e9a479771357e38b0b3b16abd351e

testing/config/tooltool-manifests/linux64/ccov.manifest

@@ -0,0 +1,9 @@
+[
+  {
+    "size": 752315,
+    "digest": "8e86d7ebbb11758a385bb2af65b3f6444a5c6a1468c35a77f60d253c82affa007c61dc9b73025f8a146678a3ce1b8d050064bccf5efa33357588db10dac44150",
+    "algorithm": "sha512",
+    "filename": "grcov-linux-standalone-x86_64.tar.bz2",
+    "unpack": false
+  }
+]

testing/mozharness/mozharness/mozilla/testing/codecoverage.py

@@ -11,6 +11,7 @@ from mozharness.base.script import (
     PreScriptAction,
     PostScriptAction,
 )
+from mozharness.mozilla.tooltool import TooltoolMixin
 
 code_coverage_config_options = [
     [["--code-coverage"],
@@ -60,12 +61,32 @@ class CodeCoverageMixin(object):
     def _set_gcov_prefix(self, action):
         if not self.code_coverage_enabled:
             return
+        # Set the GCOV directory.
         self.gcov_dir = tempfile.mkdtemp()
         os.environ['GCOV_PREFIX'] = self.gcov_dir
-        # Set JSVM directory also.
+
+        # Set JSVM directory.
         self.jsvm_dir = tempfile.mkdtemp()
         os.environ['JS_CODE_COVERAGE_OUTPUT_DIR'] = self.jsvm_dir
 
+        # Install grcov on the test machine
+        # Get the path to the build machines gcno files.
+        self.url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip')
+        dirs = self.query_abs_dirs()
+
+        # Create the grcov directory, get the tooltool manifest, and finally
+        # download and unpack the grcov binary.
+        self.grcov_dir = tempfile.mkdtemp()
+        manifest = os.path.join(dirs.get('abs_test_install_dir', os.path.join(dirs['abs_work_dir'], 'tests')), \
+            'config/tooltool-manifests/linux64/ccov.manifest')
+
+        tooltool_path = self._fetch_tooltool_py()
+        cmd = [tooltool_path, '--url', 'https://api.pub.build.mozilla.org/tooltool/', 'fetch', \
+            '-m', manifest, '-o', '-c', '/home/worker/tooltool-cache']
+        self.run_command(cmd, cwd=self.grcov_dir)
+        self.run_command(['tar', '-jxvf', os.path.join(self.grcov_dir, 'grcov-linux-standalone-x86_64.tar.bz2'), \
+            '-C', self.grcov_dir], cwd=self.grcov_dir)
+
     @PostScriptAction('run-tests')
     def _package_coverage_data(self, action, success=None):
         if not self.code_coverage_enabled:
@@ -92,17 +113,38 @@ class CodeCoverageMixin(object):
 
             # Package GCOV coverage data.
             dirs = self.query_abs_dirs()
-            file_path = os.path.join(
+            file_path_gcda = os.path.join(
                 dirs['abs_blob_upload_dir'], 'code-coverage-gcda.zip')
-            command = ['zip', '-r', file_path, '.']
+            command = ['zip', '-r', file_path_gcda, '.']
             self.run_command(command, cwd=rel_topsrcdir)
 
             # Package JSVM coverage data.
             dirs = self.query_abs_dirs()
-            file_path = os.path.join(
+            file_path_jsvm = os.path.join(
                 dirs['abs_blob_upload_dir'], 'code-coverage-jsvm.zip')
-            command = ['zip', '-r', file_path, '.']
+            command = ['zip', '-r', file_path_jsvm, '.']
             self.run_command(command, cwd=self.jsvm_dir)
 
+            # GRCOV post-processing
+            # Download the gcno fom the build machine.
+            self.download_file(self.url_to_gcno, file_name=None, parent_dir=self.grcov_dir)
+
+            # Run grcov on the zipped .gcno and .gcda files.
+            grcov_command = [os.path.join(self.grcov_dir, 'grcov'), '-t', 'lcov' , '-p', \
+                             '/home/worker/workspace/build/src/', '-z', \
+                             os.path.join(self.grcov_dir, 'target.code-coverage-gcno.zip'), file_path_gcda]
+
+            # 'grcov_output' will be a tuple, the first variable is the path to the lcov output,
+            # the other is the path to the standard error output.
+            grcov_output = self.get_output_from_command(grcov_command, cwd=self.grcov_dir, \
+                silent=True, tmpfile_base_path=os.path.join(self.grcov_dir, 'grcov_lcov_output'), \
+                save_tmpfiles=True, return_type='files')
+            new_output_name = grcov_output[0] + '.info'
+            os.rename(grcov_output[0], new_output_name)
+
+            # Zip the grcov output and upload it.
+            command = ['zip', os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip'), new_output_name]
+            self.run_command(command, cwd=self.grcov_dir)
         shutil.rmtree(self.gcov_dir)
         shutil.rmtree(self.jsvm_dir)
+        shutil.rmtree(self.grcov_dir)