Bug 1472236 - Add grcov code coverage builds by using a fetch task. r=dustin,marco

MozReview-Commit-ID: 3PZMNDRqxRM

--HG--
extra : rebase_source : 31c0104bf03d702db5766b40b145a9a342d0e8b2

taskcluster/ci/fetch/toolchains.yml

@@ -193,3 +193,33 @@ mpfr-3.1.5:
     gpg-signature:
       sig-url: "{url}.asc"
       key-path: build/unix/build-gcc/07F3DBBECC1A39605078094D980C197698C3739D.key
+
+grcov-linux-x86_64:
+  description: grcov binary release
+  treeherder:
+    symbol: grcov-linux-x86_64
+  run:
+    using: fetch-url
+    url: https://github.com/mozilla/grcov/releases/download/v0.2.3/grcov-linux-x86_64.tar.bz2
+    sha256: 5ed6ee891e0a1cbf1d84e27c615ec1f844c89f517f29cbec97379ded7c0894cd
+    size: 1858593
+
+grcov-osx-x86_64:
+  description: grcov binary release
+  treeherder:
+    symbol: grcov-osx-x86_64
+  run:
+    using: fetch-url
+    url: https://github.com/mozilla/grcov/releases/download/v0.2.3/grcov-osx-x86_64.tar.bz2
+    sha256: 3db4ef6c0dfaf35d39c9cacf19c60c482dae9413b5fe3bf343fa26667451c893
+    size: 919638
+
+grcov-win-i686:
+  description: grcov binary release
+  treeherder:
+    symbol: grcov-win-i686
+  run:
+    using: fetch-url
+    url: https://github.com/mozilla/grcov/releases/download/v0.2.3/grcov-win-i686.tar.bz2
+    sha256: 87c3c22cd1bb99d9c41b6a6f9e20b6c655a580021776f1a532297f5e8315ad9b
+    size: 919079

taskcluster/ci/test/kind.yml

@@ -3,9 +3,11 @@ loader: taskgraph.loader.test:loader
 kind-dependencies:
     - build
     - build-signing
+    - fetch
 
 transforms:
     - taskgraph.transforms.tests:transforms
+    - taskgraph.transforms.use_fetches:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.coalesce:transforms
     - taskgraph.transforms.task:transforms

taskcluster/taskgraph/loader/test.py

@@ -18,12 +18,6 @@ def loader(kind, path, config, params, loaded_tasks):
     Generate tasks implementing Gecko tests.
     """
 
-    # the kind on which this one depends
-    if len(config.get('kind-dependencies', [])) != 2:
-        raise Exception(
-            'Test kinds must have exactly 2 items in kind-dependencies'
-        )
-
     builds_by_platform = get_builds_by_platform(dep_kind='build', loaded_tasks=loaded_tasks)
     signed_builds_by_platform = get_builds_by_platform(
         dep_kind='build-signing', loaded_tasks=loaded_tasks

taskcluster/taskgraph/transforms/job/mozharness_test.py

@@ -95,7 +95,8 @@ def mozharness_test_on_docker(config, job, taskdesc):
         'mount-point': "{workdir}/workspace".format(**run),
     }]
 
-    env = worker['env'] = {
+    env = worker.setdefault('env', {})
+    env.update({
         'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
         'MOZHARNESS_SCRIPT': mozharness['script'],
         'MOZILLA_BUILD_URL': {'task-reference': installer_url},
@@ -103,7 +104,7 @@ def mozharness_test_on_docker(config, job, taskdesc):
         'NEED_WINDOW_MANAGER': 'true',
         'ENABLE_E10S': str(bool(test.get('e10s'))).lower(),
         'MOZ_AUTOMATION': '1',
-    }
+    })
 
     if mozharness.get('mochitest-flavor'):
         env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']

taskcluster/taskgraph/transforms/tests.py

@@ -733,6 +733,14 @@ def enable_code_coverage(config, tests):
             test.pop('when', None)
             test['optimization'] = None
 
+            # Add a fetch task for the grcov binary.
+            if 'linux' in test['build-platform']:
+                test['fetches'] = ['grcov-linux-x86_64']
+            elif 'osx' in test['build-platform']:
+                test['fetches'] = ['grcov-osx-x86_64']
+            elif 'win' in test['build-platform']:
+                test['fetches'] = ['grcov-win-i686']
+
             if 'talos' in test['test-name']:
                 test['max-run-time'] = 7200
                 if 'linux' in test['build-platform']:
@@ -1058,6 +1066,8 @@ def make_job_description(config, tests):
         run['test'] = test
 
         jobdesc['worker-type'] = test.pop('worker-type')
+        if test.get('fetches'):
+            jobdesc['fetches'] = test.pop('fetches')
 
         yield jobdesc
 

testing/mozharness/mozharness/mozilla/testing/codecoverage.py

@@ -111,34 +111,20 @@ class CodeCoverageMixin(SingleTestMixin):
         # Get the path to the build machines gcno files.
         self.url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip')
         self.url_to_chrome_map = self.query_build_dir_url('chrome-map.json')
-        dirs = self.query_abs_dirs()
 
-        # Create the grcov directory, get the tooltool manifest, and finally
-        # download and unpack the grcov binary.
+        # Create the grcov directory, then download it.
+        # TODO: use the fetch-content script to download artifacts.
         self.grcov_dir = tempfile.mkdtemp()
+        ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
+        for word in os.getenv('MOZ_FETCHES').split():
+            artifact, task = word.split('@', 1)
+            filename = os.path.basename(artifact)
+            url = ARTIFACT_URL.format(artifact=artifact, task=task)
+            self.download_file(url, parent_dir=self.grcov_dir)
 
-        if mozinfo.os == 'linux':
-            platform = 'linux64'
-            tar_file = 'grcov-linux-x86_64.tar.bz2'
-        elif mozinfo.os == 'win':
-            platform = 'win32'
-            tar_file = 'grcov-win-i686.tar.bz2'
-        elif mozinfo.os == 'mac':
-            platform = 'macosx64'
-            tar_file = 'grcov-osx-x86_64.tar.bz2'
-
-        manifest = os.path.join(dirs.get('abs_test_install_dir',
-                                         os.path.join(dirs['abs_work_dir'], 'tests')),
-                                'config/tooltool-manifests/%s/ccov.manifest' % platform)
-
-        self.tooltool_fetch(
-            manifest=manifest,
-            output_dir=self.grcov_dir,
-            cache=self.config.get('tooltool_cache')
-        )
-
-        with tarfile.open(os.path.join(self.grcov_dir, tar_file)) as tar:
-            tar.extractall(self.grcov_dir)
+            with tarfile.open(os.path.join(self.grcov_dir, filename), 'r') as tar:
+                tar.extractall(self.grcov_dir)
+            os.remove(os.path.join(self.grcov_dir, filename))
 
         # Download the gcno archive from the build machine.
         self.download_file(self.url_to_gcno, parent_dir=self.grcov_dir)