Bug 1468812 - [ci] Support MOZ_FETCHES and fetch-content in run-task r=gps

This removes the 'use-artifacts' mechanism in favour of fetches. There are a
few pieces here that need to land atomically:

1. Remove use-artifact related code
2. Call 'fetch-content' from the run-task script
3. Convert existing tasks on top of fetches (jsshell, python unittest)
4. Stop calling 'fetch-content' from toolchain setup tasks (as this now gets handled in run-task)

Depends on D2166.

Differential Revision: https://phabricator.services.mozilla.com/D2167

--HG--
extra : moz-landing-system : lando

layout/tools/reftest/selftest/conftest.py

@@ -49,8 +49,8 @@ def runtests(setup_test_harness, binary, parser):
             'utilityPath': os.path.join(package_root, 'bin'),
         })
 
-        if 'USE_ARTIFACT_PATH' in os.environ:
-            options['sandboxReadWhitelist'].append(os.environ['USE_ARTIFACT_PATH'])
+        if 'MOZ_FETCHES_DIR' in os.environ:
+            options['sandboxReadWhitelist'].append(os.environ['MOZ_FETCHES_DIR'])
     else:
         options.update({
             'extraProfileFiles': [os.path.join(build.topobjdir, 'dist', 'plugins')],

taskcluster/ci/source-test/jsshell.yml

@@ -9,50 +9,36 @@ job-defaults:
             linux64.*:
                 env:
                     SHELL: /bin/bash
+                    JSSHELL: /home/cltbld/fetches/js
                 max-run-time: 1800
     treeherder:
         kind: test
         tier: 2
     run:
-        using: run-task
-        use-artifacts:
-            build:
-                - target.jsshell.zip
+        using: mach
         workdir: /home/cltbld
     run-on-projects: ['mozilla-central', 'try']
+    fetches:
+        build:
+            - target.jsshell.zip
 
 bench-ares6:
     description: Ares6 JavaScript shell benchmark suite
     treeherder:
         symbol: js-bench(ares6)
     run:
-        command: >
-            cd $USE_ARTIFACT_PATH/build &&
-            unzip -qo -d jsshell target.jsshell.zip &&
-            export JSSHELL=$USE_ARTIFACT_PATH/build/jsshell/js &&
-            cd $GECKO_PATH &&
-            ./mach jsshell-bench --binary $JSSHELL --perfherder ares6
+        mach: jsshell-bench --binary $JSSHELL --perfherder ares6
 
 bench-sixspeed:
     description: Six-Speed JavaScript shell benchmark suite
     treeherder:
         symbol: js-bench(6speed)
     run:
-        command: >
-            cd $USE_ARTIFACT_PATH/build &&
-            unzip -qo -d jsshell target.jsshell.zip &&
-            export JSSHELL=$USE_ARTIFACT_PATH/build/jsshell/js &&
-            cd $GECKO_PATH &&
-            ./mach jsshell-bench --binary $JSSHELL --perfherder six-speed
+        mach: jsshell-bench --binary $JSSHELL --perfherder six-speed
 
 bench-asmjsapps:
     description: asm.js Apps shell benchmark suite
     treeherder:
         symbol: js-bench(asm.js-apps)
     run:
-        command: >
-            cd $USE_ARTIFACT_PATH/build &&
-            unzip -qo -d jsshell target.jsshell.zip &&
-            export JSSHELL=$USE_ARTIFACT_PATH/build/jsshell/js &&
-            cd $GECKO_PATH &&
-            ./mach jsshell-bench --binary $JSSHELL --perfherder asmjs-apps
+        mach: jsshell-bench --binary $JSSHELL --perfherder asmjs-apps

taskcluster/ci/source-test/python.yml

@@ -64,22 +64,18 @@ mochitest-harness:
                 max-run-time: 3600
     run:
         using: run-task
-        use-artifacts:
-            build:
-                - target.tar.bz2
-                - target.common.tests.zip
-                - target.mochitest.tests.zip
         command: >
             source /builds/worker/scripts/xvfb.sh &&
             start_xvfb '1600x1200x24' 0 &&
-            cd $USE_ARTIFACT_PATH/build &&
-            tar -xf target.tar.bz2 &&
-            unzip -q -d tests target.common.tests.zip &&
-            unzip -q -d tests target.mochitest.tests.zip &&
-            export GECKO_BINARY_PATH=$USE_ARTIFACT_PATH/build/firefox/firefox &&
-            export TEST_HARNESS_ROOT=$USE_ARTIFACT_PATH/build/tests &&
+            export GECKO_BINARY_PATH=$MOZ_FETCHES_DIR/firefox/firefox &&
+            export TEST_HARNESS_ROOT=$MOZ_FETCHES_DIR/tests &&
             cd /builds/worker/checkouts/gecko &&
             ./mach python-test --python 2 --subsuite mochitest
+    fetches:
+        build:
+            - target.tar.bz2
+            - target.common.tests.zip>tests
+            - target.mochitest.tests.zip>tests
     when:
         files-changed:
             - 'testing/mochitest/**'
@@ -181,22 +177,18 @@ reftest-harness:
                 max-run-time: 3600
     run:
         using: run-task
-        use-artifacts:
-            build:
-                - target.tar.bz2
-                - target.common.tests.zip
-                - target.reftest.tests.zip
         command: >
             source /builds/worker/scripts/xvfb.sh &&
             start_xvfb '1600x1200x24' 0 &&
-            cd $USE_ARTIFACT_PATH/build &&
-            tar -xf target.tar.bz2 &&
-            unzip -q -d tests target.common.tests.zip &&
-            unzip -q -d tests target.reftest.tests.zip &&
-            export GECKO_BINARY_PATH=$USE_ARTIFACT_PATH/build/firefox/firefox &&
-            export TEST_HARNESS_ROOT=$USE_ARTIFACT_PATH/build/tests &&
+            export GECKO_BINARY_PATH=$MOZ_FETCHES_DIR/firefox/firefox &&
+            export TEST_HARNESS_ROOT=$MOZ_FETCHES_DIR/tests &&
             cd /builds/worker/checkouts/gecko &&
             ./mach python-test --python 2 --subsuite reftest
+    fetches:
+        build:
+            - target.tar.bz2
+            - target.common.tests.zip>tests
+            - target.reftest.tests.zip>tests
     when:
         files-changed:
             - 'layout/tools/reftest/**'

taskcluster/docker/desktop1604-test/Dockerfile

@@ -47,6 +47,9 @@ ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh
 # %include taskcluster/scripts/run-task
 ADD topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task
 
+# %include taskcluster/scripts/misc/fetch-content
+ADD topsrcdir/taskcluster/scripts/misc/fetch-content /builds/worker/bin/fetch-content
+
 # %include taskcluster/scripts/tester/test-linux.sh
 ADD topsrcdir/taskcluster/scripts/tester/test-linux.sh /builds/worker/bin/test-linux.sh
 

taskcluster/scripts/misc/build-gcc-4.9-linux.sh

@@ -17,8 +17,6 @@ gcc_ext=bz2
 binutils_version=2.25.1
 binutils_ext=bz2
 
-$HOME_DIR/src/taskcluster/scripts/misc/fetch-content task-artifacts -d $root_dir $MOZ_FETCHES
-
 pushd $root_dir/gcc-$gcc_version
 ln -sf ../cloog-0.18.1 cloog
 ln -sf ../gmp-5.1.3 gmp

taskcluster/scripts/misc/build-gcc-6-linux.sh

@@ -17,8 +17,6 @@ gcc_ext=xz
 binutils_version=2.28.1
 binutils_ext=xz
 
-$HOME_DIR/src/taskcluster/scripts/misc/fetch-content task-artifacts -d $root_dir $MOZ_FETCHES
-
 pushd $root_dir/gcc-$gcc_version
 ln -sf ../gmp-5.1.3 gmp
 ln -sf ../isl-0.15 isl

taskcluster/scripts/misc/build-gcc-7-linux.sh

@@ -17,8 +17,6 @@ gcc_ext=xz
 binutils_version=2.28.1
 binutils_ext=xz
 
-$HOME_DIR/src/taskcluster/scripts/misc/fetch-content task-artifacts -d $root_dir $MOZ_FETCHES
-
 pushd $root_dir/gcc-$gcc_version
 ln -sf ../gmp-6.1.0 gmp
 ln -sf ../isl-0.16.1 isl

taskcluster/scripts/misc/build-gcc-mingw32.sh

@@ -20,8 +20,6 @@ binutils_ext=bz2
 binutils_configure_flags="--target=i686-w64-mingw32"
 mingw_version=bcf1f29d6dc80b6025b416bef104d2314fa9be57
 
-$HOME_DIR/src/taskcluster/scripts/misc/fetch-content task-artifacts -d $root_dir $MOZ_FETCHES
-
 pushd $root_dir/gcc-$gcc_version
 ln -sf ../gmp-5.1.3 gmp
 ln -sf ../isl-0.15 isl

taskcluster/scripts/misc/build-gcc-sixgill-plugin-linux.sh

@@ -27,8 +27,6 @@ sixgill_repo=https://hg.mozilla.org/users/sfink_mozilla.com/sixgill
 
 . $data_dir/build-gcc.sh
 
-$HOME_DIR/src/taskcluster/scripts/misc/fetch-content task-artifacts -d $root_dir $MOZ_FETCHES
-
 pushd $root_dir/gcc-$gcc_version
 ln -sf ../binutils-2.28.1 binutils
 ln -sf ../gmp-5.1.3 gmp

taskcluster/scripts/run-task

@@ -73,6 +73,18 @@ A lesser possibility is that you stumbled upon a TaskCluster platform bug
 where it fails to use new volumes for tasks.
 '''
 
+
+FETCH_CONTENT_NOT_FOUND = '''
+error: fetch-content script not found
+
+The script at `taskcluster/scripts/misc/fetch-content` could not be
+detected in the current environment.
+
+If this task clones gecko, make sure the GECKO_PATH environment variable
+is set to proper location. Otherwise, the script may need to be mounted
+or added to the task's docker image then added to the PATH.
+'''
+
 # The exit code to use when caches should be purged and the task retried.
 # This is EX_OSFILE (from sysexits.h):
 #     Some system file  does not exist, cannot be opened, or has some
@@ -468,6 +480,23 @@ def vcs_checkout(source_repo, dest, store_path,
     return revision
 
 
+def fetch_artifacts(fetches):
+    print_line(b'fetches', b'fetching artifacts\n')
+
+    fetch_content = shutil.which('fetch-content')
+    if not fetch_content and os.environ.get('GECKO_PATH'):
+        fetch_content = os.path.join(os.environ['GECKO_PATH'], 'taskcluster',
+                                     'scripts', 'misc', 'fetch-content')
+
+    if not fetch_content or not os.path.isfile(fetch_content):
+        print(FETCH_CONTENT_NOT_FOUND)
+        sys.exit(1)
+
+    cmd = [fetch_content, 'task-artifacts'] + fetches.split()
+    subprocess.run(cmd, check=True, env=os.environ)
+    print_line(b'fetches', b'finished fetching artifacts\n')
+
+
 def main(args):
     print_line(b'setup', b'run-task started in %s\n' % os.getcwd().encode('utf-8'))
     running_as_root = IS_POSIX and os.getuid() == 0
@@ -708,32 +737,18 @@ def main(args):
         print('task should be defined in terms of non-symbolic revision')
         return 1
 
-    use_artifact_path = os.environ.get('USE_ARTIFACT_PATH')
-
-    def prepare_use_artifact(key, url):
-        print_line(b'setup', b'fetching artifact from %s\n' % url.encode('utf-8'))
-        path = os.path.join(use_artifact_path, key)
-        if not os.path.isdir(path):
-            os.makedirs(path)
-
-        url = url.rstrip('/')
-        path = os.path.join(path, os.path.basename(url))
-        response = urllib.request.urlopen(url)
-        with open(path, 'wb') as fh:
-            fh.write(response.read())
-
-    use_artifacts = os.environ.get('USE_ARTIFACT_URLS')
-    if use_artifacts:
-        use_artifacts = json.loads(use_artifacts)
-        for key, urls in use_artifacts.items():
-            for url in urls:
-                prepare_use_artifact(key, url)
+    fetches = os.environ.get('MOZ_FETCHES')
+    if fetches:
+        fetch_artifacts(fetches)
 
     try:
         return run_and_prefix_output(b'task', task_args)
     finally:
-        if use_artifact_path and os.path.isdir(use_artifact_path):
-            shutil.rmtree(use_artifact_path)
+        fetches_dir = os.environ.get('MOZ_FETCHES_DIR')
+        if fetches_dir and os.path.isdir(fetches_dir):
+            print_line(b'fetches', b'removing %s\n' % fetches_dir.encode('utf-8'))
+            shutil.rmtree(fetches_dir)
+            print_line(b'fetches', b'finished\n')
 
 
 if __name__ == '__main__':

taskcluster/taskgraph/transforms/job/__init__.py

@@ -197,7 +197,7 @@ def use_fetches(config, jobs):
         env['MOZ_FETCHES'] = {'task-reference': ' '.join(job_fetches)}
 
         workdir = job['run'].get('workdir', '/builds/worker')
-        env['MOZ_FETCHES_DIR'] = '{}/fetches'.format(workdir)
+        env.setdefault('MOZ_FETCHES_DIR', '{}/fetches'.format(workdir))
         yield job
 
 

taskcluster/taskgraph/transforms/job/common.py

@@ -9,8 +9,7 @@ consistency.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import json
-from taskgraph.util.taskcluster import get_artifact_url, get_artifact_prefix
+from taskgraph.util.taskcluster import get_artifact_prefix
 
 SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
 
@@ -193,26 +192,3 @@ def docker_worker_add_tooltool(config, job, taskdesc, internal=False):
         taskdesc['scopes'].extend([
             'docker-worker:relengapi-proxy:tooltool.download.internal',
         ])
-
-
-def support_use_artifacts(config, job, taskdesc, use_artifacts):
-    """Set a JSON object of artifact URLs in an environment variable.
-
-    This will tell the run-task script to download the artifacts.
-    """
-    urls = {}
-    prefix = get_artifact_prefix(taskdesc)
-    for kind, artifacts in use_artifacts.items():
-        if kind not in taskdesc['dependencies']:
-            raise Exception("{label} can't use '{kind}' artifacts because it has no '{kind}' "
-                            "dependency!".format(label=job['label'], kind=kind))
-        task_id = '<{}>'.format(kind)
-        urls[kind] = []
-
-        for artifact in artifacts:
-            path = '/'.join([prefix, artifact])
-            urls[kind].append(get_artifact_url(task_id, path))
-
-    env = taskdesc['worker'].setdefault('env', {})
-    env['USE_ARTIFACT_URLS'] = {'task-reference': json.dumps(urls)}
-    env['USE_ARTIFACT_PATH'] = '{workdir}/use-artifacts'.format(**job['run'])

taskcluster/taskgraph/transforms/job/mozharness.py

@@ -149,6 +149,7 @@ def mozharness_on_docker_worker_setup(config, job, taskdesc):
 
     env = worker.setdefault('env', {})
     env.update({
+        'GECKO_PATH': '{workdir}/workspace/build/src'.format(**run),
         'MOZHARNESS_CONFIG': ' '.join(run['config']),
         'MOZHARNESS_SCRIPT': run['script'],
         'MH_BRANCH': config.params['project'],
@@ -204,7 +205,7 @@ def mozharness_on_docker_worker_setup(config, job, taskdesc):
 
     command = [
         '{workdir}/bin/run-task'.format(**run),
-        '--vcs-checkout', '{workdir}/workspace/build/src'.format(**run),
+        '--vcs-checkout', env['GECKO_PATH'],
         '--tools-checkout', '{workdir}/workspace/build/tools'.format(**run),
     ]
     if run['comm-checkout']:

taskcluster/taskgraph/transforms/job/run_task.py

@@ -9,7 +9,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.util.schema import Schema
-from taskgraph.transforms.job.common import support_use_artifacts, support_vcs_checkout
+from taskgraph.transforms.job.common import support_vcs_checkout
 from voluptuous import Required, Any
 
 run_task_schema = Schema({
@@ -30,14 +30,6 @@ run_task_schema = Schema({
     # gecko checkout
     Required('comm-checkout'): bool,
 
-    # maps a dependency to a list of artifact names to use from that dependency.
-    # E.g: {"build": ["target.tar.bz2"]}
-    # In the above example, the artifact would be downloaded to:
-    # $USE_ARTIFACT_PATH/build/target.tar.bz2
-    Required('use-artifacts'): Any(None, {
-        basestring: [basestring],
-    }),
-
     # The command arguments to pass to the `run-task` script, after the
     # checkout arguments.  If a list, it will be passed directly; otherwise
     # it will be included in a single argument to `bash -cx`.
@@ -54,9 +46,6 @@ def common_setup(config, job, taskdesc):
         support_vcs_checkout(config, job, taskdesc,
                              sparse=bool(run['sparse-profile']))
 
-    if run['use-artifacts']:
-        support_use_artifacts(config, job, taskdesc, run['use-artifacts'])
-
     taskdesc['worker'].setdefault('env', {})['MOZ_SCM_LEVEL'] = config.params['level']
 
 
@@ -76,7 +65,6 @@ defaults = {
     'checkout': True,
     'comm-checkout': False,
     'sparse-profile': None,
-    'use-artifacts': None,
 }
 
 

taskcluster/taskgraph/transforms/job/toolchain.py

@@ -131,12 +131,18 @@ def docker_worker_toolchain(config, job, taskdesc):
 
     support_vcs_checkout(config, job, taskdesc, sparse=True)
 
+    # Toolchain checkouts don't live under {workdir}/checkouts
+    workspace = '{workdir}/workspace/build'.format(**run)
+    gecko_path = '{}/src'.format(workspace)
+
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
         'TOOLS_DISABLE': 'true',
         'MOZ_AUTOMATION': '1',
+        'MOZ_FETCHES_DIR': workspace,
+        'GECKO_PATH': gecko_path,
     })
 
     if run['tooltool-downloads']:
@@ -145,7 +151,7 @@ def docker_worker_toolchain(config, job, taskdesc):
 
     # Use `mach` to invoke python scripts so in-tree libraries are available.
     if run['script'].endswith('.py'):
-        wrapper = 'workspace/build/src/mach python '
+        wrapper = '{}/mach python '.format(gecko_path)
     else:
         wrapper = ''
 
@@ -160,7 +166,7 @@ def docker_worker_toolchain(config, job, taskdesc):
 
     worker['command'] = [
         '{workdir}/bin/run-task'.format(**run),
-        '--vcs-checkout={workdir}/workspace/build/src'.format(**run),
+        '--vcs-checkout={}'.format(gecko_path),
     ] + sparse_profile + [
         '--',
         'bash',