Bug 1461980 - [taskgraph] Use run-task's 'use-artifacts' key to setup artifacts in mochitest and reftest selftests, r=dustin

Instead of downloading the build artifacts (rather hackily) in moztest.fixtures, this now happens
directly in the taskgraph module via the run-task script.

For now extraction and setup happens in the task's command key. It might be a good idea to figure
out a syntax to tell run-task to do this extraction, e.g something like:

run:
    using-artifacts:
        build:
            target.tar.bz2:
                extract: true
                path: /home/worker/build
                name: firefox

But for now I wanted to avoid this extra complexity, so maybe it could be done in a follow-up.

MozReview-Commit-ID: KOhFFpFdP7Y

--HG--
extra : rebase_source : dcea36661fa9c6442c76c850ccc67f8f6d924fda

layout/tools/reftest/selftest/conftest.py

@@ -48,6 +48,9 @@ def runtests(setup_test_harness, binary, parser):
             'sandboxReadWhitelist': [here, os.environ['PYTHON_TEST_TMP']],
             'utilityPath': os.path.join(package_root, 'bin'),
         })
+
+        if 'USE_ARTIFACT_PATH' in os.environ:
+            options['sandboxReadWhitelist'].append(os.environ['USE_ARTIFACT_PATH'])
     else:
         options.update({
             'extraProfileFiles': [os.path.join(build.topobjdir, 'dist', 'plugins')],

taskcluster/ci/source-test/kind.yml

@@ -23,12 +23,6 @@ jobs-from:
 # This is used by run-task based tasks to lookup which build task it
 # should depend on based on its own platform.
 dependent-build-platforms:
-   linux64-asan/opt:
-      label: build-linux64-asan/opt
-      target-name: target.tar.bz2
-   linux64/debug:
-      label: build-linux64/debug
-      target-name: target.tar.bz2
-   linux64.*:
-      label: build-linux64/opt
-      target-name: target.tar.bz2
+   linux64-asan/opt: build-linux64-asan/opt
+   linux64/debug: build-linux64/debug
+   linux64.*: build-linux64/opt

taskcluster/ci/source-test/python.yml

@@ -60,9 +60,20 @@ mochitest-harness:
                 max-run-time: 3600
     run:
         using: run-task
+        use-artifacts:
+            build:
+                - target.tar.bz2
+                - target.common.tests.zip
+                - target.mochitest.tests.zip
         command: >
             source /builds/worker/scripts/xvfb.sh &&
             start_xvfb '1600x1200x24' 0 &&
+            cd $USE_ARTIFACT_PATH/build &&
+            tar -xf target.tar.bz2 &&
+            unzip -q -d tests target.common.tests.zip &&
+            unzip -q -d tests target.mochitest.tests.zip &&
+            export GECKO_BINARY_PATH=$USE_ARTIFACT_PATH/build/firefox/firefox &&
+            export TEST_HARNESS_ROOT=$USE_ARTIFACT_PATH/build/tests &&
             cd /builds/worker/checkouts/gecko &&
             ./mach python-test --subsuite mochitest
     when:
@@ -156,9 +167,20 @@ reftest-harness:
                 max-run-time: 3600
     run:
         using: run-task
+        use-artifacts:
+            build:
+                - target.tar.bz2
+                - target.common.tests.zip
+                - target.reftest.tests.zip
         command: >
             source /builds/worker/scripts/xvfb.sh &&
             start_xvfb '1600x1200x24' 0 &&
+            cd $USE_ARTIFACT_PATH/build &&
+            tar -xf target.tar.bz2 &&
+            unzip -q -d tests target.common.tests.zip &&
+            unzip -q -d tests target.reftest.tests.zip &&
+            export GECKO_BINARY_PATH=$USE_ARTIFACT_PATH/build/firefox/firefox &&
+            export TEST_HARNESS_ROOT=$USE_ARTIFACT_PATH/build/tests &&
             cd /builds/worker/checkouts/gecko &&
             ./mach python-test --subsuite reftest
     when:

taskcluster/taskgraph/transforms/source_test.py

@@ -117,17 +117,10 @@ def add_build_dependency(config, job):
     if len(matches) > 1:
         raise Exception("More than one build platform found for '{}'.".format(key))
 
-    label = matches[0]['label']
-    target = matches[0]['target-name']
+    label = matches[0]
     deps = job.setdefault('dependencies', {})
     deps.update({'build': label})
 
-    build_artifact = 'public/build/{}'.format(target)
-    installer_url = ARTIFACT_URL.format('<build>', build_artifact)
-
-    env = job['worker'].setdefault('env', {})
-    env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
-
 
 @transforms.add
 def handle_platform(config, jobs):

testing/mozbase/moztest/moztest/selftest/fixtures.py

@@ -11,10 +11,8 @@ import os
 import shutil
 import sys
 
-import mozfile
 import mozinstall
 import pytest
-import requests
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -38,29 +36,10 @@ def _get_test_harness(suite, install_dir):
         if os.path.isdir(harness_root):
             return harness_root
 
-    # Check if it was previously set up by another test
-    harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests', suite)
-    if os.path.isdir(harness_root):
-        return harness_root
-
-    # Check if there is a test package to download
-    if 'GECKO_INSTALLER_URL' in os.environ:
-        base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
-        test_packages = requests.get(base_url + '/target.test_packages.json').json()
-
-        dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
-        for name in test_packages[suite]:
-            url = base_url + '/' + name
-            bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
-
-            r = requests.get(url, stream=True)
-            with open(bundle, 'w+b') as fh:
-                for chunk in r.iter_content(chunk_size=1024):
-                    fh.write(chunk)
-
-            mozfile.extract(bundle, dest)
-
-        return os.path.join(dest, suite)
+    if 'TEST_HARNESS_ROOT' in os.environ:
+        harness_root = os.path.join(os.environ['TEST_HARNESS_ROOT'], suite)
+        if os.path.isdir(harness_root):
+            return harness_root
 
     # Couldn't find a harness root, let caller do error handling.
     return None
@@ -85,7 +64,7 @@ def setup_test_harness(request):
                     else:
                         shutil.copytree(files_dir, test_root)
 
-        elif 'GECKO_INSTALLER_URL' in os.environ:
+        elif 'TEST_HARNESS_ROOT' in os.environ:
             # The mochitest tests will run regardless of whether a build exists or not.
             # In a local environment, they should simply be skipped if setup fails. But
             # in automation, we'll need to make sure an error is propagated up.
@@ -114,10 +93,5 @@ def binary():
         except Exception:
             pass
 
-    if 'GECKO_INSTALLER_URL' in os.environ:
-        bindir = mozinstall.install(
-            os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP'])
-        return mozinstall.get_binary(bindir, app_name='firefox')
-
     if 'GECKO_BINARY_PATH' in os.environ:
         return os.environ['GECKO_BINARY_PATH']