Backed out 3 changesets (bug 1643689) for Gecko Decision Task failure. CLOSED TREE

Backed out changeset 2912d91dd291 (bug 1643689)
Backed out changeset 50195a6883bf (bug 1643689)
Backed out changeset 6c2a31b47d0b (bug 1643689)

taskcluster/ci/test/awsy.yml

@@ -33,7 +33,6 @@ job-defaults:
             .*-devedition/.*: []  # don't run on devedition
             (linux|win|mac)(?!.*shippable).*/opt: []
             default: ['integration', 'mozilla-central', 'mozilla-beta', 'mozilla-release']
-    test-manifest-loader: null  # don't load tests in the taskgraph
     fission-run-on-projects:
         by-test-platform:
             linux1804-64-shippable-qr/opt: ['mozilla-central']

taskcluster/ci/test/browsertime-desktop.yml

@@ -13,7 +13,6 @@ job-defaults:
         by-test-platform:
             linux.*shippable[^-qr].*: ['mozilla-central']
             default: []
-    test-manifest-loader: null  # don't load tests in the taskgraph
     tier: 3
     fission-tier: 3
     virtualization:

taskcluster/ci/test/browsertime-mobile.yml

@@ -16,7 +16,6 @@ job-defaults:
                     default: []
     webrender-run-on-projects: []
     tier: 3
-    test-manifest-loader: null  # don't load tests in the taskgraph
     target:
         by-app:
             fenix:

taskcluster/ci/test/compiled.yml

@@ -4,7 +4,6 @@
 ---
 job-defaults:
     e10s: false
-    test-manifest-loader: null  # don't load tests in the taskgraph
     virtualization:
         by-test-platform:
             windows10-64.*: hardware

taskcluster/ci/test/firefox-ui.yml

@@ -6,7 +6,6 @@ job-defaults:
     suite:
         category: firefox-ui
     max-run-time: 5400
-    test-manifest-loader: null  # don't load tests in the taskgraph
     mozharness:
         script: firefox_ui_tests/functional.py
         config:

taskcluster/ci/test/marionette.yml

@@ -20,7 +20,6 @@ job-defaults:
             - --setpref=toolkit.asyncshutdown.log=true
 
     run-on-projects: built-projects
-    test-manifest-loader: null  # don't load tests in the taskgraph
     tier: default
     fission-run-on-projects:
         by-test-platform:

taskcluster/ci/test/misc.yml

@@ -2,9 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
-job-defaults:
-    test-manifest-loader: null  # don't load tests in the taskgraph
-
 geckoview-junit:
     description: "Geckoview junit run"
     suite: geckoview-junit

taskcluster/ci/test/mochitest.yml

@@ -97,7 +97,6 @@ mochitest-a11y:
     description: "Mochitest a11y run"
     treeherder-symbol: M(a11y)
     schedules-component: mochitest-a11y
-    test-manifest-loader: default  # ensure we don't run with manifest-scheduling
     loopback-video: true
     tier: default
     e10s: false
@@ -156,7 +155,6 @@ browser-screenshots:
         name: mochitest-browser-chrome-screenshots
     treeherder-symbol: M(ss)
     loopback-video: true
-    test-manifest-loader: null  # don't load tests in the taskgraph
     run-on-projects:
         by-test-platform:
             windows7-32(?:-pgo|-shippable)(?:-qr)?/opt: ['mozilla-central']
@@ -351,7 +349,6 @@ mochitest-valgrind:
         name: mochitest-valgrind-plain
     treeherder-symbol: M-V()
     run-on-projects: []
-    test-manifest-loader: null  # don't load tests in the taskgraph
     tier: 3
     loopback-video: true
     chunks: 40
@@ -375,7 +372,6 @@ mochitest-webgl1-core:
             android.*: []
             default: ['webgl-ipc']
     e10s: true
-    test-manifest-loader: null  # don't load tests in the taskgraph
     loopback-video: true
     tier: default
     max-run-time:
@@ -407,7 +403,6 @@ mochitest-webgl1-ext:
             android.*: 2
             default: 1
     e10s: true
-    test-manifest-loader: null  # don't load tests in the taskgraph
     loopback-video: true
     tier: default
     max-run-time: 2700
@@ -436,7 +431,6 @@ mochitest-webgl2-core:
             android.*: 2
             default: 1
     e10s: true
-    test-manifest-loader: null  # don't load tests in the taskgraph
     loopback-video: true
     tier: default
     max-run-time: 1800
@@ -463,7 +457,6 @@ mochitest-webgl2-ext:
             default: ['webgl-ipc']
     chunks: 4
     e10s: true
-    test-manifest-loader: null  # don't load tests in the taskgraph
     loopback-video: true
     tier: default
     max-run-time: 2700

taskcluster/ci/test/raptor-chrome.yml

@@ -10,7 +10,6 @@ job-defaults:
             default: 1800
     suite: raptor
     run-on-projects: []
-    test-manifest-loader: null  # don't load tests in the taskgraph
     tier: 2
     virtualization:
         by-test-platform:

taskcluster/ci/test/raptor-chromium.yml

@@ -15,7 +15,6 @@ job-defaults:
         by-test-platform:
             windows10-64-ccov/.*: virtual
             default: hardware
-    test-manifest-loader: null  # don't load tests in the taskgraph
     mozharness:
         script: raptor_script.py
         config:

taskcluster/ci/test/raptor-fenix.yml

@@ -6,7 +6,6 @@ job-defaults:
     max-run-time: 1800
     suite: raptor
     run-on-projects: []
-    test-manifest-loader: null  # don't load tests in the taskgraph
     tier: 1
     target:
         by-test-platform:

taskcluster/ci/test/raptor-fennec68.yml

@@ -16,7 +16,6 @@ job-defaults:
                 index: gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-api-16-release-opt
                 name: build/target.apk
     virtualization: hardware
-    test-manifest-loader: null  # don't load tests in the taskgraph
     mozharness:
         script: raptor_script.py
         config:

taskcluster/ci/test/raptor-gve.yml

@@ -12,7 +12,6 @@ job-defaults:
             android-hw(?!-p2).*(?<!-api-16-shippable)/opt.*: []
             default: ['mozilla-central']
     target: geckoview_example.apk
-    test-manifest-loader: null  # don't load tests in the taskgraph
     tier: 1
     virtualization: hardware
     mozharness:

taskcluster/ci/test/raptor-refbrow.yml

@@ -6,7 +6,6 @@ job-defaults:
     suite: raptor
     run-on-projects: []
     tier: 2
-    test-manifest-loader: null  # don't load tests in the taskgraph
     target:
         by-test-platform:
             android-hw.*-aarch64.*/.*:

taskcluster/ci/test/raptor.yml

@@ -16,7 +16,6 @@ job-defaults:
                     .*-ux/.*: 2400
                     default: 1800
     suite: raptor
-    test-manifest-loader: null  # don't load tests in the taskgraph
     run-on-projects:
         by-app:
             firefox:

taskcluster/ci/test/reftest.yml

@@ -18,7 +18,6 @@ job-defaults:
             android-hw-.*-qr/opt: ['mozilla-central', 'autoland']
             windows10-aarch64/.*: []  # Specific repos enabled below
             default: built-projects
-    test-manifest-loader: null  # don't load tests in the taskgraph
     mozharness:
         script:
             by-test-platform:

taskcluster/ci/test/talos.yml

@@ -32,7 +32,6 @@ job-defaults:
                     - talos/linux64_config_taskcluster.py
                 default:
                     - talos/linux_config.py
-    test-manifest-loader: null  # don't load tests in the taskgraph
     run-on-projects:
         by-test-name:
             talos-.*-profiling:

taskcluster/ci/test/web-platform.yml

@@ -105,7 +105,6 @@ web-platform-tests-backlog:
     variants: []
     e10s: true
     run-on-projects: ['mozilla-central']
-    test-manifest-loader: null  # don't load tests in the taskgraph
     tier: 2
     mozharness:
         chunked: true
@@ -154,7 +153,6 @@ web-platform-tests-reftest-backlog:
     variants: []
     e10s: true
     run-on-projects: ['mozilla-central']
-    test-manifest-loader: null  # don't load tests in the taskgraph
     tier: 2
     mozharness:
         extra-options:
@@ -285,7 +283,6 @@ web-platform-tests-print-reftest:
     schedules-component: web-platform-tests-print-reftest
     treeherder-symbol: W(Wp)
     chunks: 1
-    test-manifest-loader: null  # don't load tests in the taskgraph
     tier:
         by-test-platform:
             .*-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
@@ -312,7 +309,6 @@ test-verify-wpt:
             # do not run on beta or release: usually just confirms earlier results
             default: ['integration']
     built-projects-only: true
-    test-manifest-loader: null  # don't load tests in the taskgraph
     optimization:
         test-verify: ["test-verify-wpt"]
     tier: 2
@@ -332,7 +328,6 @@ test-coverage-wpt:
         by-test-platform:
             .*-ccov/.*: built-projects
             default: []
-    test-manifest-loader: null  # don't load tests in the taskgraph
     tier: 2
     mozharness:
         extra-options:

taskcluster/taskgraph/transforms/tests.py

@@ -47,11 +47,9 @@ from taskgraph.util.schema import (
 from taskgraph.optimize.schema import OptimizationSchema
 from taskgraph.util.chunking import (
     chunk_manifests,
-    get_manifest_loader,
     get_runtimes,
     guess_mozinfo_from_task,
     manifest_loaders,
-    DefaultLoader,
 )
 from taskgraph.util.taskcluster import (
     get_artifact_path,
@@ -289,6 +287,44 @@ TEST_VARIANTS = {
 }
 
 
+CHUNK_SUITES_BLACKLIST = (
+    'awsy',
+    'cppunittest',
+    'crashtest',
+    'crashtest-qr',
+    'firefox-ui-functional-local',
+    'firefox-ui-functional-remote',
+    'geckoview-junit',
+    'gtest',
+    'jittest',
+    'jsreftest',
+    'marionette',
+    'mochitest-browser-chrome-screenshots',
+    'mochitest-browser-chrome-thunderbird',
+    'mochitest-valgrind-plain',
+    'mochitest-webgl1-core',
+    'mochitest-webgl1-ext',
+    'mochitest-webgl2-core',
+    'mochitest-webgl2-ext',
+    'raptor',
+    'reftest',
+    'reftest-qr',
+    'reftest-gpu',
+    'reftest-no-accel',
+    'talos',
+    'telemetry-tests-client',
+    'test-coverage',
+    'test-coverage-wpt',
+    'test-verify',
+    'test-verify-gpu',
+    'test-verify-wpt',
+    'web-platform-tests-backlog',
+    'web-platform-tests-print-reftest',
+    'web-platform-tests-reftest-backlog',
+)
+"""These suites will be chunked at test runtime rather than here in the taskgraph."""
+
+
 DYNAMIC_CHUNK_DURATION = 20 * 60  # seconds
 """The approximate time each test chunk should take to run."""
 
@@ -396,12 +432,6 @@ test_description_schema = Schema({
         'test-platform',
         Any(int, 'dynamic')),
 
-
-    # Custom 'test_manifest_loader' to use, overriding the one configured in the
-    # parameters. When 'null', no test chunking will be performed. Can also
-    # be used to disable "manifest scheduling".
-    Optional('test-manifest-loader'): Any(None, list(manifest_loaders)),
-
     # the time (with unit) after which this task is deleted; default depends on
     # the branch (see below)
     Optional('expires-after'): text_type,
@@ -1466,10 +1496,11 @@ def set_test_verify_chunks(config, tasks):
 def set_test_manifests(config, tasks):
     """Determine the set of test manifests that should run in this task."""
 
+    loader_cls = manifest_loaders[config.params['test_manifest_loader']]
+    loader = loader_cls(config.params)
+
     for task in tasks:
-        # When a task explicitly requests no 'test_manifest_loader', test
-        # resolving will happen at test runtime rather than in the taskgraph.
-        if 'test-manifest-loader' in task and task['test-manifest-loader'] is None:
+        if task['suite'] in CHUNK_SUITES_BLACKLIST:
             yield task
             continue
 
@@ -1495,9 +1526,6 @@ def set_test_manifests(config, tasks):
 
         mozinfo = guess_mozinfo_from_task(task)
 
-        loader_name = task.pop('test-manifest-loader', config.params['test_manifest_loader'])
-        loader = get_manifest_loader(loader_name, config.params)
-
         task['test-manifests'] = loader.get_manifests(
             task['suite'],
             frozenset(mozinfo.items()),
@@ -1507,7 +1535,7 @@ def set_test_manifests(config, tasks):
         # loader, we'll only run some subset of manifests and the hardcoded
         # chunk numbers will no longer be valid. Dynamic chunking should yield
         # better results.
-        if not isinstance(loader, DefaultLoader):
+        if config.params['test_manifest_loader'] != 'default':
             task['chunks'] = "dynamic"
 
         yield task

taskcluster/taskgraph/util/chunking.py

@@ -242,16 +242,3 @@ manifest_loaders = {
     'bugbug': BugbugLoader,
     'default': DefaultLoader,
 }
-
-_loader_cache = {}
-
-
-def get_manifest_loader(name, params):
-    # Ensure we never create more than one instance of the same loader type for
-    # performance reasons.
-    if name in _loader_cache:
-        return _loader_cache[name]
-
-    loader = manifest_loaders[name](dict(params))
-    _loader_cache[name] = loader
-    return loader

taskcluster/test/test_mach_try_auto.py

@@ -7,6 +7,7 @@ import pytest
 from mozunit import main
 from tryselect.selectors.auto import TRY_AUTO_PARAMETERS
 
+from taskgraph.transforms.tests import CHUNK_SUITES_BLACKLIST
 from taskgraph.util.bugbug import push_schedules
 from taskgraph.util.chunking import BugbugLoader
 
@@ -44,16 +45,13 @@ def test_only_important_manifests(params, full_task_graph, filter_tasks):
     for task in filter_tasks(full_task_graph, lambda t: t.kind == "test"):
         attr = task.attributes.get
 
-        if "test_manifests" in task.attributes:
+        if attr("unittest_suite") in CHUNK_SUITES_BLACKLIST:
+            assert not attr("test_manifests")
+        else:
             unimportant = [
                 t for t in attr("test_manifests") if t not in important_manifests
             ]
-
-            # Manifest scheduling is disabled for mochitest-ally.
-            if attr("unittest_suite") == "mochitest-a11y":
-                assert len(unimportant) > 0
-            else:
-                assert unimportant == []
+            assert unimportant == []
 
 
 @pytest.mark.parametrize(