Backed out 3 changesets (bug 1636271) for breaking the Gecko Decision Task. CLOSED TREE

Backed out changeset 4b0f13fcf941 (bug 1636271)
Backed out changeset 4250f49877ba (bug 1636271)
Backed out changeset d3f93bd6b1f9 (bug 1636271)

taskcluster/docs/parameters.rst

@@ -135,12 +135,8 @@ syntax or reading a project-specific configuration file).
 Optimization
 ------------
 
-``optimize_strategies``
-   A python path of the form ``<module>:<object>`` containing a dictionary of
-   optimization strategies to use, overwriting the defaults.
-
 ``optimize_target_tasks``
-   If true, then target tasks are eligible for optimization.
+    If true, then target tasks are eligible for optimization.
 
 ``do_not_optimize``
    Specify tasks to not optimize out of the graph. This is a list of labels.

taskcluster/taskgraph/actions/cancel_all.py

@@ -12,7 +12,7 @@ import os
 import requests
 
 from taskgraph.util.taskcluster import (
-    list_task_group_incomplete_task_ids,
+    list_task_group_incomplete_tasks,
     cancel_task,
     CONCURRENCY,
 )
@@ -48,8 +48,7 @@ def cancel_all_action(parameters, graph_config, input, task_group_id, task_id):
             raise
 
     own_task_id = os.environ.get('TASK_ID', '')
-    to_cancel = [t for t in list_task_group_incomplete_task_ids(task_group_id) if t != own_task_id]
-
+    to_cancel = [t for t in list_task_group_incomplete_tasks(task_group_id) if t != own_task_id]
     logger.info("Cancelling {} tasks".format(len(to_cancel)))
     with futures.ThreadPoolExecutor(CONCURRENCY) as e:
         cancel_futs = [e.submit(do_cancel_task, t) for t in to_cancel]

taskcluster/taskgraph/decision.py

@@ -72,10 +72,6 @@ PER_PROJECT_PARAMETERS = {
         'target_tasks_method': 'graphics_tasks',
     },
 
-    'autoland': {
-        'optimize_strategies': 'taskgraph.optimize:project.autoland',
-    },
-
     'mozilla-central': {
         'target_tasks_method': 'default',
         'release_type': 'nightly',
@@ -322,6 +318,7 @@ def get_decision_parameters(graph_config, options):
     parameters['filters'] = [
         'target_tasks_method',
     ]
+    parameters['optimize_target_tasks'] = True
     parameters['existing_tasks'] = {}
     parameters['do_not_optimize'] = []
     parameters['build_number'] = 1
@@ -330,8 +327,6 @@ def get_decision_parameters(graph_config, options):
     parameters['message'] = try_syntax_from_message(commit_message)
     parameters['hg_branch'] = get_hg_revision_branch(GECKO, revision=parameters['head_rev'])
     parameters['next_version'] = None
-    parameters['optimize_strategies'] = None
-    parameters['optimize_target_tasks'] = True
     parameters['phabricator_diff'] = None
     parameters['release_type'] = ''
     parameters['release_eta'] = ''

taskcluster/taskgraph/generator.py

@@ -346,7 +346,7 @@ class TaskGraphGenerator(object):
 
         # this is used for testing experimental optimization strategies
         strategies = os.environ.get('TASKGRAPH_OPTIMIZE_STRATEGIES',
-                                    parameters.get('optimize_strategies'))
+                                    parameters['try_task_config'].get('optimize-strategies'))
         if strategies:
             strategies = find_object(strategies)
 

taskcluster/taskgraph/optimize/__init__.py

@@ -397,12 +397,11 @@ class Alias(CompositeStrategy):
 
 
 def split_bugbug_arg(arg):
-    """Split args for bugbug based strategies.
-
-    Many bugbug based optimizations require passing an empty dict by reference
-    to communicate to downstream strategies. This function passes the provided
-    arg to the first strategy and an empty dict to second (bugbug based)
-    strategy.
+    """
+    The bugbug optimization strageies require passing an dict as
+    scratch space for communicating with downstream stratgies.
+    This function pass the provided argument to the first strategy,
+    and a fresh dictionary to the second stratgey.
     """
     return (arg, {})
 
@@ -418,24 +417,15 @@ register_strategy('build-optimized', args=(
     'backstop',
 ))(All)
 register_strategy('build-fuzzing', args=('push-interval-10',))(Alias)
-register_strategy('test', args=('skip-unless-schedules',))(Alias)
+register_strategy('test', args=(
+    Any('skip-unless-schedules', 'bugbug-reduced-fallback', split_args=split_bugbug_arg),
+    'backstop',
+))(All)
 register_strategy('test-inclusive', args=('skip-unless-schedules',))(Alias)
+register_strategy('test-try', args=('skip-unless-schedules',))(Alias)
 
 
-# Strategy overrides used to tweak the default strategies. These are referenced
-# by the `optimize_strategies` parameter.
-
-class project(object):
-    """Strategies that should be applied per-project."""
-
-    autoland = {
-        'test': All(
-            Any('skip-unless-schedules', 'bugbug-reduced-fallback', split_args=split_bugbug_arg),
-            'backstop',
-        ),
-    }
-    """Strategy overrides that apply to autoland."""
-
+# Strategy overrides used by |mach try| and/or shadow-scheduler tasks.
 
 class experimental(object):
     """Experimental strategies either under development or used as benchmarks.

taskcluster/taskgraph/parameters.py

@@ -79,7 +79,6 @@ base_schema = Schema({
     Required('message'): text_type,
     Required('moz_build_date'): text_type,
     Required('next_version'): Any(None, text_type),
-    Required('optimize_strategies'): Any(None, text_type),
     Required('optimize_target_tasks'): bool,
     Required('owner'): text_type,
     Required('phabricator_diff'): Any(None, text_type),
@@ -162,7 +161,6 @@ class Parameters(ReadOnlyDict):
             'message': '',
             'moz_build_date': six.ensure_text(now.strftime("%Y%m%d%H%M%S")),
             'next_version': None,
-            'optimize_strategies': None,
             'optimize_target_tasks': True,
             'owner': 'nobody@mozilla.com',
             'phabricator_diff': None,

taskcluster/taskgraph/test/test_parameters.py

@@ -31,7 +31,6 @@ class TestParameters(unittest.TestCase):
         'message': 'message',
         'moz_build_date': 'moz_build_date',
         'next_version': 'next_version',
-        'optimize_strategies': None,
         'optimize_target_tasks': False,
         'owner': 'owner',
         'phabricator_diff': 'phabricator_diff',

taskcluster/taskgraph/transforms/tests.py

@@ -1817,6 +1817,9 @@ def make_job_description(config, tasks):
             jobdesc['when'] = task['when']
         elif 'optimization' in task:
             jobdesc['optimization'] = task['optimization']
+        # Pushes generated by `mach try auto` should use the non-try optimizations.
+        elif config.params.is_try() and config.params['try_mode'] != 'try_auto':
+            jobdesc['optimization'] = {'test-try': schedules}
         elif set(schedules) & set(INCLUSIVE_COMPONENTS):
             jobdesc['optimization'] = {'test-inclusive': schedules}
         else:

taskcluster/taskgraph/util/schema.py

@@ -225,6 +225,7 @@ OptimizationSchema = voluptuous.Any(
     # optimize strategy aliases for the test kind
     {'test': list(schedules.ALL_COMPONENTS)},
     {'test-inclusive': list(schedules.ALL_COMPONENTS)},
+    {'test-try': list(schedules.ALL_COMPONENTS)},
 )
 
 # shortcut for a string where task references are allowed

taskcluster/taskgraph/util/taskcluster.py

@@ -306,23 +306,17 @@ def send_email(address, subject, content, link, use_proxy=False):
     })
 
 
-def list_task_group_tasks(task_group_id):
-    """Generate the tasks in a task group"""
+def list_task_group_incomplete_tasks(task_group_id):
+    """Generate the incomplete tasks in a task group"""
     params = {}
     while True:
         url = liburls.api(get_root_url(False), 'queue', 'v1',
                           'task-group/{}/list'.format(task_group_id))
         resp = _do_request(url, force_get=True, params=params).json()
-        for task in resp['tasks']:
-            yield task
+        for task in [t['status'] for t in resp['tasks']]:
+            if task['state'] in ['running', 'pending', 'unscheduled']:
+                yield task['taskId']
         if resp.get('continuationToken'):
             params = {'continuationToken': resp.get('continuationToken')}
         else:
             break
-
-
-def list_task_group_incomplete_task_ids(task_group_id):
-    states = ('running', 'pending', 'unscheduled')
-    for task in [t['status'] for t in list_task_group_tasks(task_group_id)]:
-        if task['state'] in states:
-            yield task['taskId']

taskcluster/test/test_mach_try_auto.py

@@ -64,6 +64,7 @@ def test_only_important_manifests(params, full_task_graph, filter_tasks):
             assert unimportant == []
 
 
+@pytest.mark.xfail(reason="Bug 1641131 - shippable builds still showing up due to dependencies")
 def test_no_shippable_builds(tgg, filter_tasks):
     optimized_task_graph = tgg.optimized_task_graph
     # We can still sometimes get macosx64-shippable builds with |mach try

tools/tryselect/selectors/auto.py

@@ -4,14 +4,11 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-from taskgraph.util.python_path import find_object
-
 from ..cli import BaseTryParser
 from ..push import push_to_try
 
 
 TRY_AUTO_PARAMETERS = {
-    'optimize_strategies': 'taskgraph.optimize:tryselect.bugbug_debug_disperse',
     'optimize_target_tasks': True,
     'target_tasks_method': 'try_auto',
     'test_manifest_loader': 'bugbug',
@@ -28,43 +25,23 @@ class AutoParser(BaseTryParser):
         "env",
         "chemspill-prio",
         "disable-pgo",
+        "strategy",
         "worker-overrides",
     ]
-    arguments = [
-        [['--strategy'],
-         {'default': None,
-          'help': 'Override the default optimization strategy. Valid values '
-                  'are the experimental strategies defined at the bottom of '
-                  '`taskcluster/taskgraph/optimize/__init__.py`.'
-          }],
-    ]
-
-    def validate(self, args):
-        super(AutoParser, self).validate(args)
-
-        if args.strategy:
-            if ':' not in args.strategy:
-                args.strategy = "taskgraph.optimize:tryselect.{}".format(args.strategy)
-
-            try:
-                obj = find_object(args.strategy)
-            except (ImportError, AttributeError):
-                self.error("invalid module path '{}'".format(args.strategy))
-
-            if not isinstance(obj, dict):
-                self.error("object at '{}' must be a dict".format(args.strategy))
 
 
-def run(message='{msg}', push=True, closed_tree=False, strategy=None, try_config=None, **ignored):
+def run(message='{msg}', push=True, closed_tree=False, try_config=None, **ignored):
     msg = message.format(msg='Tasks automatically selected.')
 
+    # XXX Remove once an intelligent scheduling algorithm is running on
+    # autoland by default. This ensures `mach try auto` doesn't run SETA.
+    try_config.setdefault('optimize-strategies',
+                          'taskgraph.optimize:tryselect.bugbug_debug_disperse')
+
     params = TRY_AUTO_PARAMETERS.copy()
     if try_config:
         params['try_task_config'] = try_config
 
-    if strategy:
-        params['optimize_strategies'] = strategy
-
     task_config = {
         'version': 2,
         'parameters': params,

tools/tryselect/task_config.py

@@ -20,6 +20,7 @@ from textwrap import dedent
 
 import mozpack.path as mozpath
 from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+from taskgraph.util.python_path import find_object
 
 from .tasks import resolve_tests_by_suite
 
@@ -313,6 +314,37 @@ class GeckoProfile(TryConfig):
             }
 
 
+class OptimizeStrategies(TryConfig):
+
+    arguments = [
+        [['--strategy'],
+         {'default': None,
+          'help': 'Override the default optimization strategy. Valid values '
+                  'are the experimental strategies defined at the bottom of '
+                  '`taskcluster/taskgraph/optimize/__init__.py`.'
+          }],
+    ]
+
+    def try_config(self, strategy, **kwargs):
+        if strategy:
+            if ':' not in strategy:
+                strategy = "taskgraph.optimize:tryselect.{}".format(strategy)
+
+            try:
+                obj = find_object(strategy)
+            except (ImportError, AttributeError):
+                print("error: invalid module path '{}'".format(strategy))
+                sys.exit(1)
+
+            if not isinstance(obj, dict):
+                print("error: object at '{}' must be a dict".format(strategy))
+                sys.exit(1)
+
+            return {
+                'optimize-strategies': strategy,
+            }
+
+
 class Browsertime(TryConfig):
     arguments = [
         [['--browsertime'],
@@ -429,5 +461,6 @@ all_task_configs = {
     'pernosco': Pernosco,
     'rebuild': Rebuild,
     'routes': Routes,
+    'strategy': OptimizeStrategies,
     'worker-overrides': WorkerOverrides,
 }

tools/tryselect/test/python.ini

@@ -2,7 +2,6 @@
 subsuite=try
 
 [test_again.py]
-[test_auto.py]
 [test_chooser.py]
 requirements = tools/tryselect/selectors/chooser/requirements.txt
 [test_fuzzy.py]

tools/tryselect/test/test_auto.py

@@ -1,32 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import mozunit
-import pytest
-
-from tryselect.selectors.auto import AutoParser
-
-
-def test_strategy_validation():
-    parser = AutoParser()
-    args = parser.parse_args(["--strategy", "seta"])
-    assert args.strategy == "taskgraph.optimize:tryselect.seta"
-
-    args = parser.parse_args(["--strategy", "taskgraph.optimize:experimental.seta"])
-    assert args.strategy == "taskgraph.optimize:experimental.seta"
-
-    with pytest.raises(SystemExit):
-        parser.parse_args(["--strategy", "taskgraph.optimize:tryselect"])
-
-    with pytest.raises(SystemExit):
-        parser.parse_args(["--strategy", "foo"])
-
-    with pytest.raises(SystemExit):
-        parser.parse_args(["--strategy", "foo:bar"])
-
-
-if __name__ == "__main__":
-    mozunit.main()

tools/tryselect/test/test_auto.t

@@ -12,12 +12,13 @@ Test auto selector
   Calculated try_task_config.json:
   {
       "parameters": {
-          "optimize_strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse",
           "optimize_target_tasks": true,
           "target_tasks_method": "try_auto",
           "test_manifest_loader": "bugbug",
           "try_mode": "try_auto",
-          "try_task_config": {}
+          "try_task_config": {
+              "optimize-strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse"
+          }
       },
       "version": 2
   }
@@ -31,12 +32,13 @@ Test auto selector
   Calculated try_task_config.json:
   {
       "parameters": {
-          "optimize_strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse",
           "optimize_target_tasks": true,
           "target_tasks_method": "try_auto",
           "test_manifest_loader": "bugbug",
           "try_mode": "try_auto",
-          "try_task_config": {}
+          "try_task_config": {
+              "optimize-strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse"
+          }
       },
       "version": 2
   }
@@ -49,12 +51,13 @@ Test auto selector
   Calculated try_task_config.json:
   {
       "parameters": {
-          "optimize_strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse",
           "optimize_target_tasks": true,
           "target_tasks_method": "try_auto",
           "test_manifest_loader": "bugbug",
           "try_mode": "try_auto",
-          "try_task_config": {}
+          "try_task_config": {
+              "optimize-strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse"
+          }
       },
       "version": 2
   }

tools/tryselect/test/test_task_configs.py

@@ -45,6 +45,14 @@ TASK_CONFIG_TESTS = {
         (['--rebuild', '1'], SystemExit),
         (['--rebuild', '21'], SystemExit),
     ],
+    'strategy': [
+        ([], None),
+        (['--strategy', 'seta'], {'optimize-strategies': 'taskgraph.optimize:tryselect.seta'}),
+        (['--strategy', 'taskgraph.optimize:experimental.seta'], {'optimize-strategies': 'taskgraph.optimize:experimental.seta'}),  # noqa
+        (['--strategy', 'taskgraph.optimize:tryselect'], SystemExit),
+        (['--strategy', 'foo'], SystemExit),
+        (['--strategy', 'foo:bar'], SystemExit),
+    ],
     'worker-overrides': [
         ([], None),
         (['--worker-override', 'alias=worker/pool'],