Bug 1430006: Allow configuring scriptworkers per-graph config; r=aki

Differential Revision: https://phabricator.services.mozilla.com/D379

--HG--
extra : rebase_source : c52684c1a65017698d29979f35afed646ca5c90c

taskcluster/ci/config.yml

@@ -96,3 +96,11 @@ try:
         'win32':
             - 'sm-plain-win32'
             - 'sm-compacting-win32'
+
+scriptworker:
+    worker-types:
+        'scriptworker-prov-v1/signing-linux-v1':
+            - 'project:releng:signing:cert:release-signing'
+            - 'project:releng:signing:cert:nightly-signing'
+        'scriptworker-prov-v1/depsigning':
+            - 'project:releng:signing:cert:dep-signing'

taskcluster/docs/taskgraph.rst

@@ -176,6 +176,8 @@ using simple parameterized values, as follows:
     Multiple labels may be substituted in a single string, and ``<<>`` can be
     used to escape a literal ``<``.
 
+.. _taskgraph-graph-config:
+
 Graph Configuration
 -------------------
 
@@ -192,7 +194,7 @@ Trust Domain
 
 When publishing and signing releases, that tasks verify their definition and
 all upstream tasks come from a decision task based on a trusted tree. (see
-`chain-of-trust verification <http://scriptworker.readthedocs.io/en/latest/chain_of_trust.html>`).
+`chain-of-trust verification <http://scriptworker.readthedocs.io/en/latest/chain_of_trust.html>`_).
 Firefox and Thunderbird share the taskgraph code and in particular, they have
 separate taskgraph configurations and in particular distinct decision tasks.
 Although they use identical docker images and toolchains, in order to track the

taskcluster/taskgraph/config.py

@@ -26,6 +26,10 @@ graph_config_schema = Schema({
         # all"
         Required('ridealong-builds', default={}): {basestring: [basestring]},
     },
+    Required('scriptworker'): {
+        # Mapping of scriptworker types to scopes they accept
+        Required('worker-types'): {basestring: [basestring]}
+    },
 })
 
 

taskcluster/taskgraph/transforms/checksums_signing.py

@@ -10,7 +10,10 @@ from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
-from taskgraph.util.scriptworker import get_signing_cert_scope
+from taskgraph.util.scriptworker import (
+    get_signing_cert_scope,
+    get_worker_type_for_scope,
+)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
@@ -89,7 +92,7 @@ def make_checksums_signing_description(config, jobs):
         task = {
             'label': label,
             'description': description,
-            'worker-type': _generate_worker_type(signing_cert_scope),
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
@@ -104,8 +107,3 @@ def make_checksums_signing_description(config, jobs):
         }
 
         yield task
-
-
-def _generate_worker_type(signing_cert_scope):
-    worker_type = 'depsigning' if 'dep-signing' in signing_cert_scope else 'signing-linux-v1'
-    return 'scriptworker-prov-v1/{}'.format(worker_type)

taskcluster/taskgraph/transforms/repackage_signing.py

@@ -10,7 +10,10 @@ from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
-from taskgraph.util.scriptworker import get_signing_cert_scope_per_platform
+from taskgraph.util.scriptworker import (
+    get_signing_cert_scope_per_platform,
+    get_worker_type_for_scope,
+)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
@@ -121,7 +124,7 @@ def make_repackage_signing_description(config, jobs):
         task = {
             'label': label,
             'description': description,
-            'worker-type': _generate_worker_type(signing_cert_scope),
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
@@ -133,8 +136,3 @@ def make_repackage_signing_description(config, jobs):
         }
 
         yield task
-
-
-def _generate_worker_type(signing_cert_scope):
-    worker_type = 'depsigning' if 'dep-signing' in signing_cert_scope else 'signing-linux-v1'
-    return 'scriptworker-prov-v1/{}'.format(worker_type)

taskcluster/taskgraph/transforms/signing.py

@@ -10,7 +10,10 @@ from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
-from taskgraph.util.scriptworker import get_signing_cert_scope_per_platform
+from taskgraph.util.scriptworker import (
+    get_signing_cert_scope_per_platform,
+    get_worker_type_for_scope,
+)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
@@ -135,7 +138,7 @@ def make_task_description(config, jobs):
         task = {
             'label': label,
             'description': description,
-            'worker-type': _generate_worker_type(signing_cert_scope),
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': job['upstream-artifacts'],
                        'max-run-time': 3600},
@@ -157,8 +160,3 @@ def _generate_treeherder_platform(dep_th_platform, build_platform, build_type):
 
 def _generate_treeherder_symbol(is_nightly):
     return 'tc(Ns)' if is_nightly else 'tc(Bs)'
-
-
-def _generate_worker_type(signing_cert_scope):
-    worker_type = 'depsigning' if 'dep-signing' in signing_cert_scope else 'signing-linux-v1'
-    return 'scriptworker-prov-v1/{}'.format(worker_type)

taskcluster/taskgraph/util/scriptworker.py

@@ -12,6 +12,8 @@ project branch, and merge day uplifts more user friendly.
 In the future, we may adjust scopes by other settings as well, e.g. different
 scopes for `push-to-candidates` rather than `push-to-releases`, even if both
 happen on mozilla-beta and mozilla-release.
+
+Additional configuration is found in the :ref:`graph config <taskgraph-graph-config>`.
 """
 from __future__ import absolute_import, print_function, unicode_literals
 import functools
@@ -296,7 +298,7 @@ def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
         alias_to_project_map (list of lists): each list pair contains the
             alias and the set of projects that match.  This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
-        config (dict): the task config that defines the project.
+        config (TransformConfig): The configuration for the kind being transformed.
 
     Returns:
         string: the scope to use.
@@ -314,7 +316,7 @@ def get_scope_from_target_method(alias_to_tasks_map, alias_to_scope_map, config)
         alias_to_tasks_map (list of lists): each list pair contains the
             alias and the set of target methods that match. This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
-        config (dict): the task config that defines the target task method.
+        config (TransformConfig): The configuration for the kind being transformed.
 
     Returns:
         string: the scope to use.
@@ -340,7 +342,7 @@ def get_scope_from_target_method_and_project(alias_to_tasks_map, alias_to_projec
         alias_to_project_map (list of lists): each list pair contains the
             alias and the set of projects that match.  This is ordered.
         aliases_to_scope_map (dict of dicts): the task alias to project alias to scope
-        config (dict): the task config that defines the target task method and project.
+        config (TransformConfig): The configuration for the kind being transformed.
 
     Returns:
         string: the scope to use.
@@ -436,7 +438,7 @@ def get_release_config(config):
     Currently only applies to beetmover tasks.
 
     Args:
-        config (dict): the task config that defines the target task method.
+        config (TransformConfig): The configuration for the kind being transformed.
 
     Returns:
         dict: containing both `build_number` and `version`.  This can be used to
@@ -484,3 +486,28 @@ def get_signing_cert_scope_per_platform(build_platform, is_nightly, config):
         return get_signing_cert_scope(config)
     else:
         return 'project:releng:signing:cert:dep-signing'
+
+
+def get_worker_type_for_scope(config, scope):
+    """Get the scriptworker type that will accept the given scope.
+
+    Args:
+        config (TransformConfig): The configuration for the kind being transformed.
+        scope (string): The scope being used.
+
+    Returns:
+        string: The worker-type to use.
+    """
+    for worker_type, scopes in config.graph_config['scriptworker']['worker-types'].items():
+        if scope in scopes:
+            return worker_type
+    raise RuntimeError(
+        "Unsupported scriptworker scope {scope}. (supported scopes: {available_scopes})".format(
+            scope=scope,
+            available_scopes=sorted(
+                scope
+                for scopes in config.graph_config['scriptworker']['worker-types'].values()
+                for scope in scopes
+            ),
+        )
+    )