diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index e04541af1e16..cf89fdd5098a 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -98,6 +98,8 @@ try: - 'sm-compacting-win32' scriptworker: + # See additional configuration in taskcluster/taskgraph/util/scriptworker.py + scope-prefix: 'project:releng' worker-types: 'scriptworker-prov-v1/signing-linux-v1': - 'project:releng:signing:cert:release-signing' diff --git a/taskcluster/taskgraph/config.py b/taskcluster/taskgraph/config.py index adfc0d92e8af..34641a0a051c 100644 --- a/taskcluster/taskgraph/config.py +++ b/taskcluster/taskgraph/config.py @@ -27,6 +27,8 @@ graph_config_schema = Schema({ Required('ridealong-builds', default={}): {basestring: [basestring]}, }, Required('scriptworker'): { + # Prefix to add to scopes controlling scriptworkers + Required('scope-prefix'): basestring, # Mapping of scriptworker types to scopes they accept Required('worker-types'): {basestring: [basestring]} }, diff --git a/taskcluster/taskgraph/util/scriptworker.py b/taskcluster/taskgraph/util/scriptworker.py index aeb665d04d4e..af1c18a3f62e 100644 --- a/taskcluster/taskgraph/util/scriptworker.py +++ b/taskcluster/taskgraph/util/scriptworker.py @@ -55,9 +55,9 @@ SIGNING_SCOPE_ALIAS_TO_PROJECT = [[ """Map the signing scope aliases to the actual scopes. """ SIGNING_CERT_SCOPES = { - 'all-release-branches': 'project:releng:signing:cert:release-signing', - 'all-nightly-branches': 'project:releng:signing:cert:nightly-signing', - 'default': 'project:releng:signing:cert:dep-signing', + 'all-release-branches': 'signing:cert:release-signing', + 'all-nightly-branches': 'signing:cert:nightly-signing', + 'default': 'signing:cert:dep-signing', } DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [[ @@ -67,8 +67,8 @@ DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [[ ]] DEVEDITION_SIGNING_CERT_SCOPES = { - 'beta': 'project:releng:signing:cert:nightly-signing', - 'default': 'project:releng:signing:cert:dep-signing', + 'beta': 'signing:cert:nightly-signing', + 'default': 'signing:cert:dep-signing', } """Map beetmover scope aliases to sets of projects. @@ -93,7 +93,7 @@ Used for both `BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK` and `get_release_build_numb BEETMOVER_CANDIDATES_TARGET_TASKS = set([ 'promote_fennec', 'promote_firefox', - 'promote_devedition' + 'promote_devedition', ]) BEETMOVER_PUSH_TARGET_TASKS = set([ 'push_fennec', @@ -130,24 +130,24 @@ BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK = [[ """ BEETMOVER_BUCKET_SCOPES = { 'all-candidates-tasks': { - 'all-release-branches': 'project:releng:beetmover:bucket:release', + 'all-release-branches': 'beetmover:bucket:release', }, 'all-push-tasks': { - 'all-release-branches': 'project:releng:beetmover:bucket:release', + 'all-release-branches': 'beetmover:bucket:release', }, 'all-nightly-tasks': { - 'all-nightly-branches': 'project:releng:beetmover:bucket:nightly', + 'all-nightly-branches': 'beetmover:bucket:nightly', }, - 'default': 'project:releng:beetmover:bucket:dep', + 'default': 'beetmover:bucket:dep', } """Map the beetmover tasks aliases to the actual action scopes. """ BEETMOVER_ACTION_SCOPES = { - 'all-candidates-tasks': 'project:releng:beetmover:action:push-to-candidates', - 'all-push-tasks': 'project:releng:beetmover:action:push-to-releases', - 'all-nightly-tasks': 'project:releng:beetmover:action:push-to-nightly', - 'default': 'project:releng:beetmover:action:push-to-staging', + 'all-candidates-tasks': 'beetmover:action:push-to-candidates', + 'all-push-tasks': 'beetmover:action:push-to-releases', + 'all-nightly-tasks': 'beetmover:action:push-to-nightly', + 'default': 'beetmover:action:push-to-staging', } @@ -184,54 +184,54 @@ BALROG_SCOPE_ALIAS_TO_PROJECT = [[ """Map the balrog scope aliases to the actual scopes. """ BALROG_SERVER_SCOPES = { - 'nightly': 'project:releng:balrog:server:nightly', - 'aurora': 'project:releng:balrog:server:aurora', - 'beta': 'project:releng:balrog:server:beta', - 'release': 'project:releng:balrog:server:release', - 'esr': 'project:releng:balrog:server:esr', - 'default': 'project:releng:balrog:server:dep', + 'nightly': 'balrog:server:nightly', + 'aurora': 'balrog:server:aurora', + 'beta': 'balrog:server:beta', + 'release': 'balrog:server:release', + 'esr': 'balrog:server:esr', + 'default': 'balrog:server:dep', } """Map the balrog scope aliases to the actual channel scopes. """ BALROG_CHANNEL_SCOPES = { 'nightly': [ - 'project:releng:balrog:channel:nightly', - 'project:releng:balrog:channel:nightly-old-id', - 'project:releng:balrog:channel:aurora' + 'balrog:channel:nightly', + 'balrog:channel:nightly-old-id', + 'balrog:channel:aurora', ], 'aurora': [ - 'project:releng:balrog:channel:aurora' + 'balrog:channel:aurora', ], 'beta': [ - 'project:releng:balrog:channel:beta', - 'project:releng:balrog:channel:beta-localtest', - 'project:releng:balrog:channel:beta-cdntest' + 'balrog:channel:beta', + 'balrog:channel:beta-localtest', + 'balrog:channel:beta-cdntest', ], 'release': [ - 'project:releng:balrog:channel:release', - 'project:releng:balrog:channel:release-localtest', - 'project:releng:balrog:channel:release-cdntest' + 'balrog:channel:release', + 'balrog:channel:release-localtest', + 'balrog:channel:release-cdntest', ], 'esr': [ - 'project:releng:balrog:channel:esr', - 'project:releng:balrog:channel:esr-localtest', - 'project:releng:balrog:channel:esr-cdntest' + 'balrog:channel:esr', + 'balrog:channel:esr-localtest', + 'balrog:channel:esr-cdntest', ], 'default': [ - 'project:releng:balrog:channel:nightly', - 'project:releng:balrog:channel:nightly-old-id', - 'project:releng:balrog:channel:aurora' - 'project:releng:balrog:channel:beta', - 'project:releng:balrog:channel:beta-localtest', - 'project:releng:balrog:channel:beta-cdntest', - 'project:releng:balrog:channel:release', - 'project:releng:balrog:channel:release-localtest', - 'project:releng:balrog:channel:release-cdntest', - 'project:releng:balrog:channel:esr', - 'project:releng:balrog:channel:esr-localtest', - 'project:releng:balrog:channel:esr-cdntest' - ] + 'balrog:channel:nightly', + 'balrog:channel:nightly-old-id', + 'balrog:channel:aurora', + 'balrog:channel:beta', + 'balrog:channel:beta-localtest', + 'balrog:channel:beta-cdntest', + 'balrog:channel:release', + 'balrog:channel:release-localtest', + 'balrog:channel:release-cdntest', + 'balrog:channel:esr', + 'balrog:channel:esr-localtest', + 'balrog:channel:esr-cdntest', + ], } @@ -251,26 +251,64 @@ PUSH_APK_SCOPE_ALIAS_TO_PROJECT = [[ PUSH_APK_SCOPES = { - 'central': 'project:releng:googleplay:aurora', - 'beta': 'project:releng:googleplay:beta', - 'release': 'project:releng:googleplay:release', - 'default': 'project:releng:googleplay:invalid', + 'central': 'googleplay:aurora', + 'beta': 'googleplay:beta', + 'release': 'googleplay:release', + 'default': 'googleplay:invalid', } +def add_scope_prefix(config, scope): + """ + Prepends the scriptworker scope prefix from the :ref:`graph config + `. + + Args: + config (TransformConfig): The configuration for the kind being transformed. + scope (string): The suffix of the scope + + Returns: + string: the scope to use. + """ + return "{prefix}:{scope}".format( + prefix=config.graph_config['scriptworker']['scope-prefix'], + scope=scope, + ) +def with_scope_prefix(f): + """ + Wraps a function, calling :py:func:`add_scope_prefix` on the result of + calling the wrapped function. + + Args: + f (callable): A function that takes a ``config`` and some keyword + arguments, and returns a scope suffix. + + Returns: + callable: the wrapped function + """ + @functools.wraps(f) + def wrapper(config, **kwargs): + scope_or_scopes = f(config, **kwargs) + if isinstance(scope_or_scopes, list): + return map(functools.partial(add_scope_prefix, config), scope_or_scopes) + else: + return add_scope_prefix(config, scope_or_scopes) + + return wrapper # scope functions {{{1 -def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config): +@with_scope_prefix +def get_scope_from_project(config, alias_to_project_map, alias_to_scope_map): """Determine the restricted scope from `config.params['project']`. Args: + config (TransformConfig): The configuration for the kind being transformed. alias_to_project_map (list of lists): each list pair contains the alias and the set of projects that match. This is ordered. alias_to_scope_map (dict): the alias alias to scope - config (TransformConfig): The configuration for the kind being transformed. Returns: string: the scope to use. @@ -281,14 +319,15 @@ def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config): return alias_to_scope_map['default'] -def get_scope_from_target_method(alias_to_tasks_map, alias_to_scope_map, config): +@with_scope_prefix +def get_scope_from_target_method(config, alias_to_tasks_map, alias_to_scope_map): """Determine the restricted scope from `config.params['target_tasks_method']`. Args: + config (TransformConfig): The configuration for the kind being transformed. alias_to_tasks_map (list of lists): each list pair contains the alias and the set of target methods that match. This is ordered. alias_to_scope_map (dict): the alias alias to scope - config (TransformConfig): The configuration for the kind being transformed. Returns: string: the scope to use. @@ -299,8 +338,9 @@ def get_scope_from_target_method(alias_to_tasks_map, alias_to_scope_map, config) return alias_to_scope_map['default'] -def get_scope_from_target_method_and_project(alias_to_tasks_map, alias_to_project_map, - aliases_to_scope_map, config): +@with_scope_prefix +def get_scope_from_target_method_and_project(config, alias_to_tasks_map, + alias_to_project_map, aliases_to_scope_map): """Determine the restricted scope from both `target_tasks_method` and `project`. On certain branches, we'll need differing restricted scopes based on @@ -309,12 +349,12 @@ def get_scope_from_target_method_and_project(alias_to_tasks_map, alias_to_projec checks both. Args: + config (TransformConfig): The configuration for the kind being transformed. alias_to_tasks_map (list of lists): each list pair contains the alias and the set of target methods that match. This is ordered. alias_to_project_map (list of lists): each list pair contains the alias and the set of projects that match. This is ordered. aliases_to_scope_map (dict of dicts): the task alias to project alias to scope - config (TransformConfig): The configuration for the kind being transformed. Returns: string: the scope to use. @@ -329,14 +369,14 @@ def get_scope_from_target_method_and_project(alias_to_tasks_map, alias_to_projec return aliases_to_scope_map['default'] -def get_phase_from_target_method(alias_to_tasks_map, alias_to_phase_map, config): +def get_phase_from_target_method(config, alias_to_tasks_map, alias_to_phase_map): """Determine the phase from `config.params['target_tasks_method']`. Args: + config (TransformConfig): The configuration for the kind being transformed. alias_to_tasks_map (list of lists): each list pair contains the alias and the set of target methods that match. This is ordered. alias_to_phase_map (dict): the alias to phase map - config (TransformConfig): The configuration for the kind being transformed. Returns: string: the phase to use. @@ -349,51 +389,51 @@ def get_phase_from_target_method(alias_to_tasks_map, alias_to_phase_map, config) get_signing_cert_scope = functools.partial( get_scope_from_project, - SIGNING_SCOPE_ALIAS_TO_PROJECT, - SIGNING_CERT_SCOPES + alias_to_project_map=SIGNING_SCOPE_ALIAS_TO_PROJECT, + alias_to_scope_map=SIGNING_CERT_SCOPES, ) get_devedition_signing_cert_scope = functools.partial( get_scope_from_project, - DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT, - DEVEDITION_SIGNING_CERT_SCOPES + alias_to_project_map=DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT, + alias_to_scope_map=DEVEDITION_SIGNING_CERT_SCOPES, ) get_beetmover_bucket_scope = functools.partial( get_scope_from_target_method_and_project, - BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK, - BEETMOVER_SCOPE_ALIAS_TO_PROJECT, - BEETMOVER_BUCKET_SCOPES + alias_to_tasks_map=BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK, + alias_to_project_map=BEETMOVER_SCOPE_ALIAS_TO_PROJECT, + aliases_to_scope_map=BEETMOVER_BUCKET_SCOPES, ) get_beetmover_action_scope = functools.partial( get_scope_from_target_method, - BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK, - BEETMOVER_ACTION_SCOPES + alias_to_tasks_map=BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK, + alias_to_scope_map=BEETMOVER_ACTION_SCOPES, ) get_phase = functools.partial( get_phase_from_target_method, - BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK, - PHASES + alias_to_tasks_map=BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK, + alias_to_phase_map=PHASES, ) get_balrog_server_scope = functools.partial( get_scope_from_project, - BALROG_SCOPE_ALIAS_TO_PROJECT, - BALROG_SERVER_SCOPES + alias_to_project_map=BALROG_SCOPE_ALIAS_TO_PROJECT, + alias_to_scope_map=BALROG_SERVER_SCOPES, ) get_balrog_channel_scopes = functools.partial( get_scope_from_project, - BALROG_SCOPE_ALIAS_TO_PROJECT, - BALROG_CHANNEL_SCOPES + alias_to_project_map=BALROG_SCOPE_ALIAS_TO_PROJECT, + alias_to_scope_map=BALROG_CHANNEL_SCOPES, ) get_push_apk_scope = functools.partial( get_scope_from_project, - PUSH_APK_SCOPE_ALIAS_TO_PROJECT, - PUSH_APK_SCOPES + alias_to_project_map=PUSH_APK_SCOPE_ALIAS_TO_PROJECT, + alias_to_scope_map=PUSH_APK_SCOPES, ) @@ -451,7 +491,7 @@ def get_signing_cert_scope_per_platform(build_platform, is_nightly, config): elif is_nightly: return get_signing_cert_scope(config) else: - return 'project:releng:signing:cert:dep-signing' + return add_scope_prefix(config, 'signing:cert:dep-signing') def get_worker_type_for_scope(config, scope):