diff --git a/taskcluster/ci/beetmover-geckoview/kind.yml b/taskcluster/ci/beetmover-geckoview/kind.yml index 2f8deb2fd092..b467e6fc0ed5 100644 --- a/taskcluster/ci/beetmover-geckoview/kind.yml +++ b/taskcluster/ci/beetmover-geckoview/kind.yml @@ -30,9 +30,24 @@ not-for-build-platforms: - win64-asan-reporter-nightly/opt job-template: - # Beetmoving geckoview makes it available to the official maven repo. So we want beetmover to - # act only when the release is greenlit. - shipping-phase: ship + run-on-projects: ['mozilla-central', 'mozilla-release'] + run-on-hg-branches: + by-project: + mozilla-release: + - '^GECKOVIEW_\d+_RELBRANCH$' + default: + - '.*' + shipping-phase: + by-project: + # Beetmoving geckoview makes it available to the official maven repo. + # So we want beetmover to act only when the release is greenlit. That + # is to say: + # - right after nightly builds on mozilla-central + # - when Fennec beta was greenlit by QA on mozilla-beta (hence the ship phase) + # - at every patch uplifted on the GECKOVIEW_XX_RELBRANC on mozilla-release + # Reminder: There is no Android/geckoview build on ESR. + mozilla-release: build + default: ship bucket-scope: by-release-level: production: 'project:releng:beetmover:bucket:maven-production' diff --git a/taskcluster/docs/attributes.rst b/taskcluster/docs/attributes.rst index 1749135ac09f..8fc75ec5fe4a 100644 --- a/taskcluster/docs/attributes.rst +++ b/taskcluster/docs/attributes.rst @@ -37,6 +37,18 @@ be specified by name regardless of ``run_on_projects``. If ``run_on_projects`` is set to an empty list, then the task will not run anywhere, unless its build platform is specified explicitly in try syntax. +run_on_hg_branches +================== + +On a given project, the mercurial branch where this task should be in the target +task set. This is how requirements like "only run this RELBRANCH" get implemented. +These are either the regular expression of a branch (e.g.: "GECKOVIEW_\d+_RELBRANCH") +or the following alias: + + * `all` -- everywhere (the default) + +Like ``run_on_projects``, the same behavior applies if it is set to an empty list. + task_duplicates =============== diff --git a/taskcluster/docs/parameters.rst b/taskcluster/docs/parameters.rst index 7fc530a5d629..77f4e72658a0 100644 --- a/taskcluster/docs/parameters.rst +++ b/taskcluster/docs/parameters.rst @@ -54,6 +54,9 @@ Push Information The timestamp of the push to the repository that triggered this decision task. Expressed as an integer seconds since the UNIX epoch. +``hg_branch`` + The mercurial branch where the revision lives in. + ``build_date`` The timestamp of the build date. Defaults to ``pushdate`` and falls back to present time of taskgraph invocation. Expressed as an integer seconds since the UNIX epoch. diff --git a/taskcluster/taskgraph/actions/release_promotion.py b/taskcluster/taskgraph/actions/release_promotion.py index 45a5f9f054be..06b3b8780d41 100644 --- a/taskcluster/taskgraph/actions/release_promotion.py +++ b/taskcluster/taskgraph/actions/release_promotion.py @@ -11,8 +11,8 @@ import os from .registry import register_callback_action -from .util import (find_decision_task, find_existing_tasks_from_previous_kinds, - find_hg_revision_pushlog_id) +from .util import find_decision_task, find_existing_tasks_from_previous_kinds +from taskgraph.util.hg import find_hg_revision_pushlog_id from taskgraph.util.taskcluster import get_artifact from taskgraph.util.partials import populate_release_history from taskgraph.util.partners import ( diff --git a/taskcluster/taskgraph/actions/util.py b/taskcluster/taskgraph/actions/util.py index bd407950222c..b084ae11bde4 100644 --- a/taskcluster/taskgraph/actions/util.py +++ b/taskcluster/taskgraph/actions/util.py @@ -8,7 +8,6 @@ from __future__ import absolute_import, print_function, unicode_literals import copy import logging -import requests import os import re @@ -28,8 +27,6 @@ from taskgraph.util.taskcluster import ( logger = logging.getLogger(__name__) -PUSHLOG_TMPL = '{}/json-pushes?version=2&changeset={}&tipsonly=1&full=1' - def find_decision_task(parameters, graph_config): """Given the parameters for this action, find the taskId of the decision @@ -40,24 +37,6 @@ def find_decision_task(parameters, graph_config): parameters['pushlog_id'])) -def find_hg_revision_pushlog_id(parameters, graph_config, revision): - """Given the parameters for this action and a revision, find the - pushlog_id of the revision.""" - - repo_param = '{}head_repository'.format(graph_config['project-repo-param-prefix']) - pushlog_url = PUSHLOG_TMPL.format(parameters[repo_param], revision) - r = requests.get(pushlog_url) - r.raise_for_status() - pushes = r.json()['pushes'].keys() - if len(pushes) != 1: - raise RuntimeError( - "Unable to find a single pushlog_id for {} revision {}: {}".format( - parameters['head_repository'], revision, pushes - ) - ) - return pushes[0] - - def find_existing_tasks_from_previous_kinds(full_task_graph, previous_graph_ids, rebuild_kinds): """Given a list of previous decision/action taskIds and kinds to ignore diff --git a/taskcluster/taskgraph/cron/__init__.py b/taskcluster/taskgraph/cron/__init__.py index b46d742d1ef0..a941586acc6a 100644 --- a/taskcluster/taskgraph/cron/__init__.py +++ b/taskcluster/taskgraph/cron/__init__.py @@ -15,13 +15,11 @@ import traceback import yaml from . import decision, schema -from .util import ( - match_utc, - calculate_head_rev -) +from .util import match_utc from ..create import create_task from .. import GECKO from taskgraph.util.attributes import match_run_on_projects +from taskgraph.util.hg import calculate_head_rev from taskgraph.util.schema import resolve_keyed_by from taskgraph.util.taskcluster import get_session diff --git a/taskcluster/taskgraph/cron/util.py b/taskcluster/taskgraph/cron/util.py index e52464e9968c..460e53884f30 100644 --- a/taskcluster/taskgraph/cron/util.py +++ b/taskcluster/taskgraph/cron/util.py @@ -7,8 +7,6 @@ from __future__ import absolute_import, print_function, unicode_literals -import subprocess - def match_utc(params, sched): """Return True if params['time'] matches the given schedule. @@ -38,10 +36,3 @@ def match_utc(params, sched): return False return True - - -def calculate_head_rev(root): - # we assume that run-task has correctly checked out the revision indicated by - # GECKO_HEAD_REF, so all that remains is to see what the current revision is. - # Mercurial refers to that as `.`. - return subprocess.check_output(['hg', 'log', '-r', '.', '-T', '{node}'], cwd=root) diff --git a/taskcluster/taskgraph/decision.py b/taskcluster/taskgraph/decision.py index 5d81a3fc0b7d..a71992cef586 100644 --- a/taskcluster/taskgraph/decision.py +++ b/taskcluster/taskgraph/decision.py @@ -12,18 +12,20 @@ import logging import time import yaml -from .generator import TaskGraphGenerator +from . import GECKO +from .actions import render_actions_json from .create import create_tasks +from .generator import TaskGraphGenerator from .parameters import Parameters, get_version, get_app_version from .taskgraph import TaskGraph from .try_option_syntax import parse_message -from .actions import render_actions_json -from .util.partials import populate_release_history -from .util.yaml import load_yaml - from .util.schema import validate_schema, Schema +from taskgraph.util.hg import get_hg_revision_branch +from taskgraph.util.partials import populate_release_history +from taskgraph.util.yaml import load_yaml from voluptuous import Required, Optional + logger = logging.getLogger(__name__) ARTIFACTS_DIR = 'artifacts' @@ -192,6 +194,7 @@ def get_decision_parameters(config, options): """ product_dir = config['product-dir'] + root = options.get('root') or GECKO parameters = {n: options[n] for n in [ 'base_repository', @@ -226,6 +229,7 @@ def get_decision_parameters(config, options): parameters['build_number'] = 1 parameters['version'] = get_version(product_dir) parameters['app_version'] = get_app_version(product_dir) + parameters['hg_branch'] = get_hg_revision_branch(root, revision=parameters['head_rev']) parameters['next_version'] = None parameters['release_type'] = '' parameters['release_eta'] = '' diff --git a/taskcluster/taskgraph/parameters.py b/taskcluster/taskgraph/parameters.py index 750b4ec8953e..544f17e10953 100644 --- a/taskcluster/taskgraph/parameters.py +++ b/taskcluster/taskgraph/parameters.py @@ -59,6 +59,7 @@ PARAMETERS = { 'head_ref': get_head_ref, 'head_repository': 'https://hg.mozilla.org/mozilla-central', 'head_rev': get_head_ref, + 'hg_branch': 'default', 'level': '3', 'message': '', 'moz_build_date': lambda: datetime.now().strftime("%Y%m%d%H%M%S"), diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py index 1c311ecd1be3..94b1a65f7f2f 100644 --- a/taskcluster/taskgraph/target_tasks.py +++ b/taskcluster/taskgraph/target_tasks.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, print_function, unicode_literals from taskgraph import try_option_syntax -from taskgraph.util.attributes import match_run_on_projects +from taskgraph.util.attributes import match_run_on_projects, match_run_on_hg_branches _target_task_methods = {} @@ -41,6 +41,13 @@ def filter_for_project(task, parameters): return match_run_on_projects(parameters['project'], run_on_projects) +def filter_for_hg_branch(task, parameters): + """Filter tasks by hg branch. + If `run_on_hg_branch` is not defined, then task runs on all branches""" + run_on_hg_branches = set(task.attributes.get('run_on_hg_branches', ['all'])) + return match_run_on_hg_branches(parameters['hg_branch'], run_on_hg_branches) + + def filter_on_platforms(task, platforms): """Filter tasks on the given platform""" platform = task.attributes.get('build_platform') @@ -81,7 +88,7 @@ def filter_release_tasks(task, parameters): def standard_filter(task, parameters): return all( filter_func(task, parameters) for filter_func in - (filter_out_cron, filter_for_project) + (filter_out_cron, filter_for_project, filter_for_hg_branch) ) diff --git a/taskcluster/taskgraph/test/test_decision.py b/taskcluster/taskgraph/test/test_decision.py index cdbdd40c8296..ca6a6c40272b 100644 --- a/taskcluster/taskgraph/test/test_decision.py +++ b/taskcluster/taskgraph/test/test_decision.py @@ -11,8 +11,9 @@ import shutil import unittest import tempfile -from taskgraph import decision +from mock import patch from mozunit import main, MockedOpen +from taskgraph import decision FAKE_GRAPH_CONFIG = {'product-dir': 'browser'} @@ -65,23 +66,28 @@ class TestGetDecisionParameters(unittest.TestCase): 'level': 3, } - def test_simple_options(self): + @patch('taskgraph.decision.get_hg_revision_branch') + def test_simple_options(self, mock_get_hg_revision_branch): + mock_get_hg_revision_branch.return_value = 'default' with MockedOpen({self.ttc_file: None}): params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) self.assertEqual(params['pushlog_id'], 143) self.assertEqual(params['build_date'], 1503691511) + self.assertEqual(params['hg_branch'], 'default') self.assertEqual(params['moz_build_date'], '20170825200511') self.assertEqual(params['try_mode'], None) self.assertEqual(params['try_options'], None) self.assertEqual(params['try_task_config'], None) - def test_no_email_owner(self): + @patch('taskgraph.decision.get_hg_revision_branch') + def test_no_email_owner(self, _): self.options['owner'] = 'ffxbld' with MockedOpen({self.ttc_file: None}): params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) self.assertEqual(params['owner'], 'ffxbld@noreply.mozilla.org') - def test_try_options(self): + @patch('taskgraph.decision.get_hg_revision_branch') + def test_try_options(self, _): self.options['message'] = 'try: -b do -t all' self.options['project'] = 'try' with MockedOpen({self.ttc_file: None}): @@ -91,7 +97,8 @@ class TestGetDecisionParameters(unittest.TestCase): self.assertEqual(params['try_options']['unittests'], 'all') self.assertEqual(params['try_task_config'], None) - def test_try_task_config(self): + @patch('taskgraph.decision.get_hg_revision_branch') + def test_try_task_config(self, _): ttc = {'tasks': ['a', 'b'], 'templates': {}} self.options['project'] = 'try' with MockedOpen({self.ttc_file: json.dumps(ttc)}): diff --git a/taskcluster/taskgraph/transforms/beetmover_geckoview.py b/taskcluster/taskgraph/transforms/beetmover_geckoview.py index c91074f35e14..809233f99c6d 100644 --- a/taskcluster/taskgraph/transforms/beetmover_geckoview.py +++ b/taskcluster/taskgraph/transforms/beetmover_geckoview.py @@ -42,8 +42,13 @@ beetmover_description_schema = schema.extend({ Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], + Required('run-on-projects'): task_description_schema['run-on-projects'], + Required('run-on-hg-branches'): task_description_schema['run-on-hg-branches'], + Optional('bucket-scope'): optionally_keyed_by('release-level', basestring), - Optional('shipping-phase'): task_description_schema['shipping-phase'], + Optional('shipping-phase'): optionally_keyed_by( + 'project', task_description_schema['shipping-phase'] + ), Optional('shipping-product'): task_description_schema['shipping-product'], }) @@ -58,6 +63,22 @@ def validate(config, jobs): yield job +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + resolve_keyed_by( + job, 'run-on-hg-branches', item_name=job['label'], project=config.params['project'] + ) + resolve_keyed_by( + job, 'shipping-phase', item_name=job['label'], project=config.params['project'] + ) + resolve_keyed_by( + job, 'bucket-scope', item_name=job['label'], + **{'release-level': config.params.release_level()} + ) + yield job + + @transforms.add def make_task_description(config, jobs): for job in jobs: @@ -89,10 +110,7 @@ def make_task_description(config, jobs): if job.get('locale'): attributes['locale'] = job['locale'] - resolve_keyed_by( - job, 'bucket-scope', item_name=job['label'], - **{'release-level': config.params.release_level()} - ) + attributes['run_on_hg_branches'] = job['run-on-hg-branches'] task = { 'label': label, @@ -101,7 +119,7 @@ def make_task_description(config, jobs): 'scopes': [job['bucket-scope'], 'project:releng:beetmover:action:push-to-maven'], 'dependencies': dependencies, 'attributes': attributes, - 'run-on-projects': ['mozilla-central'], + 'run-on-projects': job['run-on-projects'], 'treeherder': treeherder, 'shipping-phase': job['shipping-phase'], } diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py index 8245902cc8e1..3e9f43d7d22c 100644 --- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -157,6 +157,9 @@ task_description_schema = Schema({ # See the attributes documentation for details. Optional('run-on-projects'): optionally_keyed_by('build-platform', [basestring]), + # Like `run_on_projects`, `run-on-hg-branches` defaults to "all". + Optional('run-on-hg-branches'): optionally_keyed_by('project', [basestring]), + # The `shipping_phase` attribute, defaulting to None. This specifies the # release promotion phase that this task belongs to. Required('shipping-phase'): Any( diff --git a/taskcluster/taskgraph/util/attributes.py b/taskcluster/taskgraph/util/attributes.py index 85543f6a9e59..3a0e5ccb52d4 100644 --- a/taskcluster/taskgraph/util/attributes.py +++ b/taskcluster/taskgraph/util/attributes.py @@ -105,6 +105,19 @@ def match_run_on_projects(project, run_on_projects): return project in run_on_projects +def match_run_on_hg_branches(hg_branch, run_on_hg_branches): + """Determine whether the given project is included in the `run-on-hg-branches` + parameter. Allows 'all'.""" + if 'all' in run_on_hg_branches: + return True + + for expected_hg_branch_pattern in run_on_hg_branches: + if re.match(expected_hg_branch_pattern, hg_branch): + return True + + return False + + def copy_attributes_from_dependent_job(dep_job): attributes = { 'build_platform': dep_job.attributes.get('build_platform'), diff --git a/taskcluster/taskgraph/util/hg.py b/taskcluster/taskgraph/util/hg.py new file mode 100644 index 000000000000..75b60a06b56d --- /dev/null +++ b/taskcluster/taskgraph/util/hg.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import requests +import subprocess + +PUSHLOG_TMPL = '{}/json-pushes?version=2&changeset={}&tipsonly=1&full=1' + + +def find_hg_revision_pushlog_id(parameters, graph_config, revision): + """Given the parameters for this action and a revision, find the + pushlog_id of the revision.""" + repo_param = '{}head_repository'.format(graph_config['project-repo-param-prefix']) + pushlog_url = PUSHLOG_TMPL.format(parameters[repo_param], revision) + r = requests.get(pushlog_url) + r.raise_for_status() + pushes = r.json()['pushes'].keys() + if len(pushes) != 1: + raise RuntimeError( + "Unable to find a single pushlog_id for {} revision {}: {}".format( + parameters['head_repository'], revision, pushes + ) + ) + return pushes[0] + + +def get_hg_revision_branch(root, revision): + """Given the parameters for a revision, find the hg_branch (aka + relbranch) of the revision.""" + return subprocess.check_output(['hg', 'identify', '--branch', '--rev', revision], cwd=root) + + +def calculate_head_rev(root): + # we assume that run-task has correctly checked out the revision indicated by + # GECKO_HEAD_REF, so all that remains is to see what the current revision is. + # Mercurial refers to that as `.`. + return subprocess.check_output(['hg', 'log', '-r', '.', '-T', '{node}'], cwd=root)