Bug 1500897 - geckoview: only spin beetmover tasks on GECKOVIEW_XX_RELBRANCH r=dustin,tomprince

geckoview: only spin beetmover tasks on GECKOVIEW_XX_RELBRANCH

Differential Revision: https://phabricator.services.mozilla.com/D9551

--HG--
extra : moz-landing-system : lando

taskcluster/ci/beetmover-geckoview/kind.yml

@@ -30,9 +30,24 @@ not-for-build-platforms:
    - win64-asan-reporter-nightly/opt
 
 job-template:
-   # Beetmoving geckoview makes it available to the official maven repo. So we want beetmover to
-   # act only when the release is greenlit.
-   shipping-phase: ship
+   run-on-projects: ['mozilla-central', 'mozilla-release']
+   run-on-hg-branches:
+      by-project:
+         mozilla-release:
+            - '^GECKOVIEW_\d+_RELBRANCH$'
+         default:
+            - '.*'
+   shipping-phase:
+      by-project:
+         # Beetmoving geckoview makes it available to the official maven repo.
+         # So we want beetmover to act only when the release is greenlit. That
+         # is to say:
+         # - right after nightly builds on mozilla-central
+         # - when Fennec beta was greenlit by QA on mozilla-beta (hence the ship phase)
+         # - at every patch uplifted on the GECKOVIEW_XX_RELBRANC on mozilla-release
+         # Reminder: There is no Android/geckoview build on ESR.
+         mozilla-release: build
+         default: ship
    bucket-scope:
       by-release-level:
          production: 'project:releng:beetmover:bucket:maven-production'

taskcluster/docs/attributes.rst

@@ -37,6 +37,18 @@ be specified by name regardless of ``run_on_projects``.
 If ``run_on_projects`` is set to an empty list, then the task will not run
 anywhere, unless its build platform is specified explicitly in try syntax.
 
+run_on_hg_branches
+==================
+
+On a given project, the mercurial branch where this task should be in the target
+task set.  This is how requirements like "only run this RELBRANCH" get implemented.
+These are either the regular expression of a branch (e.g.: "GECKOVIEW_\d+_RELBRANCH")
+or the following alias:
+
+ * `all` -- everywhere (the default)
+
+Like ``run_on_projects``, the same behavior applies if it is set to an empty list.
+
 task_duplicates
 ===============
 

taskcluster/docs/parameters.rst

@@ -54,6 +54,9 @@ Push Information
    The timestamp of the push to the repository that triggered this decision
    task.  Expressed as an integer seconds since the UNIX epoch.
 
+``hg_branch``
+  The mercurial branch where the revision lives in.
+
 ``build_date``
    The timestamp of the build date. Defaults to ``pushdate`` and falls back to present time of
    taskgraph invocation. Expressed as an integer seconds since the UNIX epoch.

taskcluster/taskgraph/actions/release_promotion.py

@@ -11,8 +11,8 @@ import os
 
 from .registry import register_callback_action
 
-from .util import (find_decision_task, find_existing_tasks_from_previous_kinds,
-                   find_hg_revision_pushlog_id)
+from .util import find_decision_task, find_existing_tasks_from_previous_kinds
+from taskgraph.util.hg import find_hg_revision_pushlog_id
 from taskgraph.util.taskcluster import get_artifact
 from taskgraph.util.partials import populate_release_history
 from taskgraph.util.partners import (

taskcluster/taskgraph/actions/util.py

@@ -8,7 +8,6 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 import logging
-import requests
 import os
 import re
 
@@ -28,8 +27,6 @@ from taskgraph.util.taskcluster import (
 
 logger = logging.getLogger(__name__)
 
-PUSHLOG_TMPL = '{}/json-pushes?version=2&changeset={}&tipsonly=1&full=1'
-
 
 def find_decision_task(parameters, graph_config):
     """Given the parameters for this action, find the taskId of the decision
@@ -40,24 +37,6 @@ def find_decision_task(parameters, graph_config):
         parameters['pushlog_id']))
 
 
-def find_hg_revision_pushlog_id(parameters, graph_config, revision):
-    """Given the parameters for this action and a revision, find the
-    pushlog_id of the revision."""
-
-    repo_param = '{}head_repository'.format(graph_config['project-repo-param-prefix'])
-    pushlog_url = PUSHLOG_TMPL.format(parameters[repo_param], revision)
-    r = requests.get(pushlog_url)
-    r.raise_for_status()
-    pushes = r.json()['pushes'].keys()
-    if len(pushes) != 1:
-        raise RuntimeError(
-            "Unable to find a single pushlog_id for {} revision {}: {}".format(
-                parameters['head_repository'], revision, pushes
-            )
-        )
-    return pushes[0]
-
-
 def find_existing_tasks_from_previous_kinds(full_task_graph, previous_graph_ids,
                                             rebuild_kinds):
     """Given a list of previous decision/action taskIds and kinds to ignore

taskcluster/taskgraph/cron/__init__.py

@@ -15,13 +15,11 @@ import traceback
 import yaml
 
 from . import decision, schema
-from .util import (
-    match_utc,
-    calculate_head_rev
-)
+from .util import match_utc
 from ..create import create_task
 from .. import GECKO
 from taskgraph.util.attributes import match_run_on_projects
+from taskgraph.util.hg import calculate_head_rev
 from taskgraph.util.schema import resolve_keyed_by
 from taskgraph.util.taskcluster import get_session
 

taskcluster/taskgraph/cron/util.py

@@ -7,8 +7,6 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import subprocess
-
 
 def match_utc(params, sched):
     """Return True if params['time'] matches the given schedule.
@@ -38,10 +36,3 @@ def match_utc(params, sched):
         return False
 
     return True
-
-
-def calculate_head_rev(root):
-    # we assume that run-task has correctly checked out the revision indicated by
-    # GECKO_HEAD_REF, so all that remains is to see what the current revision is.
-    # Mercurial refers to that as `.`.
-    return subprocess.check_output(['hg', 'log', '-r', '.', '-T', '{node}'], cwd=root)

taskcluster/taskgraph/decision.py

@@ -12,18 +12,20 @@ import logging
 import time
 import yaml
 
-from .generator import TaskGraphGenerator
+from . import GECKO
+from .actions import render_actions_json
 from .create import create_tasks
+from .generator import TaskGraphGenerator
 from .parameters import Parameters, get_version, get_app_version
 from .taskgraph import TaskGraph
 from .try_option_syntax import parse_message
-from .actions import render_actions_json
-from .util.partials import populate_release_history
-from .util.yaml import load_yaml
-
 from .util.schema import validate_schema, Schema
+from taskgraph.util.hg import get_hg_revision_branch
+from taskgraph.util.partials import populate_release_history
+from taskgraph.util.yaml import load_yaml
 from voluptuous import Required, Optional
 
+
 logger = logging.getLogger(__name__)
 
 ARTIFACTS_DIR = 'artifacts'
@@ -192,6 +194,7 @@ def get_decision_parameters(config, options):
 
     """
     product_dir = config['product-dir']
+    root = options.get('root') or GECKO
 
     parameters = {n: options[n] for n in [
         'base_repository',
@@ -226,6 +229,7 @@ def get_decision_parameters(config, options):
     parameters['build_number'] = 1
     parameters['version'] = get_version(product_dir)
     parameters['app_version'] = get_app_version(product_dir)
+    parameters['hg_branch'] = get_hg_revision_branch(root, revision=parameters['head_rev'])
     parameters['next_version'] = None
     parameters['release_type'] = ''
     parameters['release_eta'] = ''

taskcluster/taskgraph/parameters.py

@@ -59,6 +59,7 @@ PARAMETERS = {
     'head_ref': get_head_ref,
     'head_repository': 'https://hg.mozilla.org/mozilla-central',
     'head_rev': get_head_ref,
+    'hg_branch': 'default',
     'level': '3',
     'message': '',
     'moz_build_date': lambda: datetime.now().strftime("%Y%m%d%H%M%S"),

taskcluster/taskgraph/target_tasks.py

@@ -7,7 +7,7 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph import try_option_syntax
-from taskgraph.util.attributes import match_run_on_projects
+from taskgraph.util.attributes import match_run_on_projects, match_run_on_hg_branches
 
 _target_task_methods = {}
 
@@ -41,6 +41,13 @@ def filter_for_project(task, parameters):
     return match_run_on_projects(parameters['project'], run_on_projects)
 
 
+def filter_for_hg_branch(task, parameters):
+    """Filter tasks by hg branch.
+    If `run_on_hg_branch` is not defined, then task runs on all branches"""
+    run_on_hg_branches = set(task.attributes.get('run_on_hg_branches', ['all']))
+    return match_run_on_hg_branches(parameters['hg_branch'], run_on_hg_branches)
+
+
 def filter_on_platforms(task, platforms):
     """Filter tasks on the given platform"""
     platform = task.attributes.get('build_platform')
@@ -81,7 +88,7 @@ def filter_release_tasks(task, parameters):
 def standard_filter(task, parameters):
     return all(
         filter_func(task, parameters) for filter_func in
-        (filter_out_cron, filter_for_project)
+        (filter_out_cron, filter_for_project, filter_for_hg_branch)
     )
 
 

taskcluster/taskgraph/test/test_decision.py

@@ -11,8 +11,9 @@ import shutil
 import unittest
 import tempfile
 
-from taskgraph import decision
+from mock import patch
 from mozunit import main, MockedOpen
+from taskgraph import decision
 
 
 FAKE_GRAPH_CONFIG = {'product-dir': 'browser'}
@@ -65,23 +66,28 @@ class TestGetDecisionParameters(unittest.TestCase):
             'level': 3,
         }
 
-    def test_simple_options(self):
+    @patch('taskgraph.decision.get_hg_revision_branch')
+    def test_simple_options(self, mock_get_hg_revision_branch):
+        mock_get_hg_revision_branch.return_value = 'default'
         with MockedOpen({self.ttc_file: None}):
             params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
         self.assertEqual(params['pushlog_id'], 143)
         self.assertEqual(params['build_date'], 1503691511)
+        self.assertEqual(params['hg_branch'], 'default')
         self.assertEqual(params['moz_build_date'], '20170825200511')
         self.assertEqual(params['try_mode'], None)
         self.assertEqual(params['try_options'], None)
         self.assertEqual(params['try_task_config'], None)
 
-    def test_no_email_owner(self):
+    @patch('taskgraph.decision.get_hg_revision_branch')
+    def test_no_email_owner(self, _):
         self.options['owner'] = 'ffxbld'
         with MockedOpen({self.ttc_file: None}):
             params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
         self.assertEqual(params['owner'], 'ffxbld@noreply.mozilla.org')
 
-    def test_try_options(self):
+    @patch('taskgraph.decision.get_hg_revision_branch')
+    def test_try_options(self, _):
         self.options['message'] = 'try: -b do -t all'
         self.options['project'] = 'try'
         with MockedOpen({self.ttc_file: None}):
@@ -91,7 +97,8 @@ class TestGetDecisionParameters(unittest.TestCase):
         self.assertEqual(params['try_options']['unittests'], 'all')
         self.assertEqual(params['try_task_config'], None)
 
-    def test_try_task_config(self):
+    @patch('taskgraph.decision.get_hg_revision_branch')
+    def test_try_task_config(self, _):
         ttc = {'tasks': ['a', 'b'], 'templates': {}}
         self.options['project'] = 'try'
         with MockedOpen({self.ttc_file: json.dumps(ttc)}):

taskcluster/taskgraph/transforms/beetmover_geckoview.py

@@ -42,8 +42,13 @@ beetmover_description_schema = schema.extend({
     Optional('label'): basestring,
     Optional('treeherder'): task_description_schema['treeherder'],
 
+    Required('run-on-projects'): task_description_schema['run-on-projects'],
+    Required('run-on-hg-branches'): task_description_schema['run-on-hg-branches'],
+
     Optional('bucket-scope'): optionally_keyed_by('release-level', basestring),
-    Optional('shipping-phase'): task_description_schema['shipping-phase'],
+    Optional('shipping-phase'): optionally_keyed_by(
+        'project', task_description_schema['shipping-phase']
+    ),
     Optional('shipping-product'): task_description_schema['shipping-product'],
 })
 
@@ -58,6 +63,22 @@ def validate(config, jobs):
         yield job
 
 
+@transforms.add
+def resolve_keys(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job, 'run-on-hg-branches', item_name=job['label'], project=config.params['project']
+        )
+        resolve_keyed_by(
+            job, 'shipping-phase', item_name=job['label'], project=config.params['project']
+        )
+        resolve_keyed_by(
+            job, 'bucket-scope', item_name=job['label'],
+            **{'release-level': config.params.release_level()}
+        )
+        yield job
+
+
 @transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
@@ -89,10 +110,7 @@ def make_task_description(config, jobs):
         if job.get('locale'):
             attributes['locale'] = job['locale']
 
-        resolve_keyed_by(
-            job, 'bucket-scope', item_name=job['label'],
-            **{'release-level': config.params.release_level()}
-        )
+        attributes['run_on_hg_branches'] = job['run-on-hg-branches']
 
         task = {
             'label': label,
@@ -101,7 +119,7 @@ def make_task_description(config, jobs):
             'scopes': [job['bucket-scope'], 'project:releng:beetmover:action:push-to-maven'],
             'dependencies': dependencies,
             'attributes': attributes,
-            'run-on-projects': ['mozilla-central'],
+            'run-on-projects': job['run-on-projects'],
             'treeherder': treeherder,
             'shipping-phase': job['shipping-phase'],
         }

taskcluster/taskgraph/transforms/task.py

@@ -157,6 +157,9 @@ task_description_schema = Schema({
     # See the attributes documentation for details.
     Optional('run-on-projects'): optionally_keyed_by('build-platform', [basestring]),
 
+    # Like `run_on_projects`, `run-on-hg-branches` defaults to "all".
+    Optional('run-on-hg-branches'): optionally_keyed_by('project', [basestring]),
+
     # The `shipping_phase` attribute, defaulting to None. This specifies the
     # release promotion phase that this task belongs to.
     Required('shipping-phase'): Any(

taskcluster/taskgraph/util/attributes.py

@@ -105,6 +105,19 @@ def match_run_on_projects(project, run_on_projects):
     return project in run_on_projects
 
 
+def match_run_on_hg_branches(hg_branch, run_on_hg_branches):
+    """Determine whether the given project is included in the `run-on-hg-branches`
+    parameter. Allows 'all'."""
+    if 'all' in run_on_hg_branches:
+        return True
+
+    for expected_hg_branch_pattern in run_on_hg_branches:
+        if re.match(expected_hg_branch_pattern, hg_branch):
+            return True
+
+    return False
+
+
 def copy_attributes_from_dependent_job(dep_job):
     attributes = {
         'build_platform': dep_job.attributes.get('build_platform'),

taskcluster/taskgraph/util/hg.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import requests
+import subprocess
+
+PUSHLOG_TMPL = '{}/json-pushes?version=2&changeset={}&tipsonly=1&full=1'
+
+
+def find_hg_revision_pushlog_id(parameters, graph_config, revision):
+    """Given the parameters for this action and a revision, find the
+    pushlog_id of the revision."""
+    repo_param = '{}head_repository'.format(graph_config['project-repo-param-prefix'])
+    pushlog_url = PUSHLOG_TMPL.format(parameters[repo_param], revision)
+    r = requests.get(pushlog_url)
+    r.raise_for_status()
+    pushes = r.json()['pushes'].keys()
+    if len(pushes) != 1:
+        raise RuntimeError(
+            "Unable to find a single pushlog_id for {} revision {}: {}".format(
+                parameters['head_repository'], revision, pushes
+            )
+        )
+    return pushes[0]
+
+
+def get_hg_revision_branch(root, revision):
+    """Given the parameters for a revision, find the hg_branch (aka
+    relbranch) of the revision."""
+    return subprocess.check_output(['hg', 'identify', '--branch', '--rev', revision], cwd=root)
+
+
+def calculate_head_rev(root):
+    # we assume that run-task has correctly checked out the revision indicated by
+    # GECKO_HEAD_REF, so all that remains is to see what the current revision is.
+    # Mercurial refers to that as `.`.
+    return subprocess.check_output(['hg', 'log', '-r', '.', '-T', '{node}'], cwd=root)