From 1265267ba8d37267c3fe2c044f10dd69aa68b52a Mon Sep 17 00:00:00 2001 From: Mihai Alexandru Michis Date: Tue, 28 May 2019 13:12:23 +0300 Subject: [PATCH] Backed out changeset 48687a76aec5 (bug 1535056) for causing nightly builds to fail. a=backout --- taskcluster/taskgraph/parameters.py | 206 ++++++++---------- taskcluster/taskgraph/test/test_decision.py | 15 +- taskcluster/taskgraph/test/test_parameters.py | 61 +----- 3 files changed, 101 insertions(+), 181 deletions(-) diff --git a/taskcluster/taskgraph/parameters.py b/taskcluster/taskgraph/parameters.py index 8f322dae5621..dfbfb7e5fbce 100644 --- a/taskcluster/taskgraph/parameters.py +++ b/taskcluster/taskgraph/parameters.py @@ -8,19 +8,11 @@ from __future__ import absolute_import, print_function, unicode_literals import os.path import json +import time from datetime import datetime from mozbuild.util import ReadOnlyDict, memoize from mozversioncontrol import get_repository_object -from taskgraph.util.schema import validate_schema -from voluptuous import ( - ALLOW_EXTRA, - Any, - Inclusive, - PREVENT_EXTRA, - Required, - Schema, -) from . import GECKO from .util.attributes import release_level @@ -53,58 +45,55 @@ def get_app_version(product_dir='browser'): return get_contents(app_version_path) -base_schema = { - Required('app_version'): basestring, - Required('base_repository'): basestring, - Required('build_date'): int, - Required('build_number'): int, - Inclusive('comm_base_repository', 'comm'): basestring, - Inclusive('comm_head_ref', 'comm'): basestring, - Inclusive('comm_head_repository', 'comm'): basestring, - Inclusive('comm_head_rev', 'comm'): basestring, - Required('do_not_optimize'): [basestring], - Required('existing_tasks'): {basestring: basestring}, - Required('filters'): [basestring], - Required('head_ref'): basestring, - Required('head_repository'): basestring, - Required('head_rev'): basestring, - Required('hg_branch'): basestring, - Required('level'): basestring, - Required('message'): basestring, - Required('moz_build_date'): basestring, - Required('next_version'): Any(None, basestring), - Required('optimize_target_tasks'): bool, - Required('owner'): basestring, - Required('phabricator_diff'): Any(None, basestring), - Required('project'): basestring, - Required('pushdate'): int, - Required('pushlog_id'): basestring, - Required('release_enable_emefree'): bool, - Required('release_enable_partners'): bool, - Required('release_eta'): Any(None, basestring), - Required('release_history'): {basestring: [dict]}, - Required('release_partners'): [basestring], - Required('release_partner_config'): Any(None, dict), - Required('release_partner_build_number'): int, - Required('release_type'): basestring, - Required('release_product'): Any(None, basestring), - Required('required_signoffs'): [basestring], - Required('signoff_urls'): dict, - Required('target_tasks_method'): Any(None, basestring), - Required('tasks_for'): Any(None, basestring), - Required('try_mode'): Any(None, basestring), - Required('try_options'): Any(None, dict), - Required('try_task_config'): Any(None, dict), - Required('version'): basestring, +# Please keep this list sorted and in sync with taskcluster/docs/parameters.rst +# Parameters are of the form: {name: default} or {name: lambda: default} +PARAMETERS = { + 'app_version': get_app_version(), + 'base_repository': 'https://hg.mozilla.org/mozilla-unified', + 'build_date': lambda: int(time.time()), + 'build_number': 1, + 'do_not_optimize': [], + 'existing_tasks': {}, + 'filters': ['target_tasks_method'], + 'head_ref': get_head_ref, + 'head_repository': 'https://hg.mozilla.org/mozilla-central', + 'head_rev': get_head_ref, + 'hg_branch': 'default', + 'level': '3', + 'message': '', + 'moz_build_date': lambda: datetime.now().strftime("%Y%m%d%H%M%S"), + 'next_version': None, + 'optimize_target_tasks': True, + 'owner': 'nobody@mozilla.com', + 'project': 'mozilla-central', + 'pushdate': lambda: int(time.time()), + 'pushlog_id': '0', + 'phabricator_diff': None, + 'release_enable_emefree': False, + 'release_enable_partners': False, + 'release_eta': '', + 'release_history': {}, + 'release_partners': None, + 'release_partner_config': None, + 'release_partner_build_number': 1, + 'release_type': 'nightly', + 'release_product': None, + 'required_signoffs': [], + 'signoff_urls': {}, + 'target_tasks_method': 'default', + 'tasks_for': 'hg-push', + 'try_mode': None, + 'try_options': None, + 'try_task_config': None, + 'version': get_version(), } - -COMM_PARAMETERS = [ - 'comm_base_repository', - 'comm_head_ref', - 'comm_head_repository', - 'comm_head_rev', -] +COMM_PARAMETERS = { + 'comm_base_repository': 'https://hg.mozilla.org/comm-central', + 'comm_head_ref': None, + 'comm_head_repository': 'https://hg.mozilla.org/comm-central', + 'comm_head_rev': None, +} class Parameters(ReadOnlyDict): @@ -115,74 +104,49 @@ class Parameters(ReadOnlyDict): if not self.strict: # apply defaults to missing parameters - kwargs = Parameters._fill_defaults(**kwargs) + for name, default in PARAMETERS.items(): + if name not in kwargs: + if callable(default): + default = default() + kwargs[name] = default + + if set(kwargs) & set(COMM_PARAMETERS.keys()): + for name, default in COMM_PARAMETERS.items(): + if name not in kwargs: + if callable(default): + default = default() + kwargs[name] = default ReadOnlyDict.__init__(self, **kwargs) - @staticmethod - def _fill_defaults(**kwargs): - now = datetime.utcnow() - epoch = datetime.utcfromtimestamp(0) - seconds_from_epoch = int((now - epoch).total_seconds()) - - defaults = { - 'app_version': get_app_version(), - 'base_repository': 'https://hg.mozilla.org/mozilla-unified', - 'build_date': seconds_from_epoch, - 'build_number': 1, - 'do_not_optimize': [], - 'existing_tasks': {}, - 'filters': ['target_tasks_method'], - 'head_ref': get_head_ref(), - 'head_repository': 'https://hg.mozilla.org/mozilla-central', - 'head_rev': get_head_ref(), - 'hg_branch': 'default', - 'level': '3', - 'message': '', - 'moz_build_date': now.strftime("%Y%m%d%H%M%S"), - 'next_version': None, - 'optimize_target_tasks': True, - 'owner': 'nobody@mozilla.com', - 'phabricator_diff': None, - 'project': 'mozilla-central', - 'pushdate': seconds_from_epoch, - 'pushlog_id': '0', - 'release_enable_emefree': False, - 'release_enable_partners': False, - 'release_eta': '', - 'release_history': {}, - 'release_partners': [], - 'release_partner_config': None, - 'release_partner_build_number': 1, - 'release_product': None, - 'release_type': 'nightly', - 'required_signoffs': [], - 'signoff_urls': {}, - 'target_tasks_method': 'default', - 'tasks_for': 'hg-push', - 'try_mode': None, - 'try_options': None, - 'try_task_config': None, - 'version': get_version(), - } - - if set(COMM_PARAMETERS) & set(kwargs): - defaults.update({ - 'comm_base_repository': 'https://hg.mozilla.org/comm-central', - 'comm_head_repository': 'https://hg.mozilla.org/comm-central', - }) - - for name, default in defaults.items(): - if name not in kwargs: - kwargs[name] = default - - return kwargs - def check(self): - schema = Schema(base_schema, extra=PREVENT_EXTRA if self.strict else ALLOW_EXTRA) - validate_schema(schema, self.copy(), 'Invalid parameters:') + names = set(self) + valid = set(PARAMETERS.keys()) + valid_comm = set(COMM_PARAMETERS.keys()) + msg = [] + + missing = valid - names + if missing: + msg.append("missing parameters: " + ", ".join(missing)) + + extra = names - valid + + if extra & set(valid_comm): + # If any comm_* parameters are specified, ensure all of them are specified. + missing = valid_comm - extra + if missing: + msg.append("missing parameters: " + ", ".join(missing)) + extra = extra - valid_comm + + if extra and self.strict: + msg.append("extra parameters: " + ", ".join(extra)) + + if msg: + raise ParameterMismatch("; ".join(msg)) def __getitem__(self, k): + if not (k in PARAMETERS.keys() or k in COMM_PARAMETERS.keys()): + raise KeyError("no such parameter {!r}".format(k)) try: return super(Parameters, self).__getitem__(k) except KeyError: diff --git a/taskcluster/taskgraph/test/test_decision.py b/taskcluster/taskgraph/test/test_decision.py index 05aab4900e78..1e4bcd715869 100644 --- a/taskcluster/taskgraph/test/test_decision.py +++ b/taskcluster/taskgraph/test/test_decision.py @@ -59,11 +59,11 @@ class TestGetDecisionParameters(unittest.TestCase): 'head_ref': 'ef01', 'message': '', 'project': 'mozilla-central', - 'pushlog_id': '143', + 'pushlog_id': 143, 'pushdate': 1503691511, 'owner': 'nobody@mozilla.com', 'tasks_for': 'hg-push', - 'level': '3', + 'level': 3, } @patch('taskgraph.decision.get_hg_revision_branch') @@ -71,7 +71,7 @@ class TestGetDecisionParameters(unittest.TestCase): mock_get_hg_revision_branch.return_value = 'default' with MockedOpen({self.ttc_file: None}): params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) - self.assertEqual(params['pushlog_id'], '143') + self.assertEqual(params['pushlog_id'], 143) self.assertEqual(params['build_date'], 1503691511) self.assertEqual(params['hg_branch'], 'default') self.assertEqual(params['moz_build_date'], '20170825200511') @@ -80,8 +80,7 @@ class TestGetDecisionParameters(unittest.TestCase): self.assertEqual(params['try_task_config'], None) @patch('taskgraph.decision.get_hg_revision_branch') - def test_no_email_owner(self, mock_get_hg_revision_branch): - mock_get_hg_revision_branch.return_value = 'default' + def test_no_email_owner(self, _): self.options['owner'] = 'ffxbld' with MockedOpen({self.ttc_file: None}): params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) @@ -89,9 +88,8 @@ class TestGetDecisionParameters(unittest.TestCase): @patch('taskgraph.decision.get_hg_revision_branch') @patch('taskgraph.decision.get_hg_commit_message') - def test_try_options(self, mock_get_hg_commit_message, mock_get_hg_revision_branch): + def test_try_options(self, mock_get_hg_commit_message, _): mock_get_hg_commit_message.return_value = 'try: -b do -t all' - mock_get_hg_revision_branch.return_value = 'default' self.options['project'] = 'try' with MockedOpen({self.ttc_file: None}): params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) @@ -102,9 +100,8 @@ class TestGetDecisionParameters(unittest.TestCase): @patch('taskgraph.decision.get_hg_revision_branch') @patch('taskgraph.decision.get_hg_commit_message') - def test_try_task_config(self, mock_get_hg_commit_message, mock_get_hg_revision_branch): + def test_try_task_config(self, mock_get_hg_commit_message, _): mock_get_hg_commit_message.return_value = 'Fuzzy query=foo' - mock_get_hg_revision_branch.return_value = 'default' ttc = {'tasks': ['a', 'b'], 'templates': {}} self.options['project'] = 'try' with MockedOpen({self.ttc_file: json.dumps(ttc)}): diff --git a/taskcluster/taskgraph/test/test_parameters.py b/taskcluster/taskgraph/test/test_parameters.py index 9623cacb8b31..5d3ea3895793 100644 --- a/taskcluster/taskgraph/test/test_parameters.py +++ b/taskcluster/taskgraph/test/test_parameters.py @@ -8,53 +8,17 @@ import unittest from taskgraph.parameters import ( Parameters, + ParameterMismatch, load_parameters_file, + PARAMETERS, + COMM_PARAMETERS, ) from mozunit import main, MockedOpen class TestParameters(unittest.TestCase): - vals = { - 'app_version': 'app_version', - 'base_repository': 'base_repository', - 'build_date': 0, - 'build_number': 0, - 'do_not_optimize': [], - 'existing_tasks': {}, - 'filters': [], - 'head_ref': 'head_ref', - 'head_repository': 'head_repository', - 'head_rev': 'head_rev', - 'hg_branch': 'hg_branch', - 'level': 'level', - 'message': 'message', - 'moz_build_date': 'moz_build_date', - 'next_version': 'next_version', - 'optimize_target_tasks': False, - 'owner': 'owner', - 'phabricator_diff': 'phabricator_diff', - 'project': 'project', - 'pushdate': 0, - 'pushlog_id': 'pushlog_id', - 'release_enable_emefree': False, - 'release_enable_partners': False, - 'release_eta': None, - 'release_history': {}, - 'release_partners': [], - 'release_partner_config': None, - 'release_partner_build_number': 1, - 'release_type': 'release_type', - 'release_product': None, - 'required_signoffs': [], - 'signoff_urls': {}, - 'target_tasks_method': 'target_tasks_method', - 'tasks_for': 'tasks_for', - 'try_mode': 'try_mode', - 'try_options': None, - 'try_task_config': None, - 'version': 'version', - } + vals = {n: n for n in PARAMETERS.keys()} def test_Parameters_immutable(self): p = Parameters(**self.vals) @@ -69,8 +33,8 @@ class TestParameters(unittest.TestCase): def test_Parameters_invalid_KeyError(self): """even if the value is present, if it's not a valid property, raise KeyError""" - p = Parameters(xyz=10, strict=True, **self.vals) - self.assertRaises(Exception, lambda: p.check()) + p = Parameters(xyz=10, **self.vals) + self.assertRaises(KeyError, lambda: p['xyz']) def test_Parameters_get(self): p = Parameters(head_ref=10, level=20) @@ -82,14 +46,14 @@ class TestParameters(unittest.TestCase): def test_Parameters_check_missing(self): p = Parameters() - self.assertRaises(Exception, lambda: p.check()) + self.assertRaises(ParameterMismatch, lambda: p.check()) p = Parameters(strict=False) p.check() # should not raise def test_Parameters_check_extra(self): p = Parameters(xyz=10, **self.vals) - self.assertRaises(Exception, lambda: p.check()) + self.assertRaises(ParameterMismatch, lambda: p.check()) p = Parameters(strict=False, xyz=10, **self.vals) p.check() # should not raise @@ -127,12 +91,7 @@ class TestParameters(unittest.TestCase): class TestCommParameters(unittest.TestCase): - vals = dict({ - 'comm_base_repository': 'comm_base_repository', - 'comm_head_ref': 'comm_head_ref', - 'comm_head_repository': 'comm_head_repository', - 'comm_head_rev': 'comm_head_rev', - }.items() + TestParameters.vals.items()) + vals = {n: n for n in PARAMETERS.keys() + COMM_PARAMETERS.keys()} def test_Parameters_check(self): """ @@ -146,7 +105,7 @@ class TestCommParameters(unittest.TestCase): If any of the comm parameters are specified, all of them must be specified. """ vals = self.vals.copy() - del vals['comm_base_repository'] + del vals[next(iter(COMM_PARAMETERS.keys()))] p = Parameters(**vals) self.assertRaises(Exception, p.check)