mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge mozilla-central to autoland. CLOSED TREE

This commit is contained in:
Mihai Alexandru Michis 2019-05-28 13:18:44 +03:00
Родитель 5432231319 1265267ba8
Коммит db3a14598c
3 изменённых файлов: 101 добавлений и 181 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

206
taskcluster/taskgraph/parameters.py
Просмотреть файл

@ -8,19 +8,11 @@ from __future__ import absolute_import, print_function, unicode_literals
import os.path
import json
import time
from datetime import datetime
from mozbuild.util import ReadOnlyDict, memoize
from mozversioncontrol import get_repository_object
from taskgraph.util.schema import validate_schema
from voluptuous import (
ALLOW_EXTRA,
Any,
Inclusive,
PREVENT_EXTRA,
Required,
Schema,
)
from . import GECKO
from .util.attributes import release_level
@ -53,58 +45,55 @@ def get_app_version(product_dir='browser'):
return get_contents(app_version_path)
base_schema = {
Required('app_version'): basestring,
Required('base_repository'): basestring,
Required('build_date'): int,
Required('build_number'): int,
Inclusive('comm_base_repository', 'comm'): basestring,
Inclusive('comm_head_ref', 'comm'): basestring,
Inclusive('comm_head_repository', 'comm'): basestring,
Inclusive('comm_head_rev', 'comm'): basestring,
Required('do_not_optimize'): [basestring],
Required('existing_tasks'): {basestring: basestring},
Required('filters'): [basestring],
Required('head_ref'): basestring,
Required('head_repository'): basestring,
Required('head_rev'): basestring,
Required('hg_branch'): basestring,
Required('level'): basestring,
Required('message'): basestring,
Required('moz_build_date'): basestring,
Required('next_version'): Any(None, basestring),
Required('optimize_target_tasks'): bool,
Required('owner'): basestring,
Required('phabricator_diff'): Any(None, basestring),
Required('project'): basestring,
Required('pushdate'): int,
Required('pushlog_id'): basestring,
Required('release_enable_emefree'): bool,
Required('release_enable_partners'): bool,
Required('release_eta'): Any(None, basestring),
Required('release_history'): {basestring: [dict]},
Required('release_partners'): [basestring],
Required('release_partner_config'): Any(None, dict),
Required('release_partner_build_number'): int,
Required('release_type'): basestring,
Required('release_product'): Any(None, basestring),
Required('required_signoffs'): [basestring],
Required('signoff_urls'): dict,
Required('target_tasks_method'): Any(None, basestring),
Required('tasks_for'): Any(None, basestring),
Required('try_mode'): Any(None, basestring),
Required('try_options'): Any(None, dict),
Required('try_task_config'): Any(None, dict),
Required('version'): basestring,
# Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
# Parameters are of the form: {name: default} or {name: lambda: default}
PARAMETERS = {
'app_version': get_app_version(),
'base_repository': 'https://hg.mozilla.org/mozilla-unified',
'build_date': lambda: int(time.time()),
'build_number': 1,
'do_not_optimize': [],
'existing_tasks': {},
'filters': ['target_tasks_method'],
'head_ref': get_head_ref,
'head_repository': 'https://hg.mozilla.org/mozilla-central',
'head_rev': get_head_ref,
'hg_branch': 'default',
'level': '3',
'message': '',
'moz_build_date': lambda: datetime.now().strftime("%Y%m%d%H%M%S"),
'next_version': None,
'optimize_target_tasks': True,
'owner': 'nobody@mozilla.com',
'project': 'mozilla-central',
'pushdate': lambda: int(time.time()),
'pushlog_id': '0',
'phabricator_diff': None,
'release_enable_emefree': False,
'release_enable_partners': False,
'release_eta': '',
'release_history': {},
'release_partners': None,
'release_partner_config': None,
'release_partner_build_number': 1,
'release_type': 'nightly',
'release_product': None,
'required_signoffs': [],
'signoff_urls': {},
'target_tasks_method': 'default',
'tasks_for': 'hg-push',
'try_mode': None,
'try_options': None,
'try_task_config': None,
'version': get_version(),
}
COMM_PARAMETERS = [
'comm_base_repository',
'comm_head_ref',
'comm_head_repository',
'comm_head_rev',
]
COMM_PARAMETERS = {
'comm_base_repository': 'https://hg.mozilla.org/comm-central',
'comm_head_ref': None,
'comm_head_repository': 'https://hg.mozilla.org/comm-central',
'comm_head_rev': None,
}
class Parameters(ReadOnlyDict):
@ -115,74 +104,49 @@ class Parameters(ReadOnlyDict):
if not self.strict:
# apply defaults to missing parameters
kwargs = Parameters._fill_defaults(**kwargs)
for name, default in PARAMETERS.items():
if name not in kwargs:
if callable(default):
default = default()
kwargs[name] = default
if set(kwargs) & set(COMM_PARAMETERS.keys()):
for name, default in COMM_PARAMETERS.items():
if name not in kwargs:
if callable(default):
default = default()
kwargs[name] = default
ReadOnlyDict.__init__(self, **kwargs)
@staticmethod
def _fill_defaults(**kwargs):
now = datetime.utcnow()
epoch = datetime.utcfromtimestamp(0)
seconds_from_epoch = int((now - epoch).total_seconds())
defaults = {
'app_version': get_app_version(),
'base_repository': 'https://hg.mozilla.org/mozilla-unified',
'build_date': seconds_from_epoch,
'build_number': 1,
'do_not_optimize': [],
'existing_tasks': {},
'filters': ['target_tasks_method'],
'head_ref': get_head_ref(),
'head_repository': 'https://hg.mozilla.org/mozilla-central',
'head_rev': get_head_ref(),
'hg_branch': 'default',
'level': '3',
'message': '',
'moz_build_date': now.strftime("%Y%m%d%H%M%S"),
'next_version': None,
'optimize_target_tasks': True,
'owner': 'nobody@mozilla.com',
'phabricator_diff': None,
'project': 'mozilla-central',
'pushdate': seconds_from_epoch,
'pushlog_id': '0',
'release_enable_emefree': False,
'release_enable_partners': False,
'release_eta': '',
'release_history': {},
'release_partners': [],
'release_partner_config': None,
'release_partner_build_number': 1,
'release_product': None,
'release_type': 'nightly',
'required_signoffs': [],
'signoff_urls': {},
'target_tasks_method': 'default',
'tasks_for': 'hg-push',
'try_mode': None,
'try_options': None,
'try_task_config': None,
'version': get_version(),
}
if set(COMM_PARAMETERS) & set(kwargs):
defaults.update({
'comm_base_repository': 'https://hg.mozilla.org/comm-central',
'comm_head_repository': 'https://hg.mozilla.org/comm-central',
})
for name, default in defaults.items():
if name not in kwargs:
kwargs[name] = default
return kwargs
def check(self):
schema = Schema(base_schema, extra=PREVENT_EXTRA if self.strict else ALLOW_EXTRA)
validate_schema(schema, self.copy(), 'Invalid parameters:')
names = set(self)
valid = set(PARAMETERS.keys())
valid_comm = set(COMM_PARAMETERS.keys())
msg = []
missing = valid - names
if missing:
msg.append("missing parameters: " + ", ".join(missing))
extra = names - valid
if extra & set(valid_comm):
# If any comm_* parameters are specified, ensure all of them are specified.
missing = valid_comm - extra
if missing:
msg.append("missing parameters: " + ", ".join(missing))
extra = extra - valid_comm
if extra and self.strict:
msg.append("extra parameters: " + ", ".join(extra))
if msg:
raise ParameterMismatch("; ".join(msg))
def __getitem__(self, k):
if not (k in PARAMETERS.keys() or k in COMM_PARAMETERS.keys()):
raise KeyError("no such parameter {!r}".format(k))
try:
return super(Parameters, self).__getitem__(k)
except KeyError:

15
taskcluster/taskgraph/test/test_decision.py
Просмотреть файл

@ -59,11 +59,11 @@ class TestGetDecisionParameters(unittest.TestCase):
'head_ref': 'ef01',
'message': '',
'project': 'mozilla-central',
'pushlog_id': '143',
'pushlog_id': 143,
'pushdate': 1503691511,
'owner': 'nobody@mozilla.com',
'tasks_for': 'hg-push',
'level': '3',
'level': 3,
}
@patch('taskgraph.decision.get_hg_revision_branch')
@ -71,7 +71,7 @@ class TestGetDecisionParameters(unittest.TestCase):
mock_get_hg_revision_branch.return_value = 'default'
with MockedOpen({self.ttc_file: None}):
params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
self.assertEqual(params['pushlog_id'], '143')
self.assertEqual(params['pushlog_id'], 143)
self.assertEqual(params['build_date'], 1503691511)
self.assertEqual(params['hg_branch'], 'default')
self.assertEqual(params['moz_build_date'], '20170825200511')
@ -80,8 +80,7 @@ class TestGetDecisionParameters(unittest.TestCase):
self.assertEqual(params['try_task_config'], None)
@patch('taskgraph.decision.get_hg_revision_branch')
def test_no_email_owner(self, mock_get_hg_revision_branch):
mock_get_hg_revision_branch.return_value = 'default'
def test_no_email_owner(self, _):
self.options['owner'] = 'ffxbld'
with MockedOpen({self.ttc_file: None}):
params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
@ -89,9 +88,8 @@ class TestGetDecisionParameters(unittest.TestCase):
@patch('taskgraph.decision.get_hg_revision_branch')
@patch('taskgraph.decision.get_hg_commit_message')
def test_try_options(self, mock_get_hg_commit_message, mock_get_hg_revision_branch):
def test_try_options(self, mock_get_hg_commit_message, _):
mock_get_hg_commit_message.return_value = 'try: -b do -t all'
mock_get_hg_revision_branch.return_value = 'default'
self.options['project'] = 'try'
with MockedOpen({self.ttc_file: None}):
params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
@ -102,9 +100,8 @@ class TestGetDecisionParameters(unittest.TestCase):
@patch('taskgraph.decision.get_hg_revision_branch')
@patch('taskgraph.decision.get_hg_commit_message')
def test_try_task_config(self, mock_get_hg_commit_message, mock_get_hg_revision_branch):
def test_try_task_config(self, mock_get_hg_commit_message, _):
mock_get_hg_commit_message.return_value = 'Fuzzy query=foo'
mock_get_hg_revision_branch.return_value = 'default'
ttc = {'tasks': ['a', 'b'], 'templates': {}}
self.options['project'] = 'try'
with MockedOpen({self.ttc_file: json.dumps(ttc)}):

61
taskcluster/taskgraph/test/test_parameters.py
Просмотреть файл

@ -8,53 +8,17 @@ import unittest
from taskgraph.parameters import (
Parameters,
ParameterMismatch,
load_parameters_file,
PARAMETERS,
COMM_PARAMETERS,
)
from mozunit import main, MockedOpen
class TestParameters(unittest.TestCase):
vals = {
'app_version': 'app_version',
'base_repository': 'base_repository',
'build_date': 0,
'build_number': 0,
'do_not_optimize': [],
'existing_tasks': {},
'filters': [],
'head_ref': 'head_ref',
'head_repository': 'head_repository',
'head_rev': 'head_rev',
'hg_branch': 'hg_branch',
'level': 'level',
'message': 'message',
'moz_build_date': 'moz_build_date',
'next_version': 'next_version',
'optimize_target_tasks': False,
'owner': 'owner',
'phabricator_diff': 'phabricator_diff',
'project': 'project',
'pushdate': 0,
'pushlog_id': 'pushlog_id',
'release_enable_emefree': False,
'release_enable_partners': False,
'release_eta': None,
'release_history': {},
'release_partners': [],
'release_partner_config': None,
'release_partner_build_number': 1,
'release_type': 'release_type',
'release_product': None,
'required_signoffs': [],
'signoff_urls': {},
'target_tasks_method': 'target_tasks_method',
'tasks_for': 'tasks_for',
'try_mode': 'try_mode',
'try_options': None,
'try_task_config': None,
'version': 'version',
}
vals = {n: n for n in PARAMETERS.keys()}
def test_Parameters_immutable(self):
p = Parameters(**self.vals)
@ -69,8 +33,8 @@ class TestParameters(unittest.TestCase):
def test_Parameters_invalid_KeyError(self):
"""even if the value is present, if it's not a valid property, raise KeyError"""
p = Parameters(xyz=10, strict=True, **self.vals)
self.assertRaises(Exception, lambda: p.check())
p = Parameters(xyz=10, **self.vals)
self.assertRaises(KeyError, lambda: p['xyz'])
def test_Parameters_get(self):
p = Parameters(head_ref=10, level=20)
@ -82,14 +46,14 @@ class TestParameters(unittest.TestCase):
def test_Parameters_check_missing(self):
p = Parameters()
self.assertRaises(Exception, lambda: p.check())
self.assertRaises(ParameterMismatch, lambda: p.check())
p = Parameters(strict=False)
p.check() # should not raise
def test_Parameters_check_extra(self):
p = Parameters(xyz=10, **self.vals)
self.assertRaises(Exception, lambda: p.check())
self.assertRaises(ParameterMismatch, lambda: p.check())
p = Parameters(strict=False, xyz=10, **self.vals)
p.check() # should not raise
@ -127,12 +91,7 @@ class TestParameters(unittest.TestCase):
class TestCommParameters(unittest.TestCase):
vals = dict({
'comm_base_repository': 'comm_base_repository',
'comm_head_ref': 'comm_head_ref',
'comm_head_repository': 'comm_head_repository',
'comm_head_rev': 'comm_head_rev',
}.items() + TestParameters.vals.items())
vals = {n: n for n in PARAMETERS.keys() + COMM_PARAMETERS.keys()}
def test_Parameters_check(self):
"""
@ -146,7 +105,7 @@ class TestCommParameters(unittest.TestCase):
If any of the comm parameters are specified, all of them must be specified.
"""
vals = self.vals.copy()
del vals['comm_base_repository']
del vals[next(iter(COMM_PARAMETERS.keys()))]
p = Parameters(**vals)
self.assertRaises(Exception, p.check)