зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1275409: remove mach_util; r=wcosta
MozReview-Commit-ID: 7FaRjPFBFkn --HG-- extra : rebase_source : 4912f4615ee5dc4f41f38e95abe5404d13c21d66
This commit is contained in:
Dustin J. Mitchell
Родитель
271fc2bc85
Коммит
b1c92a8289
|
|
@ -4,27 +4,20 @@
|
|||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import time
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import copy
|
||||
import re
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict, namedtuple
|
||||
|
||||
from . import base
|
||||
from ..types import Task
|
||||
from functools import partial
|
||||
from mozpack.path import match as mozpackmatch
|
||||
from slugid import nice as slugid
|
||||
from taskcluster_graph.mach_util import (
|
||||
merge_dicts,
|
||||
gaia_info,
|
||||
configure_dependent_task,
|
||||
set_interactive_task,
|
||||
remove_caches_from_task,
|
||||
query_vcs_info
|
||||
)
|
||||
from taskcluster_graph.commit_parser import parse_commit
|
||||
from taskgraph.util.time import (
|
||||
json_time_from_now,
|
||||
|
|
@ -34,6 +27,9 @@ from taskgraph.util.templates import Templates
|
|||
import taskcluster_graph.build_task
|
||||
from taskgraph.util.docker import docker_image
|
||||
|
||||
|
||||
ROOT = os.path.dirname(os.path.realpath(__file__))
|
||||
GECKO = os.path.realpath(os.path.join(ROOT, '..', '..', '..'))
|
||||
# TASKID_PLACEHOLDER is the "internal" form of a taskid; it is substituted with
|
||||
# actual taskIds at the very last minute, in get_task_definition
|
||||
TASKID_PLACEHOLDER = 'TaskLabel=={}'
|
||||
|
|
@ -58,6 +54,146 @@ logger = logging.getLogger(__name__)
|
|||
def mklabel():
|
||||
return TASKID_PLACEHOLDER.format(slugid())
|
||||
|
||||
def merge_dicts(*dicts):
|
||||
merged_dict = {}
|
||||
for dictionary in dicts:
|
||||
merged_dict.update(dictionary)
|
||||
return merged_dict
|
||||
|
||||
def gaia_info():
|
||||
'''Fetch details from in tree gaia.json (which links this version of
|
||||
gecko->gaia) and construct the usual base/head/ref/rev pairing...'''
|
||||
gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
|
||||
|
||||
if gaia['git'] is None or \
|
||||
gaia['git']['remote'] == '' or \
|
||||
gaia['git']['git_revision'] == '' or \
|
||||
gaia['git']['branch'] == '':
|
||||
|
||||
# Just use the hg params...
|
||||
return {
|
||||
'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
||||
'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
||||
'gaia_ref': gaia['revision'],
|
||||
'gaia_rev': gaia['revision']
|
||||
}
|
||||
|
||||
else:
|
||||
# Use git
|
||||
return {
|
||||
'gaia_base_repository': gaia['git']['remote'],
|
||||
'gaia_head_repository': gaia['git']['remote'],
|
||||
'gaia_rev': gaia['git']['git_revision'],
|
||||
'gaia_ref': gaia['git']['branch'],
|
||||
}
|
||||
|
||||
def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
|
||||
"""Configure a build dependent task. This is shared between post-build and test tasks.
|
||||
|
||||
:param task_path: location to the task yaml
|
||||
:param parameters: parameters to load the template
|
||||
:param taskid: taskid of the dependent task
|
||||
:param templates: reference to the template builder
|
||||
:param build_treeherder_config: parent treeherder config
|
||||
:return: the configured task
|
||||
"""
|
||||
task = templates.load(task_path, parameters)
|
||||
task['taskId'] = taskid
|
||||
|
||||
if 'requires' not in task:
|
||||
task['requires'] = []
|
||||
|
||||
task['requires'].append(parameters['build_slugid'])
|
||||
|
||||
if 'treeherder' not in task['task']['extra']:
|
||||
task['task']['extra']['treeherder'] = {}
|
||||
|
||||
# Copy over any treeherder configuration from the build so
|
||||
# tests show up under the same platform...
|
||||
treeherder_config = task['task']['extra']['treeherder']
|
||||
|
||||
treeherder_config['collection'] = \
|
||||
build_treeherder_config.get('collection', {})
|
||||
|
||||
treeherder_config['build'] = \
|
||||
build_treeherder_config.get('build', {})
|
||||
|
||||
treeherder_config['machine'] = \
|
||||
build_treeherder_config.get('machine', {})
|
||||
|
||||
if 'routes' not in task['task']:
|
||||
task['task']['routes'] = []
|
||||
|
||||
if 'scopes' not in task['task']:
|
||||
task['task']['scopes'] = []
|
||||
|
||||
return task
|
||||
|
||||
def set_interactive_task(task, interactive):
|
||||
r"""Make the task interactive.
|
||||
|
||||
:param task: task definition.
|
||||
:param interactive: True if the task should be interactive.
|
||||
"""
|
||||
if not interactive:
|
||||
return
|
||||
|
||||
payload = task["task"]["payload"]
|
||||
if "features" not in payload:
|
||||
payload["features"] = {}
|
||||
payload["features"]["interactive"] = True
|
||||
|
||||
def remove_caches_from_task(task):
|
||||
r"""Remove all caches but tc-vcs from the task.
|
||||
|
||||
:param task: task definition.
|
||||
"""
|
||||
whitelist = [
|
||||
re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"),
|
||||
re.compile("^tooltool-cache$"),
|
||||
]
|
||||
try:
|
||||
caches = task["task"]["payload"]["cache"]
|
||||
for cache in caches.keys():
|
||||
if not any(pat.match(cache) for pat in whitelist):
|
||||
caches.pop(cache)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def query_vcs_info(repository, revision):
|
||||
"""Query the pushdate and pushid of a repository/revision.
|
||||
|
||||
This is intended to be used on hg.mozilla.org/mozilla-central and
|
||||
similar. It may or may not work for other hg repositories.
|
||||
"""
|
||||
if not repository or not revision:
|
||||
logger.warning('cannot query vcs info because vcs info not provided')
|
||||
return None
|
||||
|
||||
VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
|
||||
|
||||
try:
|
||||
import requests
|
||||
url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
|
||||
revision)
|
||||
logger.debug("Querying version control for metadata: %s", url)
|
||||
contents = requests.get(url).json()
|
||||
|
||||
changesets = []
|
||||
for c in contents['changesets']:
|
||||
changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
|
||||
|
||||
pushid = contents['changesets'][-1]['pushid']
|
||||
pushdate = contents['changesets'][-1]['pushdate'][0]
|
||||
|
||||
return VCSInfo(pushid, pushdate, changesets)
|
||||
|
||||
except Exception:
|
||||
logger.exception("Error querying VCS info for '%s' revision '%s'",
|
||||
repository, revision)
|
||||
return None
|
||||
|
||||
|
||||
def set_expiration(task, timestamp):
|
||||
task_def = task['task']
|
||||
task_def['expires'] = timestamp
|
||||
|
|
|
|||
|
|
@ -1,165 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
from __future__ import absolute_import
|
||||
|
||||
from collections import defaultdict
|
||||
import os
|
||||
import json
|
||||
import copy
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
from collections import namedtuple
|
||||
|
||||
ROOT = os.path.dirname(os.path.realpath(__file__))
|
||||
GECKO = os.path.realpath(os.path.join(ROOT, '..', '..', '..'))
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def merge_dicts(*dicts):
|
||||
merged_dict = {}
|
||||
for dictionary in dicts:
|
||||
merged_dict.update(dictionary)
|
||||
return merged_dict
|
||||
|
||||
def gaia_info():
|
||||
'''
|
||||
Fetch details from in tree gaia.json (which links this version of
|
||||
gecko->gaia) and construct the usual base/head/ref/rev pairing...
|
||||
'''
|
||||
gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
|
||||
|
||||
if gaia['git'] is None or \
|
||||
gaia['git']['remote'] == '' or \
|
||||
gaia['git']['git_revision'] == '' or \
|
||||
gaia['git']['branch'] == '':
|
||||
|
||||
# Just use the hg params...
|
||||
return {
|
||||
'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
||||
'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
||||
'gaia_ref': gaia['revision'],
|
||||
'gaia_rev': gaia['revision']
|
||||
}
|
||||
|
||||
else:
|
||||
# Use git
|
||||
return {
|
||||
'gaia_base_repository': gaia['git']['remote'],
|
||||
'gaia_head_repository': gaia['git']['remote'],
|
||||
'gaia_rev': gaia['git']['git_revision'],
|
||||
'gaia_ref': gaia['git']['branch'],
|
||||
}
|
||||
|
||||
def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
|
||||
"""
|
||||
Configure a build dependent task. This is shared between post-build and test tasks.
|
||||
|
||||
:param task_path: location to the task yaml
|
||||
:param parameters: parameters to load the template
|
||||
:param taskid: taskid of the dependent task
|
||||
:param templates: reference to the template builder
|
||||
:param build_treeherder_config: parent treeherder config
|
||||
:return: the configured task
|
||||
"""
|
||||
task = templates.load(task_path, parameters)
|
||||
task['taskId'] = taskid
|
||||
|
||||
if 'requires' not in task:
|
||||
task['requires'] = []
|
||||
|
||||
task['requires'].append(parameters['build_slugid'])
|
||||
|
||||
if 'treeherder' not in task['task']['extra']:
|
||||
task['task']['extra']['treeherder'] = {}
|
||||
|
||||
# Copy over any treeherder configuration from the build so
|
||||
# tests show up under the same platform...
|
||||
treeherder_config = task['task']['extra']['treeherder']
|
||||
|
||||
treeherder_config['collection'] = \
|
||||
build_treeherder_config.get('collection', {})
|
||||
|
||||
treeherder_config['build'] = \
|
||||
build_treeherder_config.get('build', {})
|
||||
|
||||
treeherder_config['machine'] = \
|
||||
build_treeherder_config.get('machine', {})
|
||||
|
||||
if 'routes' not in task['task']:
|
||||
task['task']['routes'] = []
|
||||
|
||||
if 'scopes' not in task['task']:
|
||||
task['task']['scopes'] = []
|
||||
|
||||
return task
|
||||
|
||||
def set_interactive_task(task, interactive):
|
||||
r"""Make the task interactive.
|
||||
|
||||
:param task: task definition.
|
||||
:param interactive: True if the task should be interactive.
|
||||
"""
|
||||
if not interactive:
|
||||
return
|
||||
|
||||
payload = task["task"]["payload"]
|
||||
if "features" not in payload:
|
||||
payload["features"] = {}
|
||||
payload["features"]["interactive"] = True
|
||||
|
||||
def remove_caches_from_task(task):
|
||||
r"""Remove all caches but tc-vcs from the task.
|
||||
|
||||
:param task: task definition.
|
||||
"""
|
||||
whitelist = [
|
||||
re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"),
|
||||
re.compile("^tooltool-cache$"),
|
||||
]
|
||||
try:
|
||||
caches = task["task"]["payload"]["cache"]
|
||||
for cache in caches.keys():
|
||||
if not any(pat.match(cache) for pat in whitelist):
|
||||
caches.pop(cache)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def query_vcs_info(repository, revision):
|
||||
"""Query the pushdate and pushid of a repository/revision.
|
||||
This is intended to be used on hg.mozilla.org/mozilla-central and
|
||||
similar. It may or may not work for other hg repositories.
|
||||
"""
|
||||
if not repository or not revision:
|
||||
logger.warning('cannot query vcs info because vcs info not provided')
|
||||
return None
|
||||
|
||||
VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
|
||||
|
||||
try:
|
||||
import requests
|
||||
url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
|
||||
revision)
|
||||
logger.debug("Querying version control for metadata: %s" % url)
|
||||
contents = requests.get(url).json()
|
||||
|
||||
changesets = []
|
||||
for c in contents['changesets']:
|
||||
changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
|
||||
|
||||
pushid = contents['changesets'][-1]['pushid']
|
||||
pushdate = contents['changesets'][-1]['pushdate'][0]
|
||||
|
||||
return VCSInfo(pushid, pushdate, changesets)
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Error querying VCS info for '%s' revision '%s'" % (
|
||||
repository, revision,
|
||||
)
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
Загрузка…
Ссылка в новой задаче