Bug 1275409: remove mach_util; r=wcosta

MozReview-Commit-ID: 7FaRjPFBFkn

--HG--
extra : rebase_source : 4912f4615ee5dc4f41f38e95abe5404d13c21d66
This commit is contained in:
Dustin J. Mitchell 2016-06-04 20:59:19 +00:00
Родитель 271fc2bc85
Коммит b1c92a8289
2 изменённых файлов: 149 добавлений и 178 удалений

Просмотреть файл

@ -4,27 +4,20 @@
from __future__ import absolute_import, print_function, unicode_literals
import time
import os
import sys
import json
import copy
import re
import json
import logging
import os
import re
import sys
import time
from collections import defaultdict, namedtuple
from . import base
from ..types import Task
from functools import partial
from mozpack.path import match as mozpackmatch
from slugid import nice as slugid
from taskcluster_graph.mach_util import (
merge_dicts,
gaia_info,
configure_dependent_task,
set_interactive_task,
remove_caches_from_task,
query_vcs_info
)
from taskcluster_graph.commit_parser import parse_commit
from taskgraph.util.time import (
json_time_from_now,
@ -34,6 +27,9 @@ from taskgraph.util.templates import Templates
import taskcluster_graph.build_task
from taskgraph.util.docker import docker_image
ROOT = os.path.dirname(os.path.realpath(__file__))
GECKO = os.path.realpath(os.path.join(ROOT, '..', '..', '..'))
# TASKID_PLACEHOLDER is the "internal" form of a taskid; it is substituted with
# actual taskIds at the very last minute, in get_task_definition
TASKID_PLACEHOLDER = 'TaskLabel=={}'
@ -58,6 +54,146 @@ logger = logging.getLogger(__name__)
def mklabel():
return TASKID_PLACEHOLDER.format(slugid())
def merge_dicts(*dicts):
merged_dict = {}
for dictionary in dicts:
merged_dict.update(dictionary)
return merged_dict
def gaia_info():
'''Fetch details from in tree gaia.json (which links this version of
gecko->gaia) and construct the usual base/head/ref/rev pairing...'''
gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
if gaia['git'] is None or \
gaia['git']['remote'] == '' or \
gaia['git']['git_revision'] == '' or \
gaia['git']['branch'] == '':
# Just use the hg params...
return {
'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
'gaia_ref': gaia['revision'],
'gaia_rev': gaia['revision']
}
else:
# Use git
return {
'gaia_base_repository': gaia['git']['remote'],
'gaia_head_repository': gaia['git']['remote'],
'gaia_rev': gaia['git']['git_revision'],
'gaia_ref': gaia['git']['branch'],
}
def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
"""Configure a build dependent task. This is shared between post-build and test tasks.
:param task_path: location to the task yaml
:param parameters: parameters to load the template
:param taskid: taskid of the dependent task
:param templates: reference to the template builder
:param build_treeherder_config: parent treeherder config
:return: the configured task
"""
task = templates.load(task_path, parameters)
task['taskId'] = taskid
if 'requires' not in task:
task['requires'] = []
task['requires'].append(parameters['build_slugid'])
if 'treeherder' not in task['task']['extra']:
task['task']['extra']['treeherder'] = {}
# Copy over any treeherder configuration from the build so
# tests show up under the same platform...
treeherder_config = task['task']['extra']['treeherder']
treeherder_config['collection'] = \
build_treeherder_config.get('collection', {})
treeherder_config['build'] = \
build_treeherder_config.get('build', {})
treeherder_config['machine'] = \
build_treeherder_config.get('machine', {})
if 'routes' not in task['task']:
task['task']['routes'] = []
if 'scopes' not in task['task']:
task['task']['scopes'] = []
return task
def set_interactive_task(task, interactive):
r"""Make the task interactive.
:param task: task definition.
:param interactive: True if the task should be interactive.
"""
if not interactive:
return
payload = task["task"]["payload"]
if "features" not in payload:
payload["features"] = {}
payload["features"]["interactive"] = True
def remove_caches_from_task(task):
r"""Remove all caches but tc-vcs from the task.
:param task: task definition.
"""
whitelist = [
re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"),
re.compile("^tooltool-cache$"),
]
try:
caches = task["task"]["payload"]["cache"]
for cache in caches.keys():
if not any(pat.match(cache) for pat in whitelist):
caches.pop(cache)
except KeyError:
pass
def query_vcs_info(repository, revision):
"""Query the pushdate and pushid of a repository/revision.
This is intended to be used on hg.mozilla.org/mozilla-central and
similar. It may or may not work for other hg repositories.
"""
if not repository or not revision:
logger.warning('cannot query vcs info because vcs info not provided')
return None
VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
try:
import requests
url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
revision)
logger.debug("Querying version control for metadata: %s", url)
contents = requests.get(url).json()
changesets = []
for c in contents['changesets']:
changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
pushid = contents['changesets'][-1]['pushid']
pushdate = contents['changesets'][-1]['pushdate'][0]
return VCSInfo(pushid, pushdate, changesets)
except Exception:
logger.exception("Error querying VCS info for '%s' revision '%s'",
repository, revision)
return None
def set_expiration(task, timestamp):
task_def = task['task']
task_def['expires'] = timestamp

Просмотреть файл

@ -1,165 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
from collections import defaultdict
import os
import json
import copy
import re
import sys
import time
import logging
from collections import namedtuple
ROOT = os.path.dirname(os.path.realpath(__file__))
GECKO = os.path.realpath(os.path.join(ROOT, '..', '..', '..'))
logger = logging.getLogger(__name__)
def merge_dicts(*dicts):
merged_dict = {}
for dictionary in dicts:
merged_dict.update(dictionary)
return merged_dict
def gaia_info():
'''
Fetch details from in tree gaia.json (which links this version of
gecko->gaia) and construct the usual base/head/ref/rev pairing...
'''
gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
if gaia['git'] is None or \
gaia['git']['remote'] == '' or \
gaia['git']['git_revision'] == '' or \
gaia['git']['branch'] == '':
# Just use the hg params...
return {
'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
'gaia_ref': gaia['revision'],
'gaia_rev': gaia['revision']
}
else:
# Use git
return {
'gaia_base_repository': gaia['git']['remote'],
'gaia_head_repository': gaia['git']['remote'],
'gaia_rev': gaia['git']['git_revision'],
'gaia_ref': gaia['git']['branch'],
}
def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
"""
Configure a build dependent task. This is shared between post-build and test tasks.
:param task_path: location to the task yaml
:param parameters: parameters to load the template
:param taskid: taskid of the dependent task
:param templates: reference to the template builder
:param build_treeherder_config: parent treeherder config
:return: the configured task
"""
task = templates.load(task_path, parameters)
task['taskId'] = taskid
if 'requires' not in task:
task['requires'] = []
task['requires'].append(parameters['build_slugid'])
if 'treeherder' not in task['task']['extra']:
task['task']['extra']['treeherder'] = {}
# Copy over any treeherder configuration from the build so
# tests show up under the same platform...
treeherder_config = task['task']['extra']['treeherder']
treeherder_config['collection'] = \
build_treeherder_config.get('collection', {})
treeherder_config['build'] = \
build_treeherder_config.get('build', {})
treeherder_config['machine'] = \
build_treeherder_config.get('machine', {})
if 'routes' not in task['task']:
task['task']['routes'] = []
if 'scopes' not in task['task']:
task['task']['scopes'] = []
return task
def set_interactive_task(task, interactive):
r"""Make the task interactive.
:param task: task definition.
:param interactive: True if the task should be interactive.
"""
if not interactive:
return
payload = task["task"]["payload"]
if "features" not in payload:
payload["features"] = {}
payload["features"]["interactive"] = True
def remove_caches_from_task(task):
r"""Remove all caches but tc-vcs from the task.
:param task: task definition.
"""
whitelist = [
re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"),
re.compile("^tooltool-cache$"),
]
try:
caches = task["task"]["payload"]["cache"]
for cache in caches.keys():
if not any(pat.match(cache) for pat in whitelist):
caches.pop(cache)
except KeyError:
pass
def query_vcs_info(repository, revision):
"""Query the pushdate and pushid of a repository/revision.
This is intended to be used on hg.mozilla.org/mozilla-central and
similar. It may or may not work for other hg repositories.
"""
if not repository or not revision:
logger.warning('cannot query vcs info because vcs info not provided')
return None
VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
try:
import requests
url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
revision)
logger.debug("Querying version control for metadata: %s" % url)
contents = requests.get(url).json()
changesets = []
for c in contents['changesets']:
changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
pushid = contents['changesets'][-1]['pushid']
pushdate = contents['changesets'][-1]['pushdate'][0]
return VCSInfo(pushid, pushdate, changesets)
except Exception:
logger.exception(
"Error querying VCS info for '%s' revision '%s'" % (
repository, revision,
)
)
return None