From b1c92a82891de4c8dff7b0470ee66923c67ce316 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 4 Jun 2016 20:59:19 +0000 Subject: [PATCH] Bug 1275409: remove mach_util; r=wcosta MozReview-Commit-ID: 7FaRjPFBFkn --HG-- extra : rebase_source : 4912f4615ee5dc4f41f38e95abe5404d13c21d66 --- taskcluster/taskgraph/kind/legacy.py | 162 +++++++++++++++-- .../taskcluster_graph/mach_util.py | 165 ------------------ 2 files changed, 149 insertions(+), 178 deletions(-) delete mode 100644 testing/taskcluster/taskcluster_graph/mach_util.py diff --git a/taskcluster/taskgraph/kind/legacy.py b/taskcluster/taskgraph/kind/legacy.py index b1bd91ffea47..0bd914408158 100644 --- a/taskcluster/taskgraph/kind/legacy.py +++ b/taskcluster/taskgraph/kind/legacy.py @@ -4,27 +4,20 @@ from __future__ import absolute_import, print_function, unicode_literals -import time -import os -import sys -import json import copy -import re +import json import logging +import os +import re +import sys +import time +from collections import defaultdict, namedtuple from . import base from ..types import Task from functools import partial from mozpack.path import match as mozpackmatch from slugid import nice as slugid -from taskcluster_graph.mach_util import ( - merge_dicts, - gaia_info, - configure_dependent_task, - set_interactive_task, - remove_caches_from_task, - query_vcs_info -) from taskcluster_graph.commit_parser import parse_commit from taskgraph.util.time import ( json_time_from_now, @@ -34,6 +27,9 @@ from taskgraph.util.templates import Templates import taskcluster_graph.build_task from taskgraph.util.docker import docker_image + +ROOT = os.path.dirname(os.path.realpath(__file__)) +GECKO = os.path.realpath(os.path.join(ROOT, '..', '..', '..')) # TASKID_PLACEHOLDER is the "internal" form of a taskid; it is substituted with # actual taskIds at the very last minute, in get_task_definition TASKID_PLACEHOLDER = 'TaskLabel=={}' @@ -58,6 +54,146 @@ logger = logging.getLogger(__name__) def mklabel(): return TASKID_PLACEHOLDER.format(slugid()) +def merge_dicts(*dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + +def gaia_info(): + '''Fetch details from in tree gaia.json (which links this version of + gecko->gaia) and construct the usual base/head/ref/rev pairing...''' + gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json'))) + + if gaia['git'] is None or \ + gaia['git']['remote'] == '' or \ + gaia['git']['git_revision'] == '' or \ + gaia['git']['branch'] == '': + + # Just use the hg params... + return { + 'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']), + 'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']), + 'gaia_ref': gaia['revision'], + 'gaia_rev': gaia['revision'] + } + + else: + # Use git + return { + 'gaia_base_repository': gaia['git']['remote'], + 'gaia_head_repository': gaia['git']['remote'], + 'gaia_rev': gaia['git']['git_revision'], + 'gaia_ref': gaia['git']['branch'], + } + +def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config): + """Configure a build dependent task. This is shared between post-build and test tasks. + + :param task_path: location to the task yaml + :param parameters: parameters to load the template + :param taskid: taskid of the dependent task + :param templates: reference to the template builder + :param build_treeherder_config: parent treeherder config + :return: the configured task + """ + task = templates.load(task_path, parameters) + task['taskId'] = taskid + + if 'requires' not in task: + task['requires'] = [] + + task['requires'].append(parameters['build_slugid']) + + if 'treeherder' not in task['task']['extra']: + task['task']['extra']['treeherder'] = {} + + # Copy over any treeherder configuration from the build so + # tests show up under the same platform... + treeherder_config = task['task']['extra']['treeherder'] + + treeherder_config['collection'] = \ + build_treeherder_config.get('collection', {}) + + treeherder_config['build'] = \ + build_treeherder_config.get('build', {}) + + treeherder_config['machine'] = \ + build_treeherder_config.get('machine', {}) + + if 'routes' not in task['task']: + task['task']['routes'] = [] + + if 'scopes' not in task['task']: + task['task']['scopes'] = [] + + return task + +def set_interactive_task(task, interactive): + r"""Make the task interactive. + + :param task: task definition. + :param interactive: True if the task should be interactive. + """ + if not interactive: + return + + payload = task["task"]["payload"] + if "features" not in payload: + payload["features"] = {} + payload["features"]["interactive"] = True + +def remove_caches_from_task(task): + r"""Remove all caches but tc-vcs from the task. + + :param task: task definition. + """ + whitelist = [ + re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"), + re.compile("^tooltool-cache$"), + ] + try: + caches = task["task"]["payload"]["cache"] + for cache in caches.keys(): + if not any(pat.match(cache) for pat in whitelist): + caches.pop(cache) + except KeyError: + pass + +def query_vcs_info(repository, revision): + """Query the pushdate and pushid of a repository/revision. + + This is intended to be used on hg.mozilla.org/mozilla-central and + similar. It may or may not work for other hg repositories. + """ + if not repository or not revision: + logger.warning('cannot query vcs info because vcs info not provided') + return None + + VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets']) + + try: + import requests + url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), + revision) + logger.debug("Querying version control for metadata: %s", url) + contents = requests.get(url).json() + + changesets = [] + for c in contents['changesets']: + changesets.append({k: c[k] for k in ('desc', 'files', 'node')}) + + pushid = contents['changesets'][-1]['pushid'] + pushdate = contents['changesets'][-1]['pushdate'][0] + + return VCSInfo(pushid, pushdate, changesets) + + except Exception: + logger.exception("Error querying VCS info for '%s' revision '%s'", + repository, revision) + return None + + def set_expiration(task, timestamp): task_def = task['task'] task_def['expires'] = timestamp diff --git a/testing/taskcluster/taskcluster_graph/mach_util.py b/testing/taskcluster/taskcluster_graph/mach_util.py deleted file mode 100644 index b6436f1b5e28..000000000000 --- a/testing/taskcluster/taskcluster_graph/mach_util.py +++ /dev/null @@ -1,165 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import - -from collections import defaultdict -import os -import json -import copy -import re -import sys -import time -import logging -from collections import namedtuple - -ROOT = os.path.dirname(os.path.realpath(__file__)) -GECKO = os.path.realpath(os.path.join(ROOT, '..', '..', '..')) - -logger = logging.getLogger(__name__) - -def merge_dicts(*dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - -def gaia_info(): - ''' - Fetch details from in tree gaia.json (which links this version of - gecko->gaia) and construct the usual base/head/ref/rev pairing... - ''' - gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json'))) - - if gaia['git'] is None or \ - gaia['git']['remote'] == '' or \ - gaia['git']['git_revision'] == '' or \ - gaia['git']['branch'] == '': - - # Just use the hg params... - return { - 'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']), - 'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']), - 'gaia_ref': gaia['revision'], - 'gaia_rev': gaia['revision'] - } - - else: - # Use git - return { - 'gaia_base_repository': gaia['git']['remote'], - 'gaia_head_repository': gaia['git']['remote'], - 'gaia_rev': gaia['git']['git_revision'], - 'gaia_ref': gaia['git']['branch'], - } - -def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config): - """ - Configure a build dependent task. This is shared between post-build and test tasks. - - :param task_path: location to the task yaml - :param parameters: parameters to load the template - :param taskid: taskid of the dependent task - :param templates: reference to the template builder - :param build_treeherder_config: parent treeherder config - :return: the configured task - """ - task = templates.load(task_path, parameters) - task['taskId'] = taskid - - if 'requires' not in task: - task['requires'] = [] - - task['requires'].append(parameters['build_slugid']) - - if 'treeherder' not in task['task']['extra']: - task['task']['extra']['treeherder'] = {} - - # Copy over any treeherder configuration from the build so - # tests show up under the same platform... - treeherder_config = task['task']['extra']['treeherder'] - - treeherder_config['collection'] = \ - build_treeherder_config.get('collection', {}) - - treeherder_config['build'] = \ - build_treeherder_config.get('build', {}) - - treeherder_config['machine'] = \ - build_treeherder_config.get('machine', {}) - - if 'routes' not in task['task']: - task['task']['routes'] = [] - - if 'scopes' not in task['task']: - task['task']['scopes'] = [] - - return task - -def set_interactive_task(task, interactive): - r"""Make the task interactive. - - :param task: task definition. - :param interactive: True if the task should be interactive. - """ - if not interactive: - return - - payload = task["task"]["payload"] - if "features" not in payload: - payload["features"] = {} - payload["features"]["interactive"] = True - -def remove_caches_from_task(task): - r"""Remove all caches but tc-vcs from the task. - - :param task: task definition. - """ - whitelist = [ - re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"), - re.compile("^tooltool-cache$"), - ] - try: - caches = task["task"]["payload"]["cache"] - for cache in caches.keys(): - if not any(pat.match(cache) for pat in whitelist): - caches.pop(cache) - except KeyError: - pass - -def query_vcs_info(repository, revision): - """Query the pushdate and pushid of a repository/revision. - This is intended to be used on hg.mozilla.org/mozilla-central and - similar. It may or may not work for other hg repositories. - """ - if not repository or not revision: - logger.warning('cannot query vcs info because vcs info not provided') - return None - - VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets']) - - try: - import requests - url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), - revision) - logger.debug("Querying version control for metadata: %s" % url) - contents = requests.get(url).json() - - changesets = [] - for c in contents['changesets']: - changesets.append({k: c[k] for k in ('desc', 'files', 'node')}) - - pushid = contents['changesets'][-1]['pushid'] - pushdate = contents['changesets'][-1]['pushdate'][0] - - return VCSInfo(pushid, pushdate, changesets) - - except Exception: - logger.exception( - "Error querying VCS info for '%s' revision '%s'" % ( - repository, revision, - ) - ) - return None - -