зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1280956 - Use in-tree linter job to flake8 test taskcluster directory. r=dustin
MozReview-Commit-ID: FsWmAnnycZ2 --HG-- extra : rebase_source : 04a32cea2de133cb75472092cffb8a215f7dc603
This commit is contained in:
Justin Wood
Родитель
e07dcaf301
Коммит
15b23fced1
|
|
@ -545,6 +545,7 @@ tasks:
|
|||
- 'python/mozlint/**'
|
||||
- 'tools/lint/**'
|
||||
- 'testing/docker/lint/**'
|
||||
- 'taskcluster/**'
|
||||
taskgraph-tests:
|
||||
task: tasks/tests/taskgraph-tests.yml
|
||||
root: true
|
||||
|
|
|
|||
|
|
@ -78,66 +78,66 @@ class MachCommands(MachCommandBase):
|
|||
sys.exit(1)
|
||||
|
||||
@ShowTaskGraphSubCommand('taskgraph', 'tasks',
|
||||
description="Show all tasks in the taskgraph")
|
||||
description="Show all tasks in the taskgraph")
|
||||
def taskgraph_tasks(self, **options):
|
||||
return self.show_taskgraph('full_task_set', options)
|
||||
|
||||
@ShowTaskGraphSubCommand('taskgraph', 'full',
|
||||
description="Show the full taskgraph")
|
||||
description="Show the full taskgraph")
|
||||
def taskgraph_full(self, **options):
|
||||
return self.show_taskgraph('full_task_graph', options)
|
||||
|
||||
@ShowTaskGraphSubCommand('taskgraph', 'target',
|
||||
description="Show the target task set")
|
||||
description="Show the target task set")
|
||||
def taskgraph_target(self, **options):
|
||||
return self.show_taskgraph('target_task_set', options)
|
||||
|
||||
@ShowTaskGraphSubCommand('taskgraph', 'target-graph',
|
||||
description="Show the target taskgraph")
|
||||
description="Show the target taskgraph")
|
||||
def taskgraph_target_taskgraph(self, **options):
|
||||
return self.show_taskgraph('target_task_graph', options)
|
||||
|
||||
@ShowTaskGraphSubCommand('taskgraph', 'optimized',
|
||||
description="Show the optimized taskgraph")
|
||||
description="Show the optimized taskgraph")
|
||||
def taskgraph_optimized(self, **options):
|
||||
return self.show_taskgraph('optimized_task_graph', options)
|
||||
|
||||
@SubCommand('taskgraph', 'decision',
|
||||
description="Run the decision task")
|
||||
@CommandArgument('--root', '-r',
|
||||
default='taskcluster/ci',
|
||||
help="root of the taskgraph definition relative to topsrcdir")
|
||||
default='taskcluster/ci',
|
||||
help="root of the taskgraph definition relative to topsrcdir")
|
||||
@CommandArgument('--base-repository',
|
||||
required=True,
|
||||
help='URL for "base" repository to clone')
|
||||
required=True,
|
||||
help='URL for "base" repository to clone')
|
||||
@CommandArgument('--head-repository',
|
||||
required=True,
|
||||
help='URL for "head" repository to fetch revision from')
|
||||
required=True,
|
||||
help='URL for "head" repository to fetch revision from')
|
||||
@CommandArgument('--head-ref',
|
||||
required=True,
|
||||
help='Reference (this is same as rev usually for hg)')
|
||||
required=True,
|
||||
help='Reference (this is same as rev usually for hg)')
|
||||
@CommandArgument('--head-rev',
|
||||
required=True,
|
||||
help='Commit revision to use from head repository')
|
||||
required=True,
|
||||
help='Commit revision to use from head repository')
|
||||
@CommandArgument('--message',
|
||||
required=True,
|
||||
help='Commit message to be parsed. Example: "try: -b do -p all -u all"')
|
||||
required=True,
|
||||
help='Commit message to be parsed. Example: "try: -b do -p all -u all"')
|
||||
@CommandArgument('--revision-hash',
|
||||
required=True,
|
||||
help='Treeherder revision hash (long revision id) to attach results to')
|
||||
required=True,
|
||||
help='Treeherder revision hash (long revision id) to attach results to')
|
||||
@CommandArgument('--project',
|
||||
required=True,
|
||||
help='Project to use for creating task graph. Example: --project=try')
|
||||
required=True,
|
||||
help='Project to use for creating task graph. Example: --project=try')
|
||||
@CommandArgument('--pushlog-id',
|
||||
dest='pushlog_id',
|
||||
required=True,
|
||||
default=0)
|
||||
dest='pushlog_id',
|
||||
required=True,
|
||||
default=0)
|
||||
@CommandArgument('--owner',
|
||||
required=True,
|
||||
help='email address of who owns this graph')
|
||||
required=True,
|
||||
help='email address of who owns this graph')
|
||||
@CommandArgument('--level',
|
||||
required=True,
|
||||
help='SCM level of this repository')
|
||||
required=True,
|
||||
help='SCM level of this repository')
|
||||
def taskgraph_decision(self, **options):
|
||||
"""Run the decision task: generate a task graph and submit to
|
||||
TaskCluster. This is only meant to be called within decision tasks,
|
||||
|
|
@ -149,11 +149,10 @@ class MachCommands(MachCommandBase):
|
|||
try:
|
||||
self.setup_logging()
|
||||
return taskgraph.decision.taskgraph_decision(options)
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def setup_logging(self, quiet=False, verbose=True):
|
||||
"""
|
||||
Set up Python logging for all loggers, sending results to stderr (so
|
||||
|
|
@ -170,7 +169,6 @@ class MachCommands(MachCommandBase):
|
|||
|
||||
# all of the taskgraph logging is unstructured logging
|
||||
self.log_manager.enable_unstructured()
|
||||
|
||||
|
||||
def show_taskgraph(self, graph_attr, options):
|
||||
import taskgraph.parameters
|
||||
|
|
@ -192,7 +190,7 @@ class MachCommands(MachCommandBase):
|
|||
|
||||
show_method = getattr(self, 'show_taskgraph_' + (options['format'] or 'labels'))
|
||||
show_method(tg)
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
|
@ -210,13 +208,14 @@ class MachCommands(MachCommandBase):
|
|||
@CommandProvider
|
||||
class LoadImage(object):
|
||||
@Command('taskcluster-load-image', category="ci",
|
||||
description="Load a pre-built Docker image")
|
||||
description="Load a pre-built Docker image")
|
||||
@CommandArgument('--task-id',
|
||||
help="Load the image at public/image.tar in this task, rather than "
|
||||
"searching the index")
|
||||
help="Load the image at public/image.tar in this task,"
|
||||
"rather than searching the index")
|
||||
@CommandArgument('image_name', nargs='?',
|
||||
help="Load the image of this name based on the current contents of the tree "
|
||||
"(as built for mozilla-central or mozilla-inbound)")
|
||||
help="Load the image of this name based on the current"
|
||||
"contents of the tree (as built for mozilla-central"
|
||||
"or mozilla-inbound)")
|
||||
def load_image(self, image_name, task_id):
|
||||
from taskgraph.docker import load_image_by_name, load_image_by_task_id
|
||||
if not image_name and not task_id:
|
||||
|
|
@ -229,6 +228,6 @@ class LoadImage(object):
|
|||
ok = load_image_by_name(image_name)
|
||||
if not ok:
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ if not os.path.isfile(props_path):
|
|||
props = json.load(open(props_path))
|
||||
|
||||
if args.prop == 'revision':
|
||||
print(props['revision']);
|
||||
print(props['revision'])
|
||||
|
||||
if args.prop == 'repository':
|
||||
print(urlparse.urljoin('https://hg.mozilla.org', props['repo_path']))
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import concurrent.futures as futures
|
|||
import requests
|
||||
import requests.adapters
|
||||
import json
|
||||
import collections
|
||||
import os
|
||||
import logging
|
||||
|
||||
|
|
@ -16,6 +15,7 @@ from slugid import nice as slugid
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_tasks(taskgraph, label_to_taskid):
|
||||
# TODO: use the taskGroupId of the decision task
|
||||
task_group_id = slugid()
|
||||
|
|
@ -60,11 +60,13 @@ def create_tasks(taskgraph, label_to_taskid):
|
|||
for f in futures.as_completed(fs.values()):
|
||||
f.result()
|
||||
|
||||
|
||||
def _create_task(session, task_id, label, task_def):
|
||||
# create the task using 'http://taskcluster/queue', which is proxied to the queue service
|
||||
# with credentials appropriate to this job.
|
||||
logger.debug("Creating task with taskId {} for {}".format(task_id, label))
|
||||
res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id), data=json.dumps(task_def))
|
||||
res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id),
|
||||
data=json.dumps(task_def))
|
||||
if res.status_code != 200:
|
||||
try:
|
||||
logger.error(res.json()['message'])
|
||||
|
|
|
|||
|
|
@ -103,8 +103,8 @@ def get_decision_parameters(options):
|
|||
parameters.update(PER_PROJECT_PARAMETERS[project])
|
||||
except KeyError:
|
||||
logger.warning("using default project parameters; add {} to "
|
||||
"PER_PROJECT_PARAMETERS in {} to customize behavior "
|
||||
"for this project".format(project, __file__))
|
||||
"PER_PROJECT_PARAMETERS in {} to customize behavior "
|
||||
"for this project".format(project, __file__))
|
||||
parameters.update(PER_PROJECT_PARAMETERS['default'])
|
||||
|
||||
return Parameters(parameters)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@
|
|||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import yaml
|
||||
|
||||
from .graph import Graph
|
||||
|
|
@ -14,6 +13,7 @@ from .optimize import optimize_task_graph
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TaskGraphGenerator(object):
|
||||
"""
|
||||
The central controller for taskgraph. This handles all phases of graph
|
||||
|
|
@ -61,7 +61,6 @@ class TaskGraphGenerator(object):
|
|||
"""
|
||||
return self._run_until('full_task_set')
|
||||
|
||||
|
||||
@property
|
||||
def full_task_graph(self):
|
||||
"""
|
||||
|
|
@ -181,7 +180,8 @@ class TaskGraphGenerator(object):
|
|||
do_not_optimize = set()
|
||||
if not self.parameters.get('optimize_target_tasks', True):
|
||||
do_not_optimize = target_task_set.graph.nodes
|
||||
optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph, do_not_optimize)
|
||||
optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
|
||||
do_not_optimize)
|
||||
yield 'label_to_taskid', label_to_taskid
|
||||
yield 'optimized_task_graph', optimized_task_graph
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals
|
|||
|
||||
import collections
|
||||
|
||||
|
||||
class Graph(object):
|
||||
"""
|
||||
Generic representation of a directed acyclic graph with labeled edges
|
||||
|
|
@ -54,7 +55,9 @@ class Graph(object):
|
|||
nodes, edges = set(), set()
|
||||
while (new_nodes, new_edges) != (nodes, edges):
|
||||
nodes, edges = new_nodes, new_edges
|
||||
add_edges = set((left, right, name) for (left, right, name) in self.edges if left in nodes)
|
||||
add_edges = set((left, right, name)
|
||||
for (left, right, name) in self.edges
|
||||
if left in nodes)
|
||||
add_nodes = set(right for (_, right, _) in add_edges)
|
||||
new_nodes = nodes | add_nodes
|
||||
new_edges = edges | add_edges
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from __future__ import absolute_import, print_function, unicode_literals
|
|||
import os
|
||||
import abc
|
||||
|
||||
|
||||
class Kind(object):
|
||||
"""
|
||||
A kind represents a collection of tasks that share common characteristics.
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import logging
|
|||
import json
|
||||
import os
|
||||
import urllib2
|
||||
import hashlib
|
||||
import tarfile
|
||||
import time
|
||||
|
||||
|
|
@ -54,7 +53,7 @@ class DockerImageKind(base.Kind):
|
|||
'from_now': json_time_from_now,
|
||||
'now': current_json_time(),
|
||||
'source': '{repo}file/{rev}/testing/taskcluster/tasks/image.yml'
|
||||
.format(repo=params['head_repository'], rev=params['head_rev']),
|
||||
.format(repo=params['head_repository'], rev=params['head_rev']),
|
||||
}
|
||||
|
||||
tasks = []
|
||||
|
|
@ -69,12 +68,14 @@ class DockerImageKind(base.Kind):
|
|||
image_parameters['artifact_path'] = 'public/image.tar'
|
||||
image_parameters['image_name'] = image_name
|
||||
|
||||
image_artifact_path = "public/decision_task/image_contexts/{}/context.tar.gz".format(image_name)
|
||||
image_artifact_path = \
|
||||
"public/decision_task/image_contexts/{}/context.tar.gz".format(image_name)
|
||||
if os.environ.get('TASK_ID'):
|
||||
destination = os.path.join(
|
||||
os.environ['HOME'],
|
||||
"artifacts/decision_task/image_contexts/{}/context.tar.gz".format(image_name))
|
||||
image_parameters['context_url'] = ARTIFACT_URL.format(os.environ['TASK_ID'], image_artifact_path)
|
||||
image_parameters['context_url'] = ARTIFACT_URL.format(
|
||||
os.environ['TASK_ID'], image_artifact_path)
|
||||
self.create_context_tar(context_path, destination, image_name)
|
||||
else:
|
||||
# skip context generation since this isn't a decision task
|
||||
|
|
@ -94,7 +95,8 @@ class DockerImageKind(base.Kind):
|
|||
# up on mozilla-central at some point if most tasks use this as a common image
|
||||
# for a given context hash, a worker within Taskcluster does not need to contain
|
||||
# the same image per branch.
|
||||
index_paths = ['docker.images.v1.{}.{}.hash.{}'.format(project, image_name, context_hash)
|
||||
index_paths = ['docker.images.v1.{}.{}.hash.{}'.format(
|
||||
project, image_name, context_hash)
|
||||
for project in ['mozilla-central', params['project']]]
|
||||
|
||||
tasks.append(Task(self, 'build-docker-image-' + image_name,
|
||||
|
|
@ -116,7 +118,8 @@ class DockerImageKind(base.Kind):
|
|||
# continues trying other branches in case mozilla-central has an expired
|
||||
# artifact, but 'project' might not. Only return no task ID if all
|
||||
# branches have been tried
|
||||
request = urllib2.Request(ARTIFACT_URL.format(existing_task['taskId'], 'public/image.tar'))
|
||||
request = urllib2.Request(
|
||||
ARTIFACT_URL.format(existing_task['taskId'], 'public/image.tar'))
|
||||
request.get_method = lambda: 'HEAD'
|
||||
urllib2.urlopen(request)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,13 +9,11 @@ import json
|
|||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict, namedtuple
|
||||
from collections import namedtuple
|
||||
|
||||
from . import base
|
||||
from ..types import Task
|
||||
from functools import partial
|
||||
from mozpack.path import match as mozpackmatch
|
||||
from slugid import nice as slugid
|
||||
from taskgraph.util.legacy_commit_parser import parse_commit
|
||||
|
|
@ -49,15 +47,18 @@ TRY_EXPIRATION = "14 days"
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def mklabel():
|
||||
return TASKID_PLACEHOLDER.format(slugid())
|
||||
|
||||
|
||||
def merge_dicts(*dicts):
|
||||
merged_dict = {}
|
||||
for dictionary in dicts:
|
||||
merged_dict.update(dictionary)
|
||||
return merged_dict
|
||||
|
||||
|
||||
def gaia_info():
|
||||
'''Fetch details from in tree gaia.json (which links this version of
|
||||
gecko->gaia) and construct the usual base/head/ref/rev pairing...'''
|
||||
|
|
@ -68,13 +69,13 @@ def gaia_info():
|
|||
gaia['git']['git_revision'] == '' or \
|
||||
gaia['git']['branch'] == '':
|
||||
|
||||
# Just use the hg params...
|
||||
return {
|
||||
'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
||||
'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
||||
'gaia_ref': gaia['revision'],
|
||||
'gaia_rev': gaia['revision']
|
||||
}
|
||||
# Just use the hg params...
|
||||
return {
|
||||
'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
||||
'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
||||
'gaia_ref': gaia['revision'],
|
||||
'gaia_rev': gaia['revision']
|
||||
}
|
||||
|
||||
else:
|
||||
# Use git
|
||||
|
|
@ -85,6 +86,7 @@ def gaia_info():
|
|||
'gaia_ref': gaia['git']['branch'],
|
||||
}
|
||||
|
||||
|
||||
def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
|
||||
"""Configure a build dependent task. This is shared between post-build and test tasks.
|
||||
|
||||
|
|
@ -128,6 +130,7 @@ def configure_dependent_task(task_path, parameters, taskid, templates, build_tre
|
|||
|
||||
return task
|
||||
|
||||
|
||||
def set_interactive_task(task, interactive):
|
||||
r"""Make the task interactive.
|
||||
|
||||
|
|
@ -142,6 +145,7 @@ def set_interactive_task(task, interactive):
|
|||
payload["features"] = {}
|
||||
payload["features"]["interactive"] = True
|
||||
|
||||
|
||||
def remove_caches_from_task(task):
|
||||
r"""Remove all caches but tc-vcs from the task.
|
||||
|
||||
|
|
@ -159,6 +163,7 @@ def remove_caches_from_task(task):
|
|||
except KeyError:
|
||||
pass
|
||||
|
||||
|
||||
def query_vcs_info(repository, revision):
|
||||
"""Query the pushdate and pushid of a repository/revision.
|
||||
|
||||
|
|
@ -189,7 +194,7 @@ def query_vcs_info(repository, revision):
|
|||
|
||||
except Exception:
|
||||
logger.exception("Error querying VCS info for '%s' revision '%s'",
|
||||
repository, revision)
|
||||
repository, revision)
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -210,12 +215,14 @@ def set_expiration(task, timestamp):
|
|||
for artifact in artifacts.values() if hasattr(artifacts, "values") else artifacts:
|
||||
artifact['expires'] = timestamp
|
||||
|
||||
|
||||
def format_treeherder_route(destination, project, revision, pushlog_id):
|
||||
return "{}.v2.{}.{}.{}".format(destination,
|
||||
project,
|
||||
revision,
|
||||
pushlog_id)
|
||||
|
||||
|
||||
def decorate_task_treeherder_routes(task, project, revision, pushlog_id):
|
||||
"""Decorate the given task with treeherder routes.
|
||||
|
||||
|
|
@ -243,6 +250,7 @@ def decorate_task_treeherder_routes(task, project, revision, pushlog_id):
|
|||
pushlog_id)
|
||||
task['routes'].append(route)
|
||||
|
||||
|
||||
def decorate_task_json_routes(task, json_routes, parameters):
|
||||
"""Decorate the given task with routes.json routes.
|
||||
|
||||
|
|
@ -256,9 +264,11 @@ def decorate_task_json_routes(task, json_routes, parameters):
|
|||
|
||||
task['routes'] = routes
|
||||
|
||||
|
||||
class BuildTaskValidationException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def validate_build_task(task):
|
||||
'''The build tasks have some required fields in extra this function ensures
|
||||
they are there. '''
|
||||
|
|
@ -281,6 +291,7 @@ def validate_build_task(task):
|
|||
raise BuildTaskValidationException('task.extra.locations.tests or '
|
||||
'task.extra.locations.tests_packages missing')
|
||||
|
||||
|
||||
class LegacyKind(base.Kind):
|
||||
"""
|
||||
This kind generates a full task graph from the old YAML files in
|
||||
|
|
@ -318,7 +329,8 @@ class LegacyKind(base.Kind):
|
|||
if vcs_info:
|
||||
push_epoch = vcs_info.pushdate
|
||||
|
||||
logger.debug('{} commits influencing task scheduling:'.format(len(vcs_info.changesets)))
|
||||
logger.debug(
|
||||
'{} commits influencing task scheduling:'.format(len(vcs_info.changesets)))
|
||||
for c in vcs_info.changesets:
|
||||
logger.debug("{cset} {desc}".format(
|
||||
cset=c['node'][0:12],
|
||||
|
|
@ -370,7 +382,8 @@ class LegacyKind(base.Kind):
|
|||
graph['scopes'].add("queue:route:{}".format(route))
|
||||
|
||||
graph['metadata'] = {
|
||||
'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']),
|
||||
'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(
|
||||
repo=params['head_repository'], rev=params['head_rev']),
|
||||
'owner': params['owner'],
|
||||
# TODO: Add full mach commands to this example?
|
||||
'description': 'Task graph generated via ./mach taskcluster-graph',
|
||||
|
|
@ -400,7 +413,7 @@ class LegacyKind(base.Kind):
|
|||
task=task['task'],
|
||||
pattern=pattern,
|
||||
path=path,
|
||||
))
|
||||
))
|
||||
return True
|
||||
|
||||
# No file patterns matched. Discard task.
|
||||
|
|
@ -421,7 +434,8 @@ class LegacyKind(base.Kind):
|
|||
interactive = cmdline_interactive or build["interactive"]
|
||||
build_parameters = merge_dicts(parameters, build['additional-parameters'])
|
||||
build_parameters['build_slugid'] = mklabel()
|
||||
build_parameters['source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format(repo=params['head_repository'], rev=params['head_rev'], file=build['task'])
|
||||
build_parameters['source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format(
|
||||
repo=params['head_repository'], rev=params['head_rev'], file=build['task'])
|
||||
build_task = templates.load(build['task'], build_parameters)
|
||||
|
||||
# Copy build_* attributes to expose them to post-build tasks
|
||||
|
|
@ -448,7 +462,7 @@ class LegacyKind(base.Kind):
|
|||
|
||||
# Ensure each build graph is valid after construction.
|
||||
validate_build_task(build_task)
|
||||
attributes = build_task['attributes'] = {'kind':'legacy', 'legacy_kind': 'build'}
|
||||
attributes = build_task['attributes'] = {'kind': 'legacy', 'legacy_kind': 'build'}
|
||||
if 'build_name' in build:
|
||||
attributes['build_platform'] = build['build_name']
|
||||
if 'build_type' in task_extra:
|
||||
|
|
@ -460,7 +474,8 @@ class LegacyKind(base.Kind):
|
|||
graph['tasks'].append(build_task)
|
||||
|
||||
for location in build_task['task']['extra'].get('locations', {}):
|
||||
build_parameters['{}_location'.format(location)] = build_task['task']['extra']['locations'][location]
|
||||
build_parameters['{}_location'.format(location)] = \
|
||||
build_task['task']['extra']['locations'][location]
|
||||
|
||||
for url in build_task['task']['extra'].get('url', {}):
|
||||
build_parameters['{}_url'.format(url)] = \
|
||||
|
|
@ -470,16 +485,19 @@ class LegacyKind(base.Kind):
|
|||
|
||||
for route in build_task['task'].get('routes', []):
|
||||
if route.startswith('index.gecko.v2') and route in all_routes:
|
||||
raise Exception("Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % (
|
||||
route,
|
||||
build_task['task']['metadata']['name'],
|
||||
all_routes[route],
|
||||
))
|
||||
raise Exception(
|
||||
"Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % (
|
||||
route,
|
||||
build_task['task']['metadata']['name'],
|
||||
all_routes[route],
|
||||
))
|
||||
all_routes[route] = build_task['task']['metadata']['name']
|
||||
|
||||
graph['scopes'].add(define_task)
|
||||
graph['scopes'] |= set(build_task['task'].get('scopes', []))
|
||||
route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', []))
|
||||
route_scopes = map(
|
||||
lambda route: 'queue:route:' + route, build_task['task'].get('routes', [])
|
||||
)
|
||||
graph['scopes'] |= set(route_scopes)
|
||||
|
||||
# Treeherder symbol configuration for the graph required for each
|
||||
|
|
@ -620,4 +638,3 @@ class LegacyKind(base.Kind):
|
|||
def optimize_task(self, task, taskgraph):
|
||||
# no optimization for the moment
|
||||
return False, None
|
||||
|
||||
|
|
|
|||
|
|
@ -75,7 +75,9 @@ def annotate_task_graph(target_task_graph, do_not_optimize, named_links_dict, la
|
|||
dependencies = [target_task_graph.tasks[l] for l in named_task_dependencies.itervalues()]
|
||||
for t in dependencies:
|
||||
if t.optimized and not t.task_id:
|
||||
raise Exception("task {} was optimized away, but {} depends on it".format(t.label, label))
|
||||
raise Exception(
|
||||
"task {} was optimized away, but {} depends on it".format(
|
||||
t.label, label))
|
||||
|
||||
# if this task is blacklisted, don't even consider optimizing
|
||||
replacement_task_id = None
|
||||
|
|
@ -131,12 +133,16 @@ def get_subgraph(annotated_task_graph, named_links_dict, label_to_taskid):
|
|||
tasks_by_taskid[task.task_id] = task
|
||||
|
||||
# resolve edges to taskIds
|
||||
edges_by_taskid = ((label_to_taskid.get(left), label_to_taskid.get(right), name)
|
||||
for (left, right, name) in annotated_task_graph.graph.edges)
|
||||
edges_by_taskid = (
|
||||
(label_to_taskid.get(left), label_to_taskid.get(right), name)
|
||||
for (left, right, name) in annotated_task_graph.graph.edges
|
||||
)
|
||||
# ..and drop edges that are no longer in the task graph
|
||||
edges_by_taskid = set((left, right, name)
|
||||
edges_by_taskid = set(
|
||||
(left, right, name)
|
||||
for (left, right, name) in edges_by_taskid
|
||||
if left in tasks_by_taskid and right in tasks_by_taskid)
|
||||
if left in tasks_by_taskid and right in tasks_by_taskid
|
||||
)
|
||||
|
||||
return TaskGraph(
|
||||
tasks_by_taskid,
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@
|
|||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import json
|
||||
import sys
|
||||
import yaml
|
||||
from mozbuild.util import ReadOnlyDict
|
||||
|
||||
|
||||
class Parameters(ReadOnlyDict):
|
||||
"""An immutable dictionary with nicer KeyError messages on failure"""
|
||||
def __getitem__(self, k):
|
||||
|
|
|
|||
|
|
@ -8,16 +8,20 @@ from __future__ import absolute_import, print_function, unicode_literals
|
|||
from taskgraph import try_option_syntax
|
||||
|
||||
_target_task_methods = {}
|
||||
|
||||
|
||||
def _target_task(name):
|
||||
def wrap(func):
|
||||
_target_task_methods[name] = func
|
||||
return func
|
||||
return wrap
|
||||
|
||||
|
||||
def get_method(method):
|
||||
"""Get a target_task_method to pass to a TaskGraphGenerator."""
|
||||
return _target_task_methods[method]
|
||||
|
||||
|
||||
@_target_task('from_parameters')
|
||||
def target_tasks_from_parameters(full_task_graph, parameters):
|
||||
"""Get the target task set from parameters['target_tasks']. This is
|
||||
|
|
@ -25,6 +29,7 @@ def target_tasks_from_parameters(full_task_graph, parameters):
|
|||
earlier run, by copying `target_tasks.json` into `parameters.yml`."""
|
||||
return parameters['target_tasks']
|
||||
|
||||
|
||||
@_target_task('try_option_syntax')
|
||||
def target_tasks_try_option_syntax(full_task_graph, parameters):
|
||||
"""Generate a list of target tasks based on try syntax in
|
||||
|
|
@ -33,6 +38,7 @@ def target_tasks_try_option_syntax(full_task_graph, parameters):
|
|||
return [t.label for t in full_task_graph.tasks.itervalues()
|
||||
if options.task_matches(t.attributes)]
|
||||
|
||||
|
||||
@_target_task('all_builds_and_tests')
|
||||
def target_tasks_all_builds_and_tests(full_task_graph, parameters):
|
||||
"""Trivially target all build and test tasks. This is used for
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from ..types import Task, TaskGraph
|
|||
|
||||
from mozunit import main
|
||||
|
||||
|
||||
class FakeKind(object):
|
||||
|
||||
def get_task_definition(self, task, deps_by_name):
|
||||
|
|
@ -83,4 +84,3 @@ class TestCreate(unittest.TestCase):
|
|||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from ..graph import Graph
|
|||
from ..types import Task, TaskGraph
|
||||
from mozunit import main
|
||||
|
||||
|
||||
class TestDecision(unittest.TestCase):
|
||||
|
||||
def test_taskgraph_to_json(self):
|
||||
|
|
@ -43,7 +44,6 @@ class TestDecision(unittest.TestCase):
|
|||
}
|
||||
})
|
||||
|
||||
|
||||
def test_write_artifact_json(self):
|
||||
data = [{'some': 'data'}]
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
|
|
@ -57,7 +57,6 @@ class TestDecision(unittest.TestCase):
|
|||
shutil.rmtree(tmpdir)
|
||||
decision.ARTIFACTS_DIR = 'artifacts'
|
||||
|
||||
|
||||
def test_write_artifact_yml(self):
|
||||
data = [{'some': 'data'}]
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
|
|
@ -74,5 +73,3 @@ class TestDecision(unittest.TestCase):
|
|||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -91,11 +91,13 @@ class TestGenerator(unittest.TestCase):
|
|||
"The optimized task graph contains task ids"
|
||||
self.target_tasks = ['t-2']
|
||||
tid = self.tgg.label_to_taskid
|
||||
self.assertEqual(self.tgg.optimized_task_graph.graph,
|
||||
graph.Graph({tid['t-0'], tid['t-1'], tid['t-2']}, {
|
||||
(tid['t-1'], tid['t-0'], 'prev'),
|
||||
(tid['t-2'], tid['t-1'], 'prev'),
|
||||
}))
|
||||
self.assertEqual(
|
||||
self.tgg.optimized_task_graph.graph,
|
||||
graph.Graph({tid['t-0'], tid['t-1'], tid['t-2']}, {
|
||||
(tid['t-1'], tid['t-0'], 'prev'),
|
||||
(tid['t-2'], tid['t-1'], 'prev'),
|
||||
})
|
||||
)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ class TestGraph(unittest.TestCase):
|
|||
('3', '2', 'green'),
|
||||
}))
|
||||
|
||||
def test_transitive_closure_disjoint(self):
|
||||
def test_transitive_closure_disjoint_edges(self):
|
||||
"transitive closure of a disjoint graph keeps those edges"
|
||||
self.assertEqual(self.disjoint.transitive_closure(set(['3', 'β'])),
|
||||
Graph(set(['1', '2', '3', 'β', 'γ']), {
|
||||
|
|
|
|||
|
|
@ -6,12 +6,10 @@ from __future__ import absolute_import, print_function, unicode_literals
|
|||
|
||||
import unittest
|
||||
import tempfile
|
||||
import shutil
|
||||
import os
|
||||
|
||||
from ..kind import docker_image
|
||||
from ..types import Task
|
||||
from mozunit import main, MockedOpen
|
||||
from mozunit import main
|
||||
|
||||
|
||||
class TestDockerImageKind(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -8,11 +8,9 @@ import unittest
|
|||
|
||||
from ..kind.legacy import (
|
||||
LegacyKind,
|
||||
TASKID_PLACEHOLDER,
|
||||
validate_build_task,
|
||||
BuildTaskValidationException
|
||||
)
|
||||
from ..types import Task
|
||||
from mozunit import main
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,10 +8,8 @@ import unittest
|
|||
|
||||
from ..optimize import optimize_task_graph, resolve_task_references
|
||||
from ..optimize import annotate_task_graph, get_subgraph
|
||||
from .. import optimize
|
||||
from .. import types
|
||||
from .. import graph
|
||||
from mozunit import main
|
||||
|
||||
|
||||
class TestResolveTaskReferences(unittest.TestCase):
|
||||
|
|
@ -47,8 +45,11 @@ class TestResolveTaskReferences(unittest.TestCase):
|
|||
|
||||
def test_invalid(self):
|
||||
"resolve_task_references raises a KeyError on reference to an invalid task"
|
||||
self.assertRaisesRegexp(KeyError, "task 'subject' has no dependency with label 'no-such'", lambda:
|
||||
resolve_task_references('subject', {'task-reference': '<no-such>'}, {}))
|
||||
self.assertRaisesRegexp(
|
||||
KeyError,
|
||||
"task 'subject' has no dependency with label 'no-such'",
|
||||
lambda: resolve_task_references('subject', {'task-reference': '<no-such>'}, {})
|
||||
)
|
||||
|
||||
|
||||
class FakeKind(object):
|
||||
|
|
@ -79,7 +80,9 @@ class TestOptimize(unittest.TestCase):
|
|||
def assert_annotations(self, graph, **annotations):
|
||||
def repl(task_id):
|
||||
return 'SLUGID' if task_id and len(task_id) == 22 else task_id
|
||||
got_annotations = {t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues()}
|
||||
got_annotations = {
|
||||
t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues()
|
||||
}
|
||||
self.assertEqual(got_annotations, annotations)
|
||||
|
||||
def test_annotate_task_graph_no_optimize(self):
|
||||
|
|
@ -92,19 +95,22 @@ class TestOptimize(unittest.TestCase):
|
|||
('task2', 'task1', 'build'),
|
||||
('task2', 'task3', 'image'),
|
||||
)
|
||||
opt = annotate_task_graph(graph, set(),
|
||||
graph.graph.named_links_dict(), {})
|
||||
self.assert_annotations(graph,
|
||||
annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {})
|
||||
self.assert_annotations(
|
||||
graph,
|
||||
task1=(False, None),
|
||||
task2=(False, None),
|
||||
task3=(False, None))
|
||||
task3=(False, None)
|
||||
)
|
||||
|
||||
def test_annotate_task_graph_taskid_without_optimize(self):
|
||||
"raises exception if kind returns a taskid without optimizing"
|
||||
self.make_kind(lambda task, deps: (False, 'some-taskid'))
|
||||
graph = self.make_graph(self.make_task('task1'))
|
||||
self.assertRaises(Exception, lambda:
|
||||
annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}))
|
||||
self.assertRaises(
|
||||
Exception,
|
||||
lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {})
|
||||
)
|
||||
|
||||
def test_annotate_task_graph_optimize_away_dependency(self):
|
||||
"raises exception if kind optimizes away a task on which another depends"
|
||||
|
|
@ -114,8 +120,10 @@ class TestOptimize(unittest.TestCase):
|
|||
self.make_task('task2'),
|
||||
('task2', 'task1', 'build'),
|
||||
)
|
||||
self.assertRaises(Exception, lambda:
|
||||
annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}))
|
||||
self.assertRaises(
|
||||
Exception,
|
||||
lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {})
|
||||
)
|
||||
|
||||
def test_annotate_task_graph_do_not_optimize(self):
|
||||
"annotating marks everything as un-optimized if in do_not_optimize"
|
||||
|
|
@ -126,16 +134,20 @@ class TestOptimize(unittest.TestCase):
|
|||
('task2', 'task1', 'build'),
|
||||
)
|
||||
label_to_taskid = {}
|
||||
opt = annotate_task_graph(graph, {'task1', 'task2'},
|
||||
graph.graph.named_links_dict(), label_to_taskid)
|
||||
self.assert_annotations(graph,
|
||||
annotate_task_graph(graph, {'task1', 'task2'},
|
||||
graph.graph.named_links_dict(), label_to_taskid)
|
||||
self.assert_annotations(
|
||||
graph,
|
||||
task1=(False, None),
|
||||
task2=(False, None))
|
||||
task2=(False, None)
|
||||
)
|
||||
self.assertEqual
|
||||
|
||||
def test_annotate_task_graph_nos_propagate(self):
|
||||
"annotating marks a task with a non-optimized dependency as non-optimized"
|
||||
self.make_kind(lambda task, deps: (False, None) if task.label == 'task1' else (True, 'taskid'))
|
||||
self.make_kind(
|
||||
lambda task, deps: (False, None) if task.label == 'task1' else (True, 'taskid')
|
||||
)
|
||||
graph = self.make_graph(
|
||||
self.make_task('task1'),
|
||||
self.make_task('task2'),
|
||||
|
|
@ -143,12 +155,14 @@ class TestOptimize(unittest.TestCase):
|
|||
('task2', 'task1', 'build'),
|
||||
('task2', 'task3', 'image'),
|
||||
)
|
||||
opt = annotate_task_graph(graph, set(),
|
||||
graph.graph.named_links_dict(), {})
|
||||
self.assert_annotations(graph,
|
||||
annotate_task_graph(graph, set(),
|
||||
graph.graph.named_links_dict(), {})
|
||||
self.assert_annotations(
|
||||
graph,
|
||||
task1=(False, None),
|
||||
task2=(False, None), # kind would have returned (True, 'taskid') here
|
||||
task3=(True, 'taskid'))
|
||||
task3=(True, 'taskid')
|
||||
)
|
||||
|
||||
def test_get_subgraph_single_dep(self):
|
||||
"when a single dependency is optimized, it is omitted from the graph"
|
||||
|
|
@ -207,8 +221,11 @@ class TestOptimize(unittest.TestCase):
|
|||
"get_subgraph resolves task references"
|
||||
graph = self.make_graph(
|
||||
self.make_task('task1', optimized=True, task_id='dep1'),
|
||||
self.make_task('task2', optimized=False,
|
||||
task_def={'payload': {'task-reference': 'http://<build>/<test>'}}),
|
||||
self.make_task(
|
||||
'task2',
|
||||
optimized=False,
|
||||
task_def={'payload': {'task-reference': 'http://<build>/<test>'}}
|
||||
),
|
||||
('task2', 'task1', 'build'),
|
||||
('task2', 'task3', 'test'),
|
||||
self.make_task('task3', optimized=False),
|
||||
|
|
@ -226,7 +243,9 @@ class TestOptimize(unittest.TestCase):
|
|||
|
||||
def test_optimize(self):
|
||||
"optimize_task_graph annotates and extracts the subgraph from a simple graph"
|
||||
self.make_kind(lambda task, deps: (True, 'dep1') if task.label == 'task1' else (False, None))
|
||||
self.make_kind(
|
||||
lambda task, deps: (True, 'dep1') if task.label == 'task1' else (False, None)
|
||||
)
|
||||
input = self.make_graph(
|
||||
self.make_task('task1'),
|
||||
self.make_task('task2'),
|
||||
|
|
|
|||
|
|
@ -9,10 +9,12 @@ import unittest
|
|||
from ..parameters import Parameters, load_parameters_file
|
||||
from mozunit import main, MockedOpen
|
||||
|
||||
|
||||
class TestParameters(unittest.TestCase):
|
||||
|
||||
def test_Parameters_immutable(self):
|
||||
p = Parameters(x=10, y=20)
|
||||
|
||||
def assign():
|
||||
p['x'] = 20
|
||||
self.assertRaises(Exception, assign)
|
||||
|
|
|
|||
|
|
@ -14,19 +14,21 @@ from mozunit import main
|
|||
# an empty graph, for things that don't look at it
|
||||
empty_graph = TaskGraph({}, Graph(set(), set()))
|
||||
|
||||
|
||||
def unittest_task(n, tp):
|
||||
return (n, Task('test', n, {
|
||||
'unittest_try_name': n,
|
||||
'test_platform': tp,
|
||||
}))
|
||||
|
||||
|
||||
def talos_task(n, tp):
|
||||
return (n, Task('test', n, {
|
||||
'talos_try_name': n,
|
||||
'test_platform': tp,
|
||||
}))
|
||||
|
||||
tasks = {k: v for k,v in [
|
||||
tasks = {k: v for k, v in [
|
||||
unittest_task('mochitest-browser-chrome', 'linux'),
|
||||
unittest_task('mochitest-browser-chrome-e10s', 'linux64'),
|
||||
unittest_task('mochitest-chrome', 'linux'),
|
||||
|
|
@ -35,9 +37,9 @@ tasks = {k: v for k,v in [
|
|||
unittest_task('gtest', 'linux64'),
|
||||
talos_task('dromaeojs', 'linux64'),
|
||||
]}
|
||||
unittest_tasks = {k: v for k,v in tasks.iteritems()
|
||||
unittest_tasks = {k: v for k, v in tasks.iteritems()
|
||||
if 'unittest_try_name' in v.attributes}
|
||||
talos_tasks = {k: v for k,v in tasks.iteritems()
|
||||
talos_tasks = {k: v for k, v in tasks.iteritems()
|
||||
if 'talos_try_name' in v.attributes}
|
||||
graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set()))
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import tempfile
|
|||
import unittest
|
||||
|
||||
from ..util import docker
|
||||
from mozunit import main, MockedOpen
|
||||
from mozunit import MockedOpen
|
||||
|
||||
|
||||
class TestDocker(unittest.TestCase):
|
||||
|
|
@ -25,8 +25,10 @@ class TestDocker(unittest.TestCase):
|
|||
f.write("FROM node\nADD a-file\n")
|
||||
with open(os.path.join(tmpdir, 'docker', 'my-image', 'a-file'), "w") as f:
|
||||
f.write("data\n")
|
||||
self.assertEqual(docker.generate_context_hash('docker/my-image'),
|
||||
'781143fcc6cc72c9024b058665265cb6bae3fb8031cad7227dd169ffbfced434')
|
||||
self.assertEqual(
|
||||
docker.generate_context_hash('docker/my-image'),
|
||||
'781143fcc6cc72c9024b058665265cb6bae3fb8031cad7227dd169ffbfced434'
|
||||
)
|
||||
finally:
|
||||
docker.GECKO = old_GECKO
|
||||
shutil.rmtree(tmpdir)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from taskgraph.util.legacy_commit_parser import (
|
|||
parse_test_opts
|
||||
)
|
||||
|
||||
|
||||
class TestCommitParser(unittest.TestCase):
|
||||
|
||||
def test_normalize_test_list_none(self):
|
||||
|
|
@ -22,64 +23,64 @@ class TestCommitParser(unittest.TestCase):
|
|||
def test_normalize_test_list_all(self):
|
||||
self.assertEqual(
|
||||
normalize_test_list({}, ['woot'], 'all'),
|
||||
[{ 'test': 'woot' }]
|
||||
[{'test': 'woot'}]
|
||||
)
|
||||
|
||||
def test_normalize_test_list_specific_tests(self):
|
||||
self.assertEqual(sorted(
|
||||
normalize_test_list({}, ['woot'], 'a,b,c')),
|
||||
sorted([{ 'test': 'a' }, { 'test': 'b' }, { 'test': 'c' }])
|
||||
sorted([{'test': 'a'}, {'test': 'b'}, {'test': 'c'}])
|
||||
)
|
||||
|
||||
def test_normalize_test_list_specific_tests_with_whitespace(self):
|
||||
self.assertEqual(sorted(
|
||||
normalize_test_list({}, ['woot'], 'a, b, c')),
|
||||
sorted([{ 'test': 'a' }, { 'test': 'b' }, { 'test': 'c' }])
|
||||
sorted([{'test': 'a'}, {'test': 'b'}, {'test': 'c'}])
|
||||
)
|
||||
|
||||
def test_normalize_test_list_with_alias(self):
|
||||
self.assertEqual(sorted(
|
||||
normalize_test_list({ "a": "alpha" }, ['woot'], 'a, b, c')),
|
||||
sorted([{ 'test': 'alpha' }, { 'test': 'b' }, { 'test': 'c' }])
|
||||
normalize_test_list({"a": "alpha"}, ['woot'], 'a, b, c')),
|
||||
sorted([{'test': 'alpha'}, {'test': 'b'}, {'test': 'c'}])
|
||||
)
|
||||
|
||||
def test_normalize_test_list_with_alias_and_chunk(self):
|
||||
self.assertEqual(
|
||||
normalize_test_list({ "a": "alpha" }, ['woot'], 'a-1, a-3'),
|
||||
[{ 'test': 'alpha', "only_chunks": set([1, 3]) }]
|
||||
normalize_test_list({"a": "alpha"}, ['woot'], 'a-1, a-3'),
|
||||
[{'test': 'alpha', "only_chunks": set([1, 3])}]
|
||||
)
|
||||
|
||||
def test_normalize_test_list_with_alias_pattern(self):
|
||||
self.assertEqual(sorted(
|
||||
normalize_test_list({ "a": '/.*oo.*/' },
|
||||
['woot', 'foo', 'bar'],
|
||||
'a, b, c')),
|
||||
sorted([{ 'test': t } for t in ['woot', 'foo', 'b', 'c']])
|
||||
self.assertEqual(
|
||||
sorted(normalize_test_list({"a": '/.*oo.*/'},
|
||||
['woot', 'foo', 'bar'],
|
||||
'a, b, c')),
|
||||
sorted([{'test': t} for t in ['woot', 'foo', 'b', 'c']])
|
||||
)
|
||||
|
||||
def test_normalize_test_list_with_alias_pattern_anchored(self):
|
||||
self.assertEqual(sorted(
|
||||
normalize_test_list({ "a": '/.*oo/' },
|
||||
['woot', 'foo', 'bar'],
|
||||
'a, b, c')),
|
||||
sorted([{ 'test': t } for t in ['foo', 'b', 'c']])
|
||||
self.assertEqual(
|
||||
sorted(normalize_test_list({"a": '/.*oo/'},
|
||||
['woot', 'foo', 'bar'],
|
||||
'a, b, c')),
|
||||
sorted([{'test': t} for t in ['foo', 'b', 'c']])
|
||||
)
|
||||
|
||||
def test_normalize_test_list_with_alias_pattern_list(self):
|
||||
self.assertEqual(sorted(
|
||||
normalize_test_list({ "a": ['/.*oo/', 'bar', '/bi.*/'] },
|
||||
['woot', 'foo', 'bar', 'bing', 'baz'],
|
||||
'a, b')),
|
||||
sorted([{ 'test': t } for t in ['foo', 'bar', 'bing', 'b']])
|
||||
self.assertEqual(
|
||||
sorted(normalize_test_list({"a": ['/.*oo/', 'bar', '/bi.*/']},
|
||||
['woot', 'foo', 'bar', 'bing', 'baz'],
|
||||
'a, b')),
|
||||
sorted([{'test': t} for t in ['foo', 'bar', 'bing', 'b']])
|
||||
)
|
||||
|
||||
def test_normalize_test_list_with_alias_pattern_list_chunks(self):
|
||||
self.assertEqual(sorted(
|
||||
normalize_test_list({ "a": ['/.*oo/', 'bar', '/bi.*/'] },
|
||||
['woot', 'foo', 'bar', 'bing', 'baz'],
|
||||
'a-1, a-4, b')),
|
||||
self.assertEqual(
|
||||
sorted(normalize_test_list({"a": ['/.*oo/', 'bar', '/bi.*/']},
|
||||
['woot', 'foo', 'bar', 'bing', 'baz'],
|
||||
'a-1, a-4, b')),
|
||||
sorted([{'test': 'b'}] + [
|
||||
{ 'test': t, 'only_chunks': set([1, 4])} for t in ['foo', 'bar', 'bing']])
|
||||
{'test': t, 'only_chunks': set([1, 4])} for t in ['foo', 'bar', 'bing']])
|
||||
)
|
||||
|
||||
def test_commit_no_tests(self):
|
||||
|
|
@ -327,8 +328,8 @@ class TestCommitParser(unittest.TestCase):
|
|||
'dependents': [{
|
||||
'allowed_build_tasks': {
|
||||
'task/linux': {
|
||||
'task':'task/web-platform-tests',
|
||||
'unittest_try_name':'web-platform-tests'
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests'
|
||||
}
|
||||
}
|
||||
}],
|
||||
|
|
@ -344,7 +345,6 @@ class TestCommitParser(unittest.TestCase):
|
|||
result, triggers = parse_commit(commit, jobs)
|
||||
self.assertEqual(expected, result)
|
||||
|
||||
|
||||
def test_specific_test_platforms(self):
|
||||
'''
|
||||
This test cases covers the platform specific test exclusion options.
|
||||
|
|
@ -658,58 +658,6 @@ class TestCommitParser(unittest.TestCase):
|
|||
}
|
||||
|
||||
expected = [
|
||||
{
|
||||
'task': 'task/linux',
|
||||
'dependents': [
|
||||
{
|
||||
'allowed_build_tasks': {
|
||||
'task/linux': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux-debug': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux64': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux64-debug': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
'additional-parameters': {}
|
||||
},
|
||||
{
|
||||
'task': 'task/linux-debug',
|
||||
'dependents': [
|
||||
{
|
||||
'allowed_build_tasks': {
|
||||
'task/linux': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux-debug': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux64': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux64-debug': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
'additional-parameters': {}
|
||||
},
|
||||
{
|
||||
'task': 'task/linux64',
|
||||
'dependents': [
|
||||
|
|
@ -734,6 +682,11 @@ class TestCommitParser(unittest.TestCase):
|
|||
}
|
||||
}
|
||||
],
|
||||
'build_name': 'linux64',
|
||||
'build_type': 'opt',
|
||||
'interactive': False,
|
||||
'post-build': [],
|
||||
'when': {},
|
||||
'additional-parameters': {}
|
||||
},
|
||||
{
|
||||
|
|
@ -760,6 +713,73 @@ class TestCommitParser(unittest.TestCase):
|
|||
}
|
||||
}
|
||||
],
|
||||
'build_name': 'linux64',
|
||||
'build_type': 'debug',
|
||||
'interactive': False,
|
||||
'post-build': [],
|
||||
'when': {},
|
||||
'additional-parameters': {}
|
||||
},
|
||||
{
|
||||
'task': 'task/linux',
|
||||
'dependents': [
|
||||
{
|
||||
'allowed_build_tasks': {
|
||||
'task/linux': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux-debug': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux64': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux64-debug': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
'build_name': 'linux',
|
||||
'build_type': 'opt',
|
||||
'interactive': False,
|
||||
'post-build': [],
|
||||
'when': {},
|
||||
'additional-parameters': {}
|
||||
},
|
||||
{
|
||||
'task': 'task/linux-debug',
|
||||
'dependents': [
|
||||
{
|
||||
'allowed_build_tasks': {
|
||||
'task/linux': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux-debug': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux64': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
},
|
||||
'task/linux64-debug': {
|
||||
'task': 'task/web-platform-tests',
|
||||
'unittest_try_name': 'web-platform-tests',
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
'build_name': 'linux',
|
||||
'build_type': 'debug',
|
||||
'interactive': False,
|
||||
'post-build': [],
|
||||
'when': {},
|
||||
'additional-parameters': {}
|
||||
}
|
||||
]
|
||||
|
|
@ -767,11 +787,12 @@ class TestCommitParser(unittest.TestCase):
|
|||
result, triggers = parse_commit(commit, jobs)
|
||||
self.assertEqual(expected, result)
|
||||
|
||||
def test_commit_with_builds_and_tests(self):
|
||||
def test_commit_long_form(self):
|
||||
'''
|
||||
This tests the long form of the try flags.
|
||||
'''
|
||||
commit = 'try: --build od --platform linux,linux64 --unittests web-platform-tests --talos none'
|
||||
commit = \
|
||||
'try: --build od --platform linux,linux64 --unittests web-platform-tests --talos none'
|
||||
jobs = {
|
||||
'flags': {
|
||||
'builds': ['linux', 'linux64'],
|
||||
|
|
@ -954,30 +975,30 @@ class TryTestParserTest(unittest.TestCase):
|
|||
def test_parse_opts_valid(self):
|
||||
self.assertEquals(
|
||||
parse_test_opts('all[Mulet Linux]'),
|
||||
[{ 'test': 'all', 'platforms': ['Mulet Linux'] }]
|
||||
[{'test': 'all', 'platforms': ['Mulet Linux']}]
|
||||
)
|
||||
|
||||
self.assertEquals(
|
||||
parse_test_opts('all[Amazing, Foobar woot,yeah]'),
|
||||
[{ 'test': 'all', 'platforms': ['Amazing', 'Foobar woot', 'yeah'] }]
|
||||
[{'test': 'all', 'platforms': ['Amazing', 'Foobar woot', 'yeah']}]
|
||||
)
|
||||
|
||||
self.assertEquals(
|
||||
parse_test_opts('a,b, c'),
|
||||
[
|
||||
{ 'test': 'a' },
|
||||
{ 'test': 'b' },
|
||||
{ 'test': 'c' },
|
||||
{'test': 'a'},
|
||||
{'test': 'b'},
|
||||
{'test': 'c'},
|
||||
]
|
||||
)
|
||||
self.assertEquals(
|
||||
parse_test_opts('woot, bar[b], baz, qux[ z ],a'),
|
||||
[
|
||||
{ 'test': 'woot' },
|
||||
{ 'test': 'bar', 'platforms': ['b'] },
|
||||
{ 'test': 'baz' },
|
||||
{ 'test': 'qux', 'platforms': ['z'] },
|
||||
{ 'test': 'a' }
|
||||
{'test': 'woot'},
|
||||
{'test': 'bar', 'platforms': ['b']},
|
||||
{'test': 'baz'},
|
||||
{'test': 'qux', 'platforms': ['z']},
|
||||
{'test': 'a'}
|
||||
]
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import os
|
||||
|
||||
import unittest
|
||||
import mozunit
|
||||
import textwrap
|
||||
|
|
@ -153,8 +151,7 @@ class TemplatesTest(unittest.TestCase):
|
|||
'a': 'overriden'
|
||||
})
|
||||
|
||||
self.assertEqual(content, { 'values': ['overriden', 'b', 'c'] });
|
||||
|
||||
self.assertEqual(content, {'values': ['overriden', 'b', 'c']})
|
||||
|
||||
def test_inheritance_circular(self):
|
||||
'''
|
||||
|
|
@ -167,7 +164,7 @@ class TemplatesTest(unittest.TestCase):
|
|||
content = self.subject.load('deep/4.yml', {
|
||||
'value': 'myvalue'
|
||||
})
|
||||
self.assertEqual(content, { 'variable': 'myvalue' })
|
||||
self.assertEqual(content, {'variable': 'myvalue'})
|
||||
|
||||
def test_inheritance_with_simple_extensions(self):
|
||||
content = self.subject.load('extend_parent.yml', {})
|
||||
|
|
@ -181,7 +178,7 @@ class TemplatesTest(unittest.TestCase):
|
|||
},
|
||||
'level': 2,
|
||||
},
|
||||
'was_list': { 'replaced': True }
|
||||
'was_list': {'replaced': True}
|
||||
})
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from taskgraph.util.time import (
|
|||
json_time_from_now
|
||||
)
|
||||
|
||||
|
||||
class FromNowTest(unittest.TestCase):
|
||||
|
||||
def test_invalid_str(self):
|
||||
|
|
@ -43,14 +44,14 @@ class FromNowTest(unittest.TestCase):
|
|||
|
||||
def test_json_from_now_utc_now(self):
|
||||
# Just here to ensure we don't raise.
|
||||
time = json_time_from_now('1 years')
|
||||
json_time_from_now('1 years')
|
||||
|
||||
def test_json_from_now(self):
|
||||
now = datetime(2014, 1, 1)
|
||||
self.assertEqual(json_time_from_now('1 years', now),
|
||||
'2015-01-01T00:00:00Z')
|
||||
'2015-01-01T00:00:00Z')
|
||||
self.assertEqual(json_time_from_now('6 days', now),
|
||||
'2014-01-07T00:00:00Z')
|
||||
'2014-01-07T00:00:00Z')
|
||||
|
||||
if __name__ == '__main__':
|
||||
mozunit.main()
|
||||
|
|
|
|||
|
|
@ -18,14 +18,17 @@ BUILD_TYPE_ALIASES = {
|
|||
'd': 'debug'
|
||||
}
|
||||
|
||||
|
||||
# mapping from shortcut name (usable with -u) to a boolean function identifying
|
||||
# matching test names
|
||||
def alias_prefix(prefix):
|
||||
return lambda name: name.startswith(prefix)
|
||||
|
||||
|
||||
def alias_contains(infix):
|
||||
return lambda name: infix in name
|
||||
|
||||
|
||||
def alias_matches(pattern):
|
||||
pattern = re.compile(pattern)
|
||||
return lambda name: pattern.match(name)
|
||||
|
|
@ -102,18 +105,18 @@ UNITTEST_PLATFORM_PRETTY_NAMES = {
|
|||
'x64': ['linux64'],
|
||||
# other commonly-used substrings for platforms not yet supported with
|
||||
# in-tree taskgraphs:
|
||||
#'10.10': [..TODO..],
|
||||
#'10.10.5': [..TODO..],
|
||||
#'10.6': [..TODO..],
|
||||
#'10.8': [..TODO..],
|
||||
#'Android 2.3 API9': [..TODO..],
|
||||
#'Android 4.3 API15+': [..TODO..],
|
||||
#'Windows 7': [..TODO..],
|
||||
#'Windows 7 VM': [..TODO..],
|
||||
#'Windows 8': [..TODO..],
|
||||
#'Windows XP': [..TODO..],
|
||||
#'win32': [..TODO..],
|
||||
#'win64': [..TODO..],
|
||||
# '10.10': [..TODO..],
|
||||
# '10.10.5': [..TODO..],
|
||||
# '10.6': [..TODO..],
|
||||
# '10.8': [..TODO..],
|
||||
# 'Android 2.3 API9': [..TODO..],
|
||||
# 'Android 4.3 API15+': [..TODO..],
|
||||
# 'Windows 7': [..TODO..],
|
||||
# 'Windows 7 VM': [..TODO..],
|
||||
# 'Windows 8': [..TODO..],
|
||||
# 'Windows XP': [..TODO..],
|
||||
# 'win32': [..TODO..],
|
||||
# 'win64': [..TODO..],
|
||||
}
|
||||
|
||||
# We have a few platforms for which we want to do some "extra" builds, or at
|
||||
|
|
@ -137,6 +140,7 @@ RIDEALONG_BUILDS = {
|
|||
|
||||
TEST_CHUNK_SUFFIX = re.compile('(.*)-([0-9]+)$')
|
||||
|
||||
|
||||
class TryOptionSyntax(object):
|
||||
|
||||
def __init__(self, message, full_task_graph):
|
||||
|
|
@ -186,10 +190,13 @@ class TryOptionSyntax(object):
|
|||
# Argument parser based on try flag flags
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-b', '--build', dest='build_types')
|
||||
parser.add_argument('-p', '--platform', nargs='?', dest='platforms', const='all', default='all')
|
||||
parser.add_argument('-u', '--unittests', nargs='?', dest='unittests', const='all', default='all')
|
||||
parser.add_argument('-p', '--platform', nargs='?',
|
||||
dest='platforms', const='all', default='all')
|
||||
parser.add_argument('-u', '--unittests', nargs='?',
|
||||
dest='unittests', const='all', default='all')
|
||||
parser.add_argument('-t', '--talos', nargs='?', dest='talos', const='all', default='all')
|
||||
parser.add_argument('-i', '--interactive', dest='interactive', action='store_true', default=False)
|
||||
parser.add_argument('-i', '--interactive',
|
||||
dest='interactive', action='store_true', default=False)
|
||||
parser.add_argument('-j', '--job', dest='jobs', action='append')
|
||||
# In order to run test jobs multiple times
|
||||
parser.add_argument('--trigger-tests', dest='trigger_tests', type=int, default=1)
|
||||
|
|
@ -198,7 +205,8 @@ class TryOptionSyntax(object):
|
|||
self.jobs = self.parse_jobs(args.jobs)
|
||||
self.build_types = self.parse_build_types(args.build_types)
|
||||
self.platforms = self.parse_platforms(args.platforms)
|
||||
self.unittests = self.parse_test_option("unittest_try_name", args.unittests, full_task_graph)
|
||||
self.unittests = self.parse_test_option(
|
||||
"unittest_try_name", args.unittests, full_task_graph)
|
||||
self.talos = self.parse_test_option("talos_try_name", args.talos, full_task_graph)
|
||||
self.trigger_tests = args.trigger_tests
|
||||
self.interactive = args.interactive
|
||||
|
|
@ -214,8 +222,8 @@ class TryOptionSyntax(object):
|
|||
def parse_build_types(self, build_types_arg):
|
||||
if build_types_arg is None:
|
||||
build_types_arg = []
|
||||
build_types = filter(None, [ BUILD_TYPE_ALIASES.get(build_type) for
|
||||
build_type in build_types_arg ])
|
||||
build_types = filter(None, [BUILD_TYPE_ALIASES.get(build_type) for
|
||||
build_type in build_types_arg])
|
||||
return build_types
|
||||
|
||||
def parse_platforms(self, platform_arg):
|
||||
|
|
@ -373,6 +381,7 @@ class TryOptionSyntax(object):
|
|||
return [test]
|
||||
|
||||
alias = UNITTEST_ALIASES[test['test']]
|
||||
|
||||
def mktest(name):
|
||||
newtest = copy.deepcopy(test)
|
||||
newtest['test'] = name
|
||||
|
|
@ -383,7 +392,6 @@ class TryOptionSyntax(object):
|
|||
|
||||
return [mktest(t) for t in exprmatch(alias)]
|
||||
|
||||
|
||||
def parse_test_chunks(self, all_tests, tests):
|
||||
'''
|
||||
Test flags may include parameters to narrow down the number of chunks in a
|
||||
|
|
@ -501,7 +509,7 @@ class TryOptionSyntax(object):
|
|||
def none_for_all(list):
|
||||
if list is None:
|
||||
return '<all>'
|
||||
return ', '.join(str (e) for e in list)
|
||||
return ', '.join(str(e) for e in list)
|
||||
|
||||
return "\n".join([
|
||||
"build_types: " + ", ".join(self.build_types),
|
||||
|
|
@ -511,4 +519,3 @@ class TryOptionSyntax(object):
|
|||
"trigger_tests: " + str(self.trigger_tests),
|
||||
"interactive: " + str(self.interactive),
|
||||
])
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
|
||||
class Task(object):
|
||||
"""
|
||||
Representation of a task in a TaskGraph.
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
|
|||
DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
|
||||
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
|
||||
|
||||
|
||||
def docker_image(name):
|
||||
'''Determine the docker image name, including repository and tag, from an
|
||||
in-tree docker file.'''
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ BUILD_TYPE_ALIASES = {
|
|||
'd': 'debug'
|
||||
}
|
||||
|
||||
|
||||
def parse_test_opts(input_str):
|
||||
'''Test argument parsing is surprisingly complicated with the "restrictions"
|
||||
logic this function is responsible for parsing this out into a easier to
|
||||
|
|
@ -109,6 +110,7 @@ def normalize_platform_list(alias, all_builds, build_list):
|
|||
return all_builds
|
||||
return [alias.get(build, build) for build in build_list.split(',')]
|
||||
|
||||
|
||||
def normalize_test_list(aliases, all_tests, job_list):
|
||||
'''
|
||||
Normalize a set of jobs (builds or tests) there are three common cases:
|
||||
|
|
@ -137,7 +139,7 @@ def normalize_test_list(aliases, all_tests, job_list):
|
|||
results = []
|
||||
all_entry = tests[0]
|
||||
for test in all_tests:
|
||||
entry = { 'test': test }
|
||||
entry = {'test': test}
|
||||
# If there are platform restrictions copy them across the list.
|
||||
if 'platforms' in all_entry:
|
||||
entry['platforms'] = list(all_entry['platforms'])
|
||||
|
|
@ -160,6 +162,7 @@ def handle_alias(test, aliases, all_tests):
|
|||
return [test]
|
||||
|
||||
alias = aliases[test['test']]
|
||||
|
||||
def mktest(name):
|
||||
newtest = copy.deepcopy(test)
|
||||
newtest['test'] = name
|
||||
|
|
@ -218,6 +221,7 @@ def parse_test_chunks(aliases, all_tests, tests):
|
|||
results = {test['test']: test for test in results}.values()
|
||||
return results
|
||||
|
||||
|
||||
def extract_tests_from_platform(test_jobs, build_platform, build_task, tests):
|
||||
'''
|
||||
Build the list of tests from the current build.
|
||||
|
|
@ -283,6 +287,7 @@ not try to build a graph or anything here but match up build flags to tasks via
|
|||
the "jobs" datastructure (see job_flags.yml)
|
||||
'''
|
||||
|
||||
|
||||
def parse_commit(message, jobs):
|
||||
'''
|
||||
:param message: Commit message that is typical to a try push.
|
||||
|
|
@ -303,9 +308,11 @@ def parse_commit(message, jobs):
|
|||
# Argument parser based on try flag flags
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-b', '--build', dest='build_types')
|
||||
parser.add_argument('-p', '--platform', nargs='?', dest='platforms', const='all', default='all')
|
||||
parser.add_argument('-p', '--platform', nargs='?',
|
||||
dest='platforms', const='all', default='all')
|
||||
parser.add_argument('-u', '--unittests', nargs='?', dest='tests', const='all', default='all')
|
||||
parser.add_argument('-i', '--interactive', dest='interactive', action='store_true', default=False)
|
||||
parser.add_argument('-i', '--interactive',
|
||||
dest='interactive', action='store_true', default=False)
|
||||
parser.add_argument('-j', '--job', dest='jobs', action='append')
|
||||
# In order to run test jobs multiple times
|
||||
parser.add_argument('--trigger-tests', dest='trigger_tests', type=int, default=1)
|
||||
|
|
@ -328,8 +335,8 @@ def parse_commit(message, jobs):
|
|||
if args.build_types is None:
|
||||
args.build_types = []
|
||||
|
||||
build_types = [ BUILD_TYPE_ALIASES.get(build_type, build_type) for
|
||||
build_type in args.build_types ]
|
||||
build_types = [BUILD_TYPE_ALIASES.get(build_type, build_type) for
|
||||
build_type in args.build_types]
|
||||
|
||||
aliases = jobs['flags'].get('aliases', {})
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import yaml
|
|||
# Key used in template inheritance...
|
||||
INHERITS_KEY = '$inherits'
|
||||
|
||||
|
||||
def merge_to(source, dest):
|
||||
'''
|
||||
Merge dict and arrays (override scalar values)
|
||||
|
|
@ -16,7 +17,7 @@ def merge_to(source, dest):
|
|||
|
||||
for key, value in source.items():
|
||||
# Override mismatching or empty types
|
||||
if type(value) != type(dest.get(key)):
|
||||
if type(value) != type(dest.get(key)): # noqa
|
||||
dest[key] = source[key]
|
||||
continue
|
||||
|
||||
|
|
@ -33,9 +34,11 @@ def merge_to(source, dest):
|
|||
|
||||
return dest
|
||||
|
||||
|
||||
class TemplatesException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Templates():
|
||||
'''
|
||||
The taskcluster integration makes heavy use of yaml to describe tasks this
|
||||
|
|
@ -54,7 +57,7 @@ class Templates():
|
|||
if not os.path.isdir(root):
|
||||
raise TemplatesException('Root must be a directory')
|
||||
|
||||
self.root = root;
|
||||
self.root = root
|
||||
|
||||
def _inherits(self, path, obj, properties, seen):
|
||||
blueprint = obj.pop(INHERITS_KEY)
|
||||
|
|
@ -86,8 +89,6 @@ class Templates():
|
|||
# Anything left in obj is merged into final results (and overrides)
|
||||
return merge_to(obj, out)
|
||||
|
||||
|
||||
|
||||
def render(self, path, content, parameters, seen):
|
||||
'''
|
||||
Renders a given yaml string.
|
||||
|
|
|
|||
|
|
@ -8,26 +8,32 @@
|
|||
import re
|
||||
import datetime
|
||||
|
||||
PATTERN=re.compile(
|
||||
PATTERN = re.compile(
|
||||
'((?:\d+)?\.?\d+) *([a-z]+)'
|
||||
)
|
||||
|
||||
|
||||
def seconds(value):
|
||||
return datetime.timedelta(seconds=int(value))
|
||||
|
||||
|
||||
def minutes(value):
|
||||
return datetime.timedelta(minutes=int(value))
|
||||
|
||||
|
||||
def hours(value):
|
||||
return datetime.timedelta(hours=int(value))
|
||||
|
||||
|
||||
def days(value):
|
||||
return datetime.timedelta(days=int(value))
|
||||
|
||||
|
||||
def months(value):
|
||||
# See warning in years(), below
|
||||
return datetime.timedelta(days=int(value) * 30)
|
||||
|
||||
|
||||
def years(value):
|
||||
# Warning here "years" are vague don't use this for really sensitive date
|
||||
# computation the idea is to give you a absolute amount of time in the
|
||||
|
|
@ -42,12 +48,15 @@ ALIASES['days'] = ALIASES['day'] = ALIASES['d'] = days
|
|||
ALIASES['months'] = ALIASES['month'] = ALIASES['mo'] = months
|
||||
ALIASES['years'] = ALIASES['year'] = ALIASES['y'] = years
|
||||
|
||||
|
||||
class InvalidString(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownTimeMeasurement(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def value_of(input_str):
|
||||
'''
|
||||
Convert a string to a json date in the future
|
||||
|
|
@ -64,12 +73,15 @@ def value_of(input_str):
|
|||
|
||||
if unit not in ALIASES:
|
||||
raise UnknownTimeMeasurement(
|
||||
'{} is not a valid time measure use one of {}'.format(unit,
|
||||
sorted(ALIASES.keys()))
|
||||
'{} is not a valid time measure use one of {}'.format(
|
||||
unit,
|
||||
sorted(ALIASES.keys())
|
||||
)
|
||||
)
|
||||
|
||||
return ALIASES[unit](value)
|
||||
|
||||
|
||||
def json_time_from_now(input_str, now=None):
|
||||
'''
|
||||
:param str input_str: Input string (see value of)
|
||||
|
|
@ -86,10 +98,10 @@ def json_time_from_now(input_str, now=None):
|
|||
# ISO dates until 'Z' (for timezone) is added...
|
||||
return time.isoformat() + 'Z'
|
||||
|
||||
|
||||
def current_json_time():
|
||||
'''
|
||||
:returns: JSON string representation of the current time.
|
||||
'''
|
||||
|
||||
return datetime.datetime.utcnow().isoformat() + 'Z'
|
||||
|
||||
|
|
|
|||
|
|
@ -124,7 +124,8 @@ LINTER = {
|
|||
'include': [
|
||||
'python/mozlint',
|
||||
'tools/lint',
|
||||
'testing/marionette/client'
|
||||
'taskcluster',
|
||||
'testing/marionette/client',
|
||||
],
|
||||
'exclude': [],
|
||||
'type': 'external',
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче