2017-07-27 18:48:53 +03:00
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
|
2018-11-23 21:17:33 +03:00
|
|
|
import glob
|
2018-08-06 13:36:09 +03:00
|
|
|
import hashlib
|
2018-11-23 21:17:33 +03:00
|
|
|
import json
|
2017-07-27 18:48:53 +03:00
|
|
|
import os
|
2018-08-21 23:00:32 +03:00
|
|
|
import re
|
2018-08-06 13:36:09 +03:00
|
|
|
import shutil
|
2017-09-28 22:25:34 +03:00
|
|
|
import sys
|
2018-11-15 19:54:13 +03:00
|
|
|
from collections import defaultdict
|
2017-07-27 18:48:53 +03:00
|
|
|
|
|
|
|
from mozboot.util import get_state_dir
|
|
|
|
from mozbuild.base import MozbuildObject
|
|
|
|
from mozpack.files import FileFinder
|
2018-11-13 13:17:01 +03:00
|
|
|
from moztest.resolve import TestResolver, get_suite_definition
|
2017-07-27 18:48:53 +03:00
|
|
|
|
2017-12-07 17:01:35 +03:00
|
|
|
import taskgraph
|
2017-07-27 18:48:53 +03:00
|
|
|
from taskgraph.generator import TaskGraphGenerator
|
2017-09-28 22:25:34 +03:00
|
|
|
from taskgraph.parameters import (
|
|
|
|
ParameterMismatch,
|
2019-02-15 00:34:49 +03:00
|
|
|
parameters_loader,
|
2017-09-28 22:25:34 +03:00
|
|
|
)
|
2018-11-23 21:17:33 +03:00
|
|
|
from taskgraph.taskgraph import TaskGraph
|
2017-07-27 18:48:53 +03:00
|
|
|
|
|
|
|
here = os.path.abspath(os.path.dirname(__file__))
|
|
|
|
build = MozbuildObject.from_environment(cwd=here)
|
|
|
|
|
|
|
|
|
2017-09-28 22:25:34 +03:00
|
|
|
PARAMETER_MISMATCH = """
|
2019-02-07 22:38:13 +03:00
|
|
|
ERROR - The parameters being used to generate tasks differ from those expected
|
|
|
|
by your working copy:
|
2017-09-28 22:25:34 +03:00
|
|
|
|
|
|
|
{}
|
|
|
|
|
|
|
|
To fix this, either rebase onto the latest mozilla-central or pass in
|
|
|
|
-p/--parameters. For more information on how to define parameters, see:
|
|
|
|
https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
2017-11-21 23:39:21 +03:00
|
|
|
def invalidate(cache, root):
|
2017-07-27 18:48:53 +03:00
|
|
|
if not os.path.isfile(cache):
|
|
|
|
return
|
|
|
|
|
2017-11-21 23:39:21 +03:00
|
|
|
tc_dir = os.path.join(root, 'taskcluster')
|
2017-07-27 18:48:53 +03:00
|
|
|
tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
|
|
|
|
cmod = os.path.getmtime(cache)
|
|
|
|
|
|
|
|
if tmod > cmod:
|
|
|
|
os.remove(cache)
|
|
|
|
|
|
|
|
|
2017-11-21 23:39:21 +03:00
|
|
|
def generate_tasks(params, full, root):
|
2017-08-15 22:22:56 +03:00
|
|
|
params = params or "project=mozilla-central"
|
|
|
|
|
2018-08-06 13:36:09 +03:00
|
|
|
# Try to delete the old taskgraph cache directory.
|
2019-02-04 23:52:31 +03:00
|
|
|
old_cache_dir = os.path.join(get_state_dir(), 'cache', 'taskgraph')
|
2018-08-06 13:36:09 +03:00
|
|
|
if os.path.isdir(old_cache_dir):
|
|
|
|
shutil.rmtree(old_cache_dir)
|
|
|
|
|
|
|
|
root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest()
|
2019-02-04 23:52:31 +03:00
|
|
|
cache_dir = os.path.join(get_state_dir(), 'cache', root_hash, 'taskgraph')
|
2018-11-23 21:17:33 +03:00
|
|
|
|
|
|
|
# Cleanup old cache files
|
|
|
|
for path in glob.glob(os.path.join(cache_dir, '*_set')):
|
|
|
|
os.remove(path)
|
|
|
|
|
|
|
|
attr = 'full_task_graph' if full else 'target_task_graph'
|
2017-08-15 22:22:56 +03:00
|
|
|
cache = os.path.join(cache_dir, attr)
|
2017-07-27 18:48:53 +03:00
|
|
|
|
2017-11-21 23:39:21 +03:00
|
|
|
invalidate(cache, root)
|
2017-07-27 18:48:53 +03:00
|
|
|
if os.path.isfile(cache):
|
|
|
|
with open(cache, 'r') as fh:
|
2018-11-23 21:17:33 +03:00
|
|
|
return TaskGraph.from_json(json.load(fh))[1]
|
2017-07-27 18:48:53 +03:00
|
|
|
|
|
|
|
if not os.path.isdir(cache_dir):
|
|
|
|
os.makedirs(cache_dir)
|
|
|
|
|
2017-08-15 22:22:56 +03:00
|
|
|
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
|
2017-07-27 18:48:53 +03:00
|
|
|
|
2017-12-07 17:01:35 +03:00
|
|
|
taskgraph.fast = True
|
2017-09-16 07:31:45 +03:00
|
|
|
cwd = os.getcwd()
|
|
|
|
os.chdir(build.topsrcdir)
|
|
|
|
|
2017-11-21 23:39:21 +03:00
|
|
|
root = os.path.join(root, 'taskcluster', 'ci')
|
2019-02-15 00:34:49 +03:00
|
|
|
params = parameters_loader(params, strict=False, overrides={'try_mode': 'try_select'})
|
|
|
|
try:
|
|
|
|
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
|
|
|
|
except ParameterMismatch as e:
|
|
|
|
print(PARAMETER_MISMATCH.format(e.args[0]))
|
|
|
|
sys.exit(1)
|
2018-08-21 23:00:32 +03:00
|
|
|
|
2017-09-16 07:31:45 +03:00
|
|
|
os.chdir(cwd)
|
|
|
|
|
2017-07-27 18:48:53 +03:00
|
|
|
with open(cache, 'w') as fh:
|
2018-11-23 21:17:33 +03:00
|
|
|
json.dump(tg.to_json(), fh)
|
|
|
|
return tg
|
2018-11-13 13:17:01 +03:00
|
|
|
|
|
|
|
|
|
|
|
def filter_tasks_by_paths(tasks, paths):
|
|
|
|
resolver = TestResolver.from_environment(cwd=here)
|
|
|
|
run_suites, run_tests = resolver.resolve_metadata(paths)
|
|
|
|
flavors = set([(t['flavor'], t.get('subsuite')) for t in run_tests])
|
|
|
|
|
|
|
|
task_regexes = set()
|
|
|
|
for flavor, subsuite in flavors:
|
|
|
|
suite = get_suite_definition(flavor, subsuite, strict=True)
|
|
|
|
if 'task_regex' not in suite:
|
|
|
|
print("warning: no tasks could be resolved from flavor '{}'{}".format(
|
|
|
|
flavor, " and subsuite '{}'".format(subsuite) if subsuite else ""))
|
|
|
|
continue
|
|
|
|
|
|
|
|
task_regexes.update(suite['task_regex'])
|
|
|
|
|
|
|
|
def match_task(task):
|
|
|
|
return any(re.search(pattern, task) for pattern in task_regexes)
|
|
|
|
|
|
|
|
return filter(match_task, tasks)
|
2018-11-15 19:54:13 +03:00
|
|
|
|
|
|
|
|
|
|
|
def resolve_tests_by_suite(paths):
|
|
|
|
resolver = TestResolver.from_environment(cwd=here)
|
|
|
|
_, run_tests = resolver.resolve_metadata(paths)
|
|
|
|
|
|
|
|
suite_to_tests = defaultdict(list)
|
|
|
|
for test in run_tests:
|
|
|
|
key = test['flavor']
|
|
|
|
subsuite = test.get('subsuite')
|
|
|
|
if subsuite:
|
|
|
|
key += '-' + subsuite
|
|
|
|
suite_to_tests[key].append(test['srcdir_relpath'])
|
|
|
|
|
|
|
|
return suite_to_tests
|