зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1382775 - Move autotry logic from |mach try| into autotry proper, r=chmanchester
This is a straightforward copy of code from the mach_commands.py to autotry.py MozReview-Commit-ID: 7TkbTff0Tv8 --HG-- extra : rebase_source : 7996131427217d9f0213af920d0d4ef0d2e7d0ac extra : source : f0693a73539265f74f79db9d1e136e7f1c16a1f0
This commit is contained in:
Andrew Halberstadt
Родитель
7b130edcb5
Коммит
2a6c9e6348
|
|
@ -4,27 +4,17 @@
|
|||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
import argparse
|
||||
|
||||
from mach.decorators import (
|
||||
CommandArgument,
|
||||
CommandProvider,
|
||||
Command,
|
||||
SubCommand,
|
||||
)
|
||||
|
||||
import mozpack.path as mozpath
|
||||
from mozbuild.base import BuildEnvironmentNotFoundException, MachCommandBase
|
||||
|
||||
CONFIG_ENVIRONMENT_NOT_FOUND = '''
|
||||
No config environment detected. This means we are unable to properly
|
||||
detect test files in the specified paths or tags. Please run:
|
||||
|
||||
$ mach configure
|
||||
|
||||
and try again.
|
||||
'''.lstrip()
|
||||
|
||||
|
||||
def syntax_parser():
|
||||
from tryselect.selectors.syntax import arg_parser
|
||||
|
|
@ -43,95 +33,35 @@ def syntax_parser():
|
|||
|
||||
|
||||
@CommandProvider
|
||||
class PushToTry(MachCommandBase):
|
||||
def normalise_list(self, items, allow_subitems=False):
|
||||
from tryselect.selectors.syntax import parse_arg
|
||||
|
||||
rv = defaultdict(list)
|
||||
for item in items:
|
||||
parsed = parse_arg(item)
|
||||
for key, values in parsed.iteritems():
|
||||
rv[key].extend(values)
|
||||
|
||||
if not allow_subitems:
|
||||
if not all(item == [] for item in rv.itervalues()):
|
||||
raise ValueError("Unexpected subitems in argument")
|
||||
return rv.keys()
|
||||
else:
|
||||
return rv
|
||||
|
||||
def validate_args(self, **kwargs):
|
||||
from tryselect.selectors.syntax import AutoTry
|
||||
|
||||
tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"]
|
||||
if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected):
|
||||
if 'AUTOTRY_PLATFORM_HINT' in os.environ:
|
||||
kwargs["platforms"] = [os.environ['AUTOTRY_PLATFORM_HINT']]
|
||||
elif tests_selected:
|
||||
print("Must specify platform when selecting tests.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("Either platforms or jobs must be specified as an argument to "
|
||||
"|mach try syntax|.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
platforms = (self.normalise_list(kwargs["platforms"])
|
||||
if kwargs["platforms"] else {})
|
||||
except ValueError as e:
|
||||
print("Error parsing -p argument:\n%s" % e.message)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
tests = (self.normalise_list(kwargs["tests"], allow_subitems=True)
|
||||
if kwargs["tests"] else {})
|
||||
except ValueError as e:
|
||||
print("Error parsing -u argument (%s):\n%s" % (kwargs["tests"], e.message))
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
talos = (self.normalise_list(kwargs["talos"], allow_subitems=True)
|
||||
if kwargs["talos"] else [])
|
||||
except ValueError as e:
|
||||
print("Error parsing -t argument:\n%s" % e.message)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
jobs = (self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {})
|
||||
except ValueError as e:
|
||||
print("Error parsing -j argument:\n%s" % e.message)
|
||||
sys.exit(1)
|
||||
|
||||
paths = []
|
||||
for p in kwargs["paths"]:
|
||||
p = mozpath.normpath(os.path.abspath(p))
|
||||
if not (os.path.isdir(p) and p.startswith(self.topsrcdir)):
|
||||
print('Specified path "%s" is not a directory under the srcdir,'
|
||||
' unable to specify tests outside of the srcdir' % p)
|
||||
sys.exit(1)
|
||||
if len(p) <= len(self.topsrcdir):
|
||||
print('Specified path "%s" is at the top of the srcdir and would'
|
||||
' select all tests.' % p)
|
||||
sys.exit(1)
|
||||
paths.append(os.path.relpath(p, self.topsrcdir))
|
||||
|
||||
try:
|
||||
tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else []
|
||||
except ValueError as e:
|
||||
print("Error parsing --tags argument:\n%s" % e.message)
|
||||
sys.exit(1)
|
||||
|
||||
extra_values = {k['dest'] for k in AutoTry.pass_through_arguments.values()}
|
||||
extra_args = {k: v for k, v in kwargs.items()
|
||||
if k in extra_values and v}
|
||||
|
||||
return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args
|
||||
class TrySelect(MachCommandBase):
|
||||
|
||||
@Command('try',
|
||||
category='testing',
|
||||
description='Push selected tests to the try server',
|
||||
parser=syntax_parser)
|
||||
def syntax(self, **kwargs):
|
||||
category='ci',
|
||||
description='Push selected tasks to the try server')
|
||||
@CommandArgument('args', nargs=argparse.REMAINDER)
|
||||
def try_default(self, args):
|
||||
"""Push selected tests to the try server.
|
||||
|
||||
The |mach try| command is a frontend for scheduling tasks to
|
||||
run on try server using selectors. A selector is a subcommand
|
||||
that provides its own set of command line arguments and are
|
||||
listed below. Currently there is only single selector called
|
||||
`syntax`, but more selectors will be added in the future.
|
||||
|
||||
If no subcommand is specified, the `syntax` selector is run by
|
||||
default. Run |mach try syntax --help| for more information on
|
||||
scheduling with the `syntax` selector.
|
||||
"""
|
||||
parser = syntax_parser()
|
||||
kwargs = vars(parser.parse_args(args))
|
||||
return self._mach_context.commands.dispatch(
|
||||
'try', subcommand='syntax', context=self._mach_context, **kwargs)
|
||||
|
||||
@SubCommand('try',
|
||||
'syntax',
|
||||
description='Push selected tasks using try syntax',
|
||||
parser=syntax_parser)
|
||||
def try_syntax(self, **kwargs):
|
||||
"""Push the current tree to try, with the specified syntax.
|
||||
|
||||
Build options, platforms and regression tests may be selected
|
||||
|
|
@ -169,107 +99,20 @@ class PushToTry(MachCommandBase):
|
|||
(available at https://github.com/glandium/git-cinnabar).
|
||||
|
||||
"""
|
||||
|
||||
from mozbuild.testing import TestResolver
|
||||
from tryselect.selectors.syntax import AutoTry
|
||||
|
||||
print("mach try is under development, please file bugs blocking 1149670.")
|
||||
try:
|
||||
if self.substs.get("MOZ_ARTIFACT_BUILDS"):
|
||||
kwargs['local_artifact_build'] = True
|
||||
except BuildEnvironmentNotFoundException:
|
||||
# If we don't have a build locally, we can't tell whether
|
||||
# an artifact build is desired, but we still want the
|
||||
# command to succeed, if possible.
|
||||
pass
|
||||
|
||||
def resolver_func():
|
||||
return self._spawn(TestResolver)
|
||||
|
||||
at = AutoTry(self.topsrcdir, resolver_func, self._mach_context)
|
||||
|
||||
if kwargs["list"]:
|
||||
at.list_presets()
|
||||
sys.exit()
|
||||
|
||||
if kwargs["load"] is not None:
|
||||
defaults = at.load_config(kwargs["load"])
|
||||
|
||||
if defaults is None:
|
||||
print("No saved configuration called %s found in autotry.ini" % kwargs["load"],
|
||||
file=sys.stderr)
|
||||
|
||||
for key, value in kwargs.iteritems():
|
||||
if value in (None, []) and key in defaults:
|
||||
kwargs[key] = defaults[key]
|
||||
|
||||
if kwargs["push"] and at.find_uncommited_changes():
|
||||
print('ERROR please commit changes before continuing')
|
||||
sys.exit(1)
|
||||
|
||||
if not any(kwargs[item] for item in ("paths", "tests", "tags")):
|
||||
kwargs["paths"], kwargs["tags"] = at.find_paths_and_tags(kwargs["verbose"])
|
||||
|
||||
builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args(**kwargs)
|
||||
|
||||
if paths or tags:
|
||||
if not os.path.exists(os.path.join(self.topobjdir, 'config.status')):
|
||||
print(CONFIG_ENVIRONMENT_NOT_FOUND)
|
||||
sys.exit(1)
|
||||
|
||||
paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
|
||||
for item in paths]
|
||||
paths_by_flavor = at.paths_by_flavor(paths=paths, tags=tags)
|
||||
|
||||
if not paths_by_flavor and not tests:
|
||||
print("No tests were found when attempting to resolve paths:\n\n\t%s" %
|
||||
paths)
|
||||
sys.exit(1)
|
||||
|
||||
if not kwargs["intersection"]:
|
||||
paths_by_flavor = at.remove_duplicates(paths_by_flavor, tests)
|
||||
else:
|
||||
paths_by_flavor = {}
|
||||
|
||||
# No point in dealing with artifacts if we aren't running any builds
|
||||
local_artifact_build = False
|
||||
if platforms:
|
||||
try:
|
||||
if self.substs.get("MOZ_ARTIFACT_BUILDS"):
|
||||
local_artifact_build = True
|
||||
except BuildEnvironmentNotFoundException:
|
||||
# If we don't have a build locally, we can't tell whether
|
||||
# an artifact build is desired, but we still want the
|
||||
# command to succeed, if possible.
|
||||
pass
|
||||
|
||||
# Add --artifact if --enable-artifact-builds is set ...
|
||||
if local_artifact_build:
|
||||
extra["artifact"] = True
|
||||
# ... unless --no-artifact is explicitly given.
|
||||
if kwargs["no_artifact"]:
|
||||
if "artifact" in extra:
|
||||
del extra["artifact"]
|
||||
|
||||
try:
|
||||
msg = at.calc_try_syntax(platforms, tests, talos, jobs, builds, paths_by_flavor, tags,
|
||||
extra, kwargs["intersection"])
|
||||
except ValueError as e:
|
||||
print(e.message)
|
||||
sys.exit(1)
|
||||
|
||||
if local_artifact_build:
|
||||
if kwargs["no_artifact"]:
|
||||
print('mozconfig has --enable-artifact-builds but '
|
||||
'--no-artifact specified, not including --artifact '
|
||||
'flag in try syntax')
|
||||
else:
|
||||
print('mozconfig has --enable-artifact-builds; including '
|
||||
'--artifact flag in try syntax (use --no-artifact '
|
||||
'to override)')
|
||||
|
||||
if kwargs["verbose"] and paths_by_flavor:
|
||||
print('The following tests will be selected: ')
|
||||
for flavor, paths in paths_by_flavor.iteritems():
|
||||
print("%s: %s" % (flavor, ",".join(paths)))
|
||||
|
||||
if kwargs["verbose"] or not kwargs["push"]:
|
||||
print('The following try syntax was calculated:\n%s' % msg)
|
||||
|
||||
if kwargs["push"]:
|
||||
at.push_to_try(msg, kwargs["verbose"])
|
||||
|
||||
if kwargs["save"] is not None:
|
||||
at.save_config(kwargs["save"], msg)
|
||||
return at.run(**kwargs)
|
||||
|
|
|
|||
|
|
@ -2,16 +2,27 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import ConfigParser
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import which
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
import ConfigParser
|
||||
import mozpack.path as mozpath
|
||||
|
||||
CONFIG_ENVIRONMENT_NOT_FOUND = '''
|
||||
No config environment detected. This means we are unable to properly
|
||||
detect test files in the specified paths or tags. Please run:
|
||||
|
||||
$ mach configure
|
||||
|
||||
and try again.
|
||||
'''.lstrip()
|
||||
|
||||
|
||||
def arg_parser():
|
||||
|
|
@ -613,3 +624,169 @@ class AutoTry(object):
|
|||
print("Pushing tests based on the following tags:\n\t%s" %
|
||||
"\n\t".join(tags))
|
||||
return paths, tags
|
||||
|
||||
def normalise_list(self, items, allow_subitems=False):
|
||||
rv = defaultdict(list)
|
||||
for item in items:
|
||||
parsed = parse_arg(item)
|
||||
for key, values in parsed.iteritems():
|
||||
rv[key].extend(values)
|
||||
|
||||
if not allow_subitems:
|
||||
if not all(item == [] for item in rv.itervalues()):
|
||||
raise ValueError("Unexpected subitems in argument")
|
||||
return rv.keys()
|
||||
else:
|
||||
return rv
|
||||
|
||||
def validate_args(self, **kwargs):
|
||||
tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"]
|
||||
if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected):
|
||||
if 'AUTOTRY_PLATFORM_HINT' in os.environ:
|
||||
kwargs["platforms"] = [os.environ['AUTOTRY_PLATFORM_HINT']]
|
||||
elif tests_selected:
|
||||
print("Must specify platform when selecting tests.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("Either platforms or jobs must be specified as an argument to autotry.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
platforms = (self.normalise_list(kwargs["platforms"])
|
||||
if kwargs["platforms"] else {})
|
||||
except ValueError as e:
|
||||
print("Error parsing -p argument:\n%s" % e.message)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
tests = (self.normalise_list(kwargs["tests"], allow_subitems=True)
|
||||
if kwargs["tests"] else {})
|
||||
except ValueError as e:
|
||||
print("Error parsing -u argument (%s):\n%s" % (kwargs["tests"], e.message))
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
talos = (self.normalise_list(kwargs["talos"], allow_subitems=True)
|
||||
if kwargs["talos"] else [])
|
||||
except ValueError as e:
|
||||
print("Error parsing -t argument:\n%s" % e.message)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
jobs = (self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {})
|
||||
except ValueError as e:
|
||||
print("Error parsing -j argument:\n%s" % e.message)
|
||||
sys.exit(1)
|
||||
|
||||
paths = []
|
||||
for p in kwargs["paths"]:
|
||||
p = mozpath.normpath(os.path.abspath(p))
|
||||
if not (os.path.isdir(p) and p.startswith(self.topsrcdir)):
|
||||
print('Specified path "%s" is not a directory under the srcdir,'
|
||||
' unable to specify tests outside of the srcdir' % p)
|
||||
sys.exit(1)
|
||||
if len(p) <= len(self.topsrcdir):
|
||||
print('Specified path "%s" is at the top of the srcdir and would'
|
||||
' select all tests.' % p)
|
||||
sys.exit(1)
|
||||
paths.append(os.path.relpath(p, self.topsrcdir))
|
||||
|
||||
try:
|
||||
tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else []
|
||||
except ValueError as e:
|
||||
print("Error parsing --tags argument:\n%s" % e.message)
|
||||
sys.exit(1)
|
||||
|
||||
extra_values = {k['dest'] for k in AutoTry.pass_through_arguments.values()}
|
||||
extra_args = {k: v for k, v in kwargs.items()
|
||||
if k in extra_values and v}
|
||||
|
||||
return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args
|
||||
|
||||
def run(self, **kwargs):
|
||||
if kwargs["list"]:
|
||||
self.list_presets()
|
||||
sys.exit()
|
||||
|
||||
if kwargs["load"] is not None:
|
||||
defaults = self.load_config(kwargs["load"])
|
||||
|
||||
if defaults is None:
|
||||
print("No saved configuration called %s found in autotry.ini" % kwargs["load"],
|
||||
file=sys.stderr)
|
||||
|
||||
for key, value in kwargs.iteritems():
|
||||
if value in (None, []) and key in defaults:
|
||||
kwargs[key] = defaults[key]
|
||||
|
||||
if kwargs["push"] and self.find_uncommited_changes():
|
||||
print('ERROR please commit changes before continuing')
|
||||
sys.exit(1)
|
||||
|
||||
if not any(kwargs[item] for item in ("paths", "tests", "tags")):
|
||||
kwargs["paths"], kwargs["tags"] = self.find_paths_and_tags(kwargs["verbose"])
|
||||
|
||||
builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args(**kwargs)
|
||||
|
||||
if paths or tags:
|
||||
if not os.path.exists(os.path.join(self.topobjdir, 'config.status')):
|
||||
print(CONFIG_ENVIRONMENT_NOT_FOUND)
|
||||
sys.exit(1)
|
||||
|
||||
paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
|
||||
for item in paths]
|
||||
paths_by_flavor = self.paths_by_flavor(paths=paths, tags=tags)
|
||||
|
||||
if not paths_by_flavor and not tests:
|
||||
print("No tests were found when attempting to resolve paths:\n\n\t%s" %
|
||||
paths)
|
||||
sys.exit(1)
|
||||
|
||||
if not kwargs["intersection"]:
|
||||
paths_by_flavor = self.remove_duplicates(paths_by_flavor, tests)
|
||||
else:
|
||||
paths_by_flavor = {}
|
||||
|
||||
# No point in dealing with artifacts if we aren't running any builds
|
||||
local_artifact_build = False
|
||||
if platforms:
|
||||
local_artifact_build = kwargs.get('local_artifact_build', False)
|
||||
|
||||
# Add --artifact if --enable-artifact-builds is set ...
|
||||
if local_artifact_build:
|
||||
extra["artifact"] = True
|
||||
# ... unless --no-artifact is explicitly given.
|
||||
if kwargs["no_artifact"]:
|
||||
if "artifact" in extra:
|
||||
del extra["artifact"]
|
||||
|
||||
try:
|
||||
msg = self.calc_try_syntax(platforms, tests, talos, jobs, builds,
|
||||
paths_by_flavor, tags, extra, kwargs["intersection"])
|
||||
except ValueError as e:
|
||||
print(e.message)
|
||||
sys.exit(1)
|
||||
|
||||
if local_artifact_build:
|
||||
if kwargs["no_artifact"]:
|
||||
print('mozconfig has --enable-artifact-builds but '
|
||||
'--no-artifact specified, not including --artifact '
|
||||
'flag in try syntax')
|
||||
else:
|
||||
print('mozconfig has --enable-artifact-builds; including '
|
||||
'--artifact flag in try syntax (use --no-artifact '
|
||||
'to override)')
|
||||
|
||||
if kwargs["verbose"] and paths_by_flavor:
|
||||
print('The following tests will be selected: ')
|
||||
for flavor, paths in paths_by_flavor.iteritems():
|
||||
print("%s: %s" % (flavor, ",".join(paths)))
|
||||
|
||||
if kwargs["verbose"] or not kwargs["push"]:
|
||||
print('The following try syntax was calculated:\n%s' % msg)
|
||||
|
||||
if kwargs["push"]:
|
||||
self.push_to_try(msg, kwargs["verbose"])
|
||||
|
||||
if kwargs["save"] is not None:
|
||||
self.save_config(kwargs["save"], msg)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче