mozilla
/
chromium-src-build
зеркало из https://github.com/mozilla/chromium-src-build.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Fuchsia: modify scripts to run non-test executables 

Fuchsia: modify scripts to run non-test executables
* Move most test runner functionality to "runner_common.py", with
  the exception of gtest-specific logic.
* Move the test-specific "main()" portions into "test_runner.py".
* Add "exe_runner.py", a very thin wrapper around the test_common
  functionality.
* Use stripped executables for building the bootfs file, which results
  in a substantially smaller file size (10x difference!)
  It's necessary for running larger executables like "headless_shell",
  which are enormous when unstripped, resulting in long mkbootfs times
  and higher RAM requirements for in-memory tmpfs disks. The unstripped
  executable remains intact alongside the stripped executable, for
  backtrace symbolization.
* Add a GN template "fuchsia_executable_runner", which generates
  executable runner scripts on non-test targets.


Bug: 746674,752364
Change-Id: Ia39c3d7fdf9bb574049f0112df84951c2e048f41
Reviewed-on: https://chromium-review.googlesource.com/602410
Commit-Queue: Kevin Marshall <kmarshall@chromium.org>
Reviewed-by: Sami Kyöstilä <skyostil@chromium.org>
Reviewed-by: Scott Graham <scottmg@chromium.org>
Cr-Original-Commit-Position: refs/heads/master@{#493524}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 143e53fb43b6dfbee260273daaa4b033c3edf1ec
This commit is contained in:
Kevin Marshall 2017-08-10 20:33:25 +00:00 коммит произвёл Commit Bot
Родитель d52819082d
Коммит 7c87e0ecbc
5 изменённых файлов: 463 добавлений и 341 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

55
config/fuchsia/rules.gni
Просмотреть файл

@ -4,12 +4,7 @@
assert(is_fuchsia)
# This template is used to generate a runner script for test binaries into the
# build dir for Fuchsia. It's generally used from the "test" template.
template("test_runner_script") {
testonly = true
_test_name = invoker.test_name
template("generate_runner_script") {
# This runtime_deps file is used at runtime and thus cannot go in
# target_gen_dir.
_target_dir_name = get_label_info(":$target_name", "dir")
@ -23,6 +18,7 @@ template("test_runner_script") {
"data_deps",
"deps",
"public_deps",
"testonly",
])
write_runtime_deps = _runtime_deps_file
}
@ -32,6 +28,9 @@ template("test_runner_script") {
[
"data_deps",
"deps",
"runner_script",
"target",
"testonly",
])
if (!defined(deps)) {
deps = []
@ -40,39 +39,67 @@ template("test_runner_script") {
data_deps = []
}
script = "//build/fuchsia/create_test_runner_script.py"
script = "//build/fuchsia/create_runner_script.py"
depfile = "$target_gen_dir/$target_name.d"
data = []
test_runner_args = []
runner_args = []
generated_script = "$root_build_dir/bin/run_${_test_name}"
generated_script = "$root_build_dir/bin/run_${target}"
outputs = [
generated_script,
]
data += [ generated_script ]
test_runner_args += [
runner_args += [
"--runner-script",
runner_script,
"--output-directory",
rebase_path(root_build_dir, root_build_dir),
]
deps += [ ":$_runtime_deps_target" ]
data += [ _runtime_deps_file ]
test_runner_args += [
runner_args += [
"--runtime-deps-path",
rebase_path(_runtime_deps_file, root_build_dir),
]
if (defined(args)) {
args = []
}
args = [
"--depfile",
rebase_path(depfile, root_build_dir),
"--script-output-path",
rebase_path(generated_script, root_build_dir),
"--test-name",
_test_name,
"--exe-name",
target,
]
args += test_runner_args
args += runner_args
}
}
# This template is used to generate a runner script for test binaries into the
# build dir for Fuchsia. It's generally used from the "test" template.
template("test_runner_script") {
generate_runner_script(target_name) {
testonly = true
runner_script = "test_runner.py"
target = invoker.test_name
forward_variables_from(invoker, "*")
}
}
# This template is used to generate a runner script for arbitrary executables
# into the build dir for Fuchsia. The template should be instantiated alongside
# an "executable" target, and referenced from the executable via its "deps"
# attribute.
template("fuchsia_executable_runner") {
generate_runner_script(target_name) {
runner_script = "exe_runner.py"
target = invoker.exe_name
forward_variables_from(invoker, "*")
}
}

51
fuchsia/create_test_runner_script.py → fuchsia/create_runner_script.py
Просмотреть файл

@ -4,8 +4,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Creates a script to run a Fushsia test (typically on QEMU) by delegating to
build/fuchsia/test_runner.py.
"""Creates a script to run a Fushsia executable by delegating to
build/fuchsia/(exe|test)_runner.py.
"""
import argparse
@ -17,7 +17,7 @@ import sys
SCRIPT_TEMPLATE = """\
#!/usr/bin/env python
#
# This file was generated by build/fuchsia/create_test_runner_script.py
# This file was generated by build/fuchsia/create_runner_script.py
import os
import sys
@ -30,14 +30,14 @@ def main():
\"\"\"
return os.path.abspath(os.path.join(script_directory, path))
test_runner_path = ResolvePath('{test_runner_path}')
test_runner_args = {test_runner_args}
test_runner_path_args = {test_runner_path_args}
for arg, path in test_runner_path_args:
test_runner_args.extend([arg, ResolvePath(path)])
runner_path = ResolvePath('{runner_path}')
runner_args = {runner_args}
runner_path_args = {runner_path_args}
for arg, path in runner_path_args:
runner_args.extend([arg, ResolvePath(path)])
os.execv(test_runner_path,
[test_runner_path] + test_runner_args + sys.argv[1:])
os.execv(runner_path,
[runner_path] + runner_args + sys.argv[1:])
if __name__ == '__main__':
sys.exit(main())
@ -65,6 +65,8 @@ def WriteDepfile(depfile_path, first_gn_output, inputs=None):
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--runner-script',
help='Name of the runner script to use.')
parser.add_argument('--script-output-path',
help='Output path for executable script.')
parser.add_argument('--depfile',
@ -75,33 +77,30 @@ def main(args):
group = parser.add_argument_group('Test runner path arguments.')
group.add_argument('--output-directory')
group.add_argument('--runtime-deps-path')
group.add_argument('--test-name')
args, test_runner_args = parser.parse_known_args(args)
group.add_argument('--exe-name')
args, runner_args = parser.parse_known_args(args)
def RelativizePathToScript(path):
"""Returns the path relative to the output script directory."""
return os.path.relpath(path, os.path.dirname(args.script_output_path))
test_runner_path = args.test_runner_path or os.path.join(
os.path.dirname(__file__), 'test_runner.py')
test_runner_path = RelativizePathToScript(test_runner_path)
runner_path = args.test_runner_path or os.path.join(
os.path.dirname(__file__), args.runner_script)
runner_path = RelativizePathToScript(runner_path)
test_runner_path_args = []
if args.output_directory:
test_runner_path_args.append(
runner_path_args = []
runner_path_args.append(
('--output-directory', RelativizePathToScript(args.output_directory)))
if args.runtime_deps_path:
test_runner_path_args.append(
runner_path_args.append(
('--runtime-deps-path', RelativizePathToScript(args.runtime_deps_path)))
if args.test_name:
test_runner_path_args.append(
('--test-name', RelativizePathToScript(args.test_name)))
runner_path_args.append(
('--exe-name', RelativizePathToScript(args.exe_name)))
with open(args.script_output_path, 'w') as script:
script.write(SCRIPT_TEMPLATE.format(
test_runner_path=str(test_runner_path),
test_runner_args=str(test_runner_args),
test_runner_path_args=str(test_runner_path_args)))
runner_path=str(runner_path),
runner_args=str(runner_args),
runner_path_args=str(runner_path_args)))
os.chmod(args.script_output_path, 0750)

47
fuchsia/exe_runner.py Executable file
Просмотреть файл

@ -0,0 +1,47 @@
#!/usr/bin/env python
#
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Packages a user.bootfs for a Fuchsia boot image, pulling in the runtime
dependencies of a binary, and then uses either QEMU from the Fuchsia SDK
to run, or starts the bootserver to allow running on a hardware device."""
import argparse
import os
import sys
from runner_common import RunFuchsia, BuildBootfs, ReadRuntimeDeps
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dry-run', '-n', action='store_true', default=False,
help='Just print commands, don\'t execute them.')
parser.add_argument('--output-directory',
type=os.path.realpath,
help=('Path to the directory in which build files are'
' located (must include build type).'))
parser.add_argument('--runtime-deps-path',
type=os.path.realpath,
help='Runtime data dependency file from GN.')
parser.add_argument('--exe-name',
type=os.path.realpath,
help='Name of the the binary executable.')
parser.add_argument('-d', '--device', action='store_true', default=False,
help='Run on hardware device instead of QEMU.')
args, child_args = parser.parse_known_args()
bootfs = BuildBootfs(
args.output_directory,
ReadRuntimeDeps(args.runtime_deps_path, args.output_directory),
args.exe_name, child_args, args.device, args.dry_run)
if not bootfs:
return 2
return RunFuchsia(bootfs, args.exe_name, args.device, args.dry_run)
if __name__ == '__main__':
sys.exit(main())

327
fuchsia/runner_common.py Executable file
Просмотреть файл

@ -0,0 +1,327 @@
#!/usr/bin/env python
#
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Packages a user.bootfs for a Fuchsia boot image, pulling in the runtime
dependencies of a binary, and then uses either QEMU from the Fuchsia SDK
to run, or starts the bootserver to allow running on a hardware device."""
import argparse
import multiprocessing
import os
import re
import shutil
import signal
import subprocess
import sys
import tempfile
DIR_SOURCE_ROOT = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
SDK_ROOT = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'fuchsia-sdk')
SYMBOLIZATION_TIMEOUT_SECS = 10
def _RunAndCheck(dry_run, args):
if dry_run:
print 'Run:', ' '.join(args)
return 0
else:
try:
subprocess.check_call(args)
return 0
except subprocess.CalledProcessError as e:
return e.returncode
def _DumpFile(dry_run, name, description):
"""Prints out the contents of |name| if |dry_run|."""
if not dry_run:
return
print
print 'Contents of %s (for %s)' % (name, description)
print '-' * 80
with open(name) as f:
sys.stdout.write(f.read())
print '-' * 80
def MakeTargetImageName(common_prefix, output_directory, location):
"""Generates the relative path name to be used in the file system image.
common_prefix: a prefix of both output_directory and location that
be removed.
output_directory: an optional prefix on location that will also be removed.
location: the file path to relativize.
.so files will be stored into the lib subdirectory to be able to be found by
default by the loader.
Examples:
>>> MakeTargetImageName(common_prefix='/work/cr/src/',
... output_directory='/work/cr/src/out/fuch',
... location='/work/cr/src/base/test/data/xyz.json')
'base/test/data/xyz.json'
>>> MakeTargetImageName(common_prefix='/work/cr/src/',
... output_directory='/work/cr/src/out/fuch',
... location='/work/cr/src/out/fuch/icudtl.dat')
'icudtl.dat'
>>> MakeTargetImageName(common_prefix='/work/cr/src/',
... output_directory='/work/cr/src/out/fuch',
... location='/work/cr/src/out/fuch/libbase.so')
'lib/libbase.so'
"""
assert output_directory.startswith(common_prefix)
output_dir_no_common_prefix = output_directory[len(common_prefix):]
assert location.startswith(common_prefix)
loc = location[len(common_prefix):]
if loc.startswith(output_dir_no_common_prefix):
loc = loc[len(output_dir_no_common_prefix)+1:]
# TODO(fuchsia): The requirements for finding/loading .so are in flux, so this
# ought to be reconsidered at some point. See https://crbug.com/732897.
if location.endswith('.so'):
loc = 'lib/' + loc
return loc
def _AddToManifest(manifest_file, target_name, source, mapper):
"""Appends |source| to the given |manifest_file| (a file object) in a format
suitable for consumption by mkbootfs.
If |source| is a file it's directly added. If |source| is a directory, its
contents are recursively added.
|source| must exist on disk at the time this function is called.
"""
if os.path.isdir(source):
files = [os.path.join(dp, f) for dp, dn, fn in os.walk(source) for f in fn]
for f in files:
# We pass None as the mapper because this should never recurse a 2nd time.
_AddToManifest(manifest_file, mapper(f), f, None)
elif os.path.exists(source):
manifest_file.write('%s=%s\n' % (target_name, source))
else:
raise Exception('%s does not exist' % source)
def ReadRuntimeDeps(deps_path, output_dir):
return [os.path.abspath(os.path.join(output_dir, x.strip()))
for x in open(deps_path)]
def _StripBinary(dry_run, bin_path):
strip_path = bin_path + '_stripped';
shutil.copyfile(bin_path, strip_path)
_RunAndCheck(dry_run, ['/usr/bin/strip', strip_path])
return strip_path
def BuildBootfs(output_directory, runtime_deps, bin_name, child_args,
device, dry_run):
locations_to_add = [os.path.abspath(os.path.join(output_directory, x.strip()))
for x in runtime_deps]
common_prefix = '/'
if len(locations_to_add) > 1:
common_prefix = os.path.commonprefix(locations_to_add)
target_source_pairs = zip(
[MakeTargetImageName(common_prefix, output_directory, loc)
for loc in locations_to_add],
locations_to_add)
# Stage the stripped binary in the boot image, keeping the original binary's
# name for symbolization purposes.
bin_path = os.path.abspath(os.path.join(output_directory, bin_name))
stripped_bin_path = _StripBinary(dry_run, bin_path)
target_source_pairs.append(
[MakeTargetImageName(common_prefix, output_directory, bin_path),
stripped_bin_path])
# Generate a script that runs the binaries and shuts down QEMU (if used).
autorun_file = tempfile.NamedTemporaryFile()
autorun_file.write('#!/bin/sh\n')
autorun_file.write('echo Executing ' + os.path.basename(bin_name) + ' ' +
' '.join(child_args) + '\n')
autorun_file.write('/system/' + os.path.basename(bin_name))
for arg in child_args:
autorun_file.write(' "%s"' % arg);
autorun_file.write('\n')
autorun_file.write('echo Process terminated.\n')
if not device:
# If shutdown of QEMU happens too soon after the program finishes, log
# statements from the end of the run will be lost, so sleep for a bit before
# shutting down. When running on device don't power off so the output and
# system can be inspected.
autorun_file.write('msleep 3000\n')
autorun_file.write('dm poweroff\n')
autorun_file.flush()
os.chmod(autorun_file.name, 0750)
_DumpFile(dry_run, autorun_file.name, 'autorun')
target_source_pairs.append(('autorun', autorun_file.name))
manifest_file = tempfile.NamedTemporaryFile()
bootfs_name = bin_name + '.bootfs'
for target, source in target_source_pairs:
_AddToManifest(manifest_file.file, target, source,
lambda x: MakeTargetImageName(
common_prefix, output_directory, x))
mkbootfs_path = os.path.join(SDK_ROOT, 'tools', 'mkbootfs')
manifest_file.flush()
_DumpFile(dry_run, manifest_file.name, 'manifest')
if _RunAndCheck(
dry_run,
[mkbootfs_path, '-o', bootfs_name,
'--target=boot', os.path.join(SDK_ROOT, 'bootdata.bin'),
'--target=system', manifest_file.name]) != 0:
return None
return bootfs_name
def _SymbolizeEntry(entry):
addr2line_output = subprocess.check_output(
['addr2line', '-Cipf', '--exe=' + entry[1], entry[2]])
prefix = '#%s: ' % entry[0]
# addr2line outputs a second line for inlining information, offset
# that to align it properly after the frame index.
addr2line_filtered = addr2line_output.strip().replace(
'(inlined', ' ' * len(prefix) + '(inlined')
if '??' in addr2line_filtered:
addr2line_filtered = "%s+%s" % (os.path.basename(entry[1]), entry[2])
return '%s%s' % (prefix, addr2line_filtered)
def _ParallelSymbolizeBacktrace(backtrace):
# Disable handling of SIGINT during sub-process creation, to prevent
# sub-processes from consuming Ctrl-C signals, rather than the parent
# process doing so.
saved_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
p = multiprocessing.Pool(multiprocessing.cpu_count())
# Restore the signal handler for the parent process.
signal.signal(signal.SIGINT, saved_sigint_handler)
symbolized = []
try:
result = p.map_async(_SymbolizeEntry, backtrace)
symbolized = result.get(SYMBOLIZATION_TIMEOUT_SECS)
if not symbolized:
return []
except multiprocessing.TimeoutError:
return ['(timeout error occurred during symbolization)']
except KeyboardInterrupt: # SIGINT
p.terminate()
return symbolized
def RunFuchsia(bootfs, exe_name, use_device, dry_run):
kernel_path = os.path.join(SDK_ROOT, 'kernel', 'magenta.bin')
if use_device:
# TODO(fuchsia): This doesn't capture stdout as there's no way to do so
# currently. See https://crbug.com/749242.
bootserver_path = os.path.join(SDK_ROOT, 'tools', 'bootserver')
bootserver_command = [bootserver_path, '-1', kernel_path, bootfs]
return _RunAndCheck(dry_run, bootserver_command)
qemu_path = os.path.join(SDK_ROOT, 'qemu', 'bin', 'qemu-system-x86_64')
qemu_command = [qemu_path,
'-m', '2048',
'-nographic',
'-net', 'none',
'-smp', '4',
'-machine', 'q35',
'-kernel', kernel_path,
'-initrd', bootfs,
# Use stdio for the guest OS only; don't attach the QEMU interactive
# monitor.
'-serial', 'stdio',
'-monitor', 'none',
# TERM=dumb tells the guest OS to not emit ANSI commands that trigger
# noisy ANSI spew from the user's terminal emulator.
'-append', 'TERM=dumb kernel.halt_on_panic=true']
if int(os.environ.get('CHROME_HEADLESS', 0)) == 0:
qemu_command += ['-enable-kvm', '-cpu', 'host,migratable=no']
else:
qemu_command += ['-cpu', 'Haswell,+smap,-check']
if dry_run:
print 'Run:', ' '.join(qemu_command)
return 0
# Set up backtrace-parsing regexps.
prefix = r'^.*> '
bt_end_re = re.compile(prefix + '(bt)?#(\d+):? end')
bt_with_offset_re = re.compile(
prefix + 'bt#(\d+): pc 0x[0-9a-f]+ sp (0x[0-9a-f]+) ' +
'\((\S+),(0x[0-9a-f]+)\)$')
in_process_re = re.compile(prefix +
'#(\d+) 0x[0-9a-f]+ \S+\+(0x[0-9a-f]+)$')
# We pass a separate stdin stream to qemu. Sharing stdin across processes
# leads to flakiness due to the OS prematurely killing the stream and the
# Python script panicking and aborting.
# The precise root cause is still nebulous, but this fix works.
# See crbug.com/741194 .
qemu_popen = subprocess.Popen(
qemu_command, stdout=subprocess.PIPE, stdin=open(os.devnull))
# A buffer of backtrace entries awaiting symbolization, stored as tuples.
# Element #0: backtrace frame number (starting at 0).
# Element #1: path to executable code corresponding to the current frame.
# Element #2: memory offset within the executable.
bt_entries = []
success = False
while True:
line = qemu_popen.stdout.readline().strip()
if not line:
break
if 'SUCCESS: all tests passed.' in line:
success = True
# Check for an end-of-backtrace marker.
if bt_end_re.match(line):
if bt_entries:
print '----- start symbolized stack'
for processed in _ParallelSymbolizeBacktrace(bt_entries):
print processed
print '----- end symbolized stack'
bt_entries = []
continue
# Try to parse this as a Fuchsia system backtrace.
m = bt_with_offset_re.match(line)
if m:
bt_entries.append((m.group(1), exe_name, m.group(4)))
continue
# Try to parse the line as an in-process backtrace entry.
m = in_process_re.match(line)
if m:
bt_entries.append((m.group(1), exe_name, m.group(2)))
continue
# Some other line, so print it. Back-traces should not be interleaved with
# other output, so while this may re-order lines we see, it should actually
# make things more readable.
print line
qemu_popen.wait()
return 0 if success else 1

320
fuchsia/test_runner.py
Просмотреть файл

@ -9,207 +9,11 @@ dependencies of a test binary, and then uses either QEMU from the Fuchsia SDK
to run, or starts the bootserver to allow running on a hardware device."""
import argparse
import multiprocessing
import os
import re
import signal
import subprocess
import sys
import tempfile
DIR_SOURCE_ROOT = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
SDK_ROOT = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'fuchsia-sdk')
SYMBOLIZATION_TIMEOUT_SECS = 10
def RunAndCheck(dry_run, args):
if dry_run:
print 'Run:', args
else:
try:
subprocess.check_call(args)
return 0
except subprocess.CalledProcessError as e:
return e.returncode
def DumpFile(dry_run, name, description):
"""Prints out the contents of |name| if |dry_run|."""
if not dry_run:
return
print
print 'Contents of %s (for %s)' % (name, description)
print '-' * 80
with open(name) as f:
sys.stdout.write(f.read())
print '-' * 80
def MakeTargetImageName(common_prefix, output_directory, location):
"""Generates the relative path name to be used in the file system image.
common_prefix: a prefix of both output_directory and location that
be removed.
output_directory: an optional prefix on location that will also be removed.
location: the file path to relativize.
.so files will be stored into the lib subdirectory to be able to be found by
default by the loader.
Examples:
>>> MakeTargetImageName(common_prefix='/work/cr/src/',
... output_directory='/work/cr/src/out/fuch',
... location='/work/cr/src/base/test/data/xyz.json')
'base/test/data/xyz.json'
>>> MakeTargetImageName(common_prefix='/work/cr/src/',
... output_directory='/work/cr/src/out/fuch',
... location='/work/cr/src/out/fuch/icudtl.dat')
'icudtl.dat'
>>> MakeTargetImageName(common_prefix='/work/cr/src/',
... output_directory='/work/cr/src/out/fuch',
... location='/work/cr/src/out/fuch/libbase.so')
'lib/libbase.so'
"""
assert output_directory.startswith(common_prefix)
output_dir_no_common_prefix = output_directory[len(common_prefix):]
assert location.startswith(common_prefix)
loc = location[len(common_prefix):]
if loc.startswith(output_dir_no_common_prefix):
loc = loc[len(output_dir_no_common_prefix)+1:]
# TODO(fuchsia): The requirements for finding/loading .so are in flux, so this
# ought to be reconsidered at some point. See https://crbug.com/732897.
if location.endswith('.so'):
loc = 'lib/' + loc
return loc
def AddToManifest(manifest_file, target_name, source, mapper):
"""Appends |source| to the given |manifest_file| (a file object) in a format
suitable for consumption by mkbootfs.
If |source| is a file it's directly added. If |source| is a directory, its
contents are recursively added.
|source| must exist on disk at the time this function is called.
"""
if os.path.isdir(source):
files = [os.path.join(dp, f) for dp, dn, fn in os.walk(source) for f in fn]
for f in files:
# We pass None as the mapper because this should never recurse a 2nd time.
AddToManifest(manifest_file, mapper(f), f, None)
elif os.path.exists(source):
manifest_file.write('%s=%s\n' % (target_name, source))
else:
raise Exception('%s does not exist' % source)
def BuildBootfs(output_directory, runtime_deps_path, test_name, child_args,
test_launcher_filter_file, device, dry_run):
with open(runtime_deps_path) as f:
lines = f.readlines()
locations_to_add = [os.path.abspath(os.path.join(output_directory, x.strip()))
for x in lines]
locations_to_add.append(
os.path.abspath(os.path.join(output_directory, test_name)))
common_prefix = os.path.commonprefix(locations_to_add)
target_source_pairs = zip(
[MakeTargetImageName(common_prefix, output_directory, loc)
for loc in locations_to_add],
locations_to_add)
if test_launcher_filter_file:
test_launcher_filter_file = os.path.normpath(
os.path.join(output_directory, test_launcher_filter_file))
filter_file_on_device = MakeTargetImageName(
common_prefix, output_directory, test_launcher_filter_file)
child_args.append('--test-launcher-filter-file=/system/' +
filter_file_on_device)
target_source_pairs.append(
[filter_file_on_device, test_launcher_filter_file])
# Generate a little script that runs the test binaries and then shuts down
# QEMU.
autorun_file = tempfile.NamedTemporaryFile()
autorun_file.write('#!/bin/sh\n')
autorun_file.write('/system/' + os.path.basename(test_name))
for arg in child_args:
autorun_file.write(' "%s"' % arg);
autorun_file.write('\n')
if not device:
# If shutdown of QEMU happens too soon after the test completion, log
# statements from the end of the run will be lost, so sleep for a bit before
# shutting down. When running on device don't power off so the output and
# system can be inspected.
autorun_file.write('msleep 3000\n')
autorun_file.write('dm poweroff\n')
autorun_file.flush()
os.chmod(autorun_file.name, 0750)
DumpFile(dry_run, autorun_file.name, 'autorun')
target_source_pairs.append(('autorun', autorun_file.name))
manifest_file = tempfile.NamedTemporaryFile()
bootfs_name = runtime_deps_path + '.bootfs'
for target, source in target_source_pairs:
AddToManifest(manifest_file.file, target, source,
lambda x: MakeTargetImageName(
common_prefix, output_directory, x))
mkbootfs_path = os.path.join(SDK_ROOT, 'tools', 'mkbootfs')
manifest_file.flush()
DumpFile(dry_run, manifest_file.name, 'manifest')
RunAndCheck(dry_run,
[mkbootfs_path, '-o', bootfs_name,
'--target=boot', os.path.join(SDK_ROOT, 'bootdata.bin'),
'--target=system', manifest_file.name,
])
return bootfs_name
def SymbolizeEntry(entry):
addr2line_output = subprocess.check_output(
['addr2line', '-Cipf', '--exe=' + entry[1], entry[2]])
prefix = '#%s: ' % entry[0]
# addr2line outputs a second line for inlining information, offset
# that to align it properly after the frame index.
addr2line_filtered = addr2line_output.strip().replace(
'(inlined', ' ' * len(prefix) + '(inlined')
if '??' in addr2line_filtered:
addr2line_filtered = "%s+%s" % (os.path.basename(entry[1]), entry[2])
return '%s%s' % (prefix, addr2line_filtered)
def ParallelSymbolizeBacktrace(backtrace):
# Disable handling of SIGINT during sub-process creation, to prevent
# sub-processes from consuming Ctrl-C signals, rather than the parent
# process doing so.
saved_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
p = multiprocessing.Pool(multiprocessing.cpu_count())
# Restore the signal handler for the parent process.
signal.signal(signal.SIGINT, saved_sigint_handler)
symbolized = []
try:
result = p.map_async(SymbolizeEntry, backtrace)
symbolized = result.get(SYMBOLIZATION_TIMEOUT_SECS)
if not symbolized:
return []
except multiprocessing.TimeoutError:
return ['(timeout error occurred during symbolization)']
except KeyboardInterrupt: # SIGINT
p.terminate()
return symbolized
from runner_common import MakeTargetImageName, RunFuchsia, BuildBootfs, \
ReadRuntimeDeps
def main():
@ -223,7 +27,7 @@ def main():
parser.add_argument('--runtime-deps-path',
type=os.path.realpath,
help='Runtime data dependency file from GN.')
parser.add_argument('--test-name',
parser.add_argument('--exe-name',
type=os.path.realpath,
help='Name of the the test')
parser.add_argument('--gtest_filter',
@ -277,109 +81,27 @@ def main():
if args.child_args:
child_args.extend(args.child_args)
bootfs = BuildBootfs(args.output_directory, args.runtime_deps_path,
args.test_name, child_args,
args.test_launcher_filter_file, args.device,
args.dry_run)
runtime_deps = ReadRuntimeDeps(args.runtime_deps_path, args.output_directory)
kernel_path = os.path.join(SDK_ROOT, 'kernel', 'magenta.bin')
if args.test_launcher_filter_file:
# Bundle the filter file in the runtime deps and compose the command-line
# flag which references it.
test_launcher_filter_file = os.path.normpath(
os.path.join(args.output_directory, args.test_launcher_filter_file))
common_prefix = os.path.commonprefix(runtime_deps)
filter_device_path = MakeTargetImageName(common_prefix,
args.output_directory,
test_launcher_filter_file)
runtime_deps.append(args.test_launcher_filter_file)
child_args.append('--test-launcher-filter-file=/system/' +
filter_device_path)
if args.device:
# TODO(fuchsia): This doesn't capture stdout as there's no way to do so
# currently. See https://crbug.com/749242.
bootserver_path = os.path.join(SDK_ROOT, 'tools', 'bootserver')
bootserver_command = [bootserver_path, '-1', kernel_path, bootfs]
return RunAndCheck(args.dry_run, bootserver_command)
bootfs = BuildBootfs(args.output_directory, runtime_deps, args.exe_name,
child_args, args.device, args.dry_run)
if not bootfs:
return 2
qemu_path = os.path.join(SDK_ROOT, 'qemu', 'bin', 'qemu-system-x86_64')
qemu_command = [qemu_path,
'-m', '2048',
'-nographic',
'-net', 'none',
'-smp', '4',
'-machine', 'q35',
'-kernel', kernel_path,
'-initrd', bootfs,
# Use stdio for the guest OS only; don't attach the QEMU interactive
# monitor.
'-serial', 'stdio',
'-monitor', 'none',
# TERM=dumb tells the guest OS to not emit ANSI commands that trigger
# noisy ANSI spew from the user's terminal emulator.
'-append', 'TERM=dumb kernel.halt_on_panic=true']
if int(os.environ.get('CHROME_HEADLESS', 0)) == 0:
qemu_command += ['-enable-kvm', '-cpu', 'host,migratable=no']
else:
qemu_command += ['-cpu', 'Haswell,+smap,-check']
if args.dry_run:
print 'Run:', qemu_command
return 0
# Set up backtrace-parsing regexps.
prefix = r'^.*> '
bt_end_re = re.compile(prefix + '(bt)?#(\d+):? end')
bt_with_offset_re = re.compile(
prefix + 'bt#(\d+): pc 0x[0-9a-f]+ sp (0x[0-9a-f]+) ' +
'\((\S+),(0x[0-9a-f]+)\)$')
in_process_re = re.compile(prefix +
'#(\d+) 0x[0-9a-f]+ \S+\+(0x[0-9a-f]+)$')
# We pass a separate stdin stream to qemu. Sharing stdin across processes
# leads to flakiness due to the OS prematurely killing the stream and the
# Python script panicking and aborting.
# The precise root cause is still nebulous, but this fix works.
# See crbug.com/741194 .
qemu_popen = subprocess.Popen(
qemu_command, stdout=subprocess.PIPE, stdin=open(os.devnull))
# A buffer of backtrace entries awaiting symbolization, stored as tuples.
# Element #0: backtrace frame number (starting at 0).
# Element #1: path to executable code corresponding to the current frame.
# Element #2: memory offset within the executable.
bt_entries = []
success = False
while True:
line = qemu_popen.stdout.readline().strip()
if not line:
break
if 'SUCCESS: all tests passed.' in line:
success = True
# Check for an end-of-backtrace marker.
if bt_end_re.match(line):
if bt_entries:
print '----- start symbolized stack'
for processed in ParallelSymbolizeBacktrace(bt_entries):
print processed
print '----- end symbolized stack'
bt_entries = []
continue
# Try to parse this as a Fuchsia system backtrace.
m = bt_with_offset_re.match(line)
if m:
bt_entries.append((m.group(1), args.test_name, m.group(4)))
continue
# Try to parse the line as an in-process backtrace entry.
m = in_process_re.match(line)
if m:
bt_entries.append((m.group(1), args.test_name, m.group(2)))
continue
# Some other line, so print it. Back-traces should not be interleaved with
# other output, so while this may re-order lines we see, it should actually
# make things more readable.
print line
qemu_popen.wait()
return 0 if success else 1
return RunFuchsia(bootfs, args.exe_name, args.device, args.dry_run)
if __name__ == '__main__':