diff --git a/config/fuchsia/rules.gni b/config/fuchsia/rules.gni index b3e7f3b63..2b6f99d85 100644 --- a/config/fuchsia/rules.gni +++ b/config/fuchsia/rules.gni @@ -4,12 +4,7 @@ assert(is_fuchsia) -# This template is used to generate a runner script for test binaries into the -# build dir for Fuchsia. It's generally used from the "test" template. -template("test_runner_script") { - testonly = true - _test_name = invoker.test_name - +template("generate_runner_script") { # This runtime_deps file is used at runtime and thus cannot go in # target_gen_dir. _target_dir_name = get_label_info(":$target_name", "dir") @@ -23,6 +18,7 @@ template("test_runner_script") { "data_deps", "deps", "public_deps", + "testonly", ]) write_runtime_deps = _runtime_deps_file } @@ -32,6 +28,9 @@ template("test_runner_script") { [ "data_deps", "deps", + "runner_script", + "target", + "testonly", ]) if (!defined(deps)) { deps = [] @@ -40,39 +39,67 @@ template("test_runner_script") { data_deps = [] } - script = "//build/fuchsia/create_test_runner_script.py" + script = "//build/fuchsia/create_runner_script.py" depfile = "$target_gen_dir/$target_name.d" data = [] - test_runner_args = [] + runner_args = [] - generated_script = "$root_build_dir/bin/run_${_test_name}" + generated_script = "$root_build_dir/bin/run_${target}" outputs = [ generated_script, ] data += [ generated_script ] - test_runner_args += [ + runner_args += [ + "--runner-script", + runner_script, "--output-directory", rebase_path(root_build_dir, root_build_dir), ] deps += [ ":$_runtime_deps_target" ] data += [ _runtime_deps_file ] - test_runner_args += [ + runner_args += [ "--runtime-deps-path", rebase_path(_runtime_deps_file, root_build_dir), ] + if (defined(args)) { + args = [] + } args = [ "--depfile", rebase_path(depfile, root_build_dir), "--script-output-path", rebase_path(generated_script, root_build_dir), - "--test-name", - _test_name, + "--exe-name", + target, ] - args += test_runner_args + args += runner_args + } +} + +# This template is used to generate a runner script for test binaries into the +# build dir for Fuchsia. It's generally used from the "test" template. +template("test_runner_script") { + generate_runner_script(target_name) { + testonly = true + runner_script = "test_runner.py" + target = invoker.test_name + forward_variables_from(invoker, "*") + } +} + +# This template is used to generate a runner script for arbitrary executables +# into the build dir for Fuchsia. The template should be instantiated alongside +# an "executable" target, and referenced from the executable via its "deps" +# attribute. +template("fuchsia_executable_runner") { + generate_runner_script(target_name) { + runner_script = "exe_runner.py" + target = invoker.exe_name + forward_variables_from(invoker, "*") } } diff --git a/fuchsia/create_test_runner_script.py b/fuchsia/create_runner_script.py similarity index 61% rename from fuchsia/create_test_runner_script.py rename to fuchsia/create_runner_script.py index c85e7ce53..f50064b9b 100755 --- a/fuchsia/create_test_runner_script.py +++ b/fuchsia/create_runner_script.py @@ -4,8 +4,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Creates a script to run a Fushsia test (typically on QEMU) by delegating to -build/fuchsia/test_runner.py. +"""Creates a script to run a Fushsia executable by delegating to +build/fuchsia/(exe|test)_runner.py. """ import argparse @@ -17,7 +17,7 @@ import sys SCRIPT_TEMPLATE = """\ #!/usr/bin/env python # -# This file was generated by build/fuchsia/create_test_runner_script.py +# This file was generated by build/fuchsia/create_runner_script.py import os import sys @@ -30,14 +30,14 @@ def main(): \"\"\" return os.path.abspath(os.path.join(script_directory, path)) - test_runner_path = ResolvePath('{test_runner_path}') - test_runner_args = {test_runner_args} - test_runner_path_args = {test_runner_path_args} - for arg, path in test_runner_path_args: - test_runner_args.extend([arg, ResolvePath(path)]) + runner_path = ResolvePath('{runner_path}') + runner_args = {runner_args} + runner_path_args = {runner_path_args} + for arg, path in runner_path_args: + runner_args.extend([arg, ResolvePath(path)]) - os.execv(test_runner_path, - [test_runner_path] + test_runner_args + sys.argv[1:]) + os.execv(runner_path, + [runner_path] + runner_args + sys.argv[1:]) if __name__ == '__main__': sys.exit(main()) @@ -65,6 +65,8 @@ def WriteDepfile(depfile_path, first_gn_output, inputs=None): def main(args): parser = argparse.ArgumentParser() + parser.add_argument('--runner-script', + help='Name of the runner script to use.') parser.add_argument('--script-output-path', help='Output path for executable script.') parser.add_argument('--depfile', @@ -75,33 +77,30 @@ def main(args): group = parser.add_argument_group('Test runner path arguments.') group.add_argument('--output-directory') group.add_argument('--runtime-deps-path') - group.add_argument('--test-name') - args, test_runner_args = parser.parse_known_args(args) + group.add_argument('--exe-name') + args, runner_args = parser.parse_known_args(args) def RelativizePathToScript(path): """Returns the path relative to the output script directory.""" return os.path.relpath(path, os.path.dirname(args.script_output_path)) - test_runner_path = args.test_runner_path or os.path.join( - os.path.dirname(__file__), 'test_runner.py') - test_runner_path = RelativizePathToScript(test_runner_path) + runner_path = args.test_runner_path or os.path.join( + os.path.dirname(__file__), args.runner_script) + runner_path = RelativizePathToScript(runner_path) - test_runner_path_args = [] - if args.output_directory: - test_runner_path_args.append( - ('--output-directory', RelativizePathToScript(args.output_directory))) - if args.runtime_deps_path: - test_runner_path_args.append( - ('--runtime-deps-path', RelativizePathToScript(args.runtime_deps_path))) - if args.test_name: - test_runner_path_args.append( - ('--test-name', RelativizePathToScript(args.test_name))) + runner_path_args = [] + runner_path_args.append( + ('--output-directory', RelativizePathToScript(args.output_directory))) + runner_path_args.append( + ('--runtime-deps-path', RelativizePathToScript(args.runtime_deps_path))) + runner_path_args.append( + ('--exe-name', RelativizePathToScript(args.exe_name))) with open(args.script_output_path, 'w') as script: script.write(SCRIPT_TEMPLATE.format( - test_runner_path=str(test_runner_path), - test_runner_args=str(test_runner_args), - test_runner_path_args=str(test_runner_path_args))) + runner_path=str(runner_path), + runner_args=str(runner_args), + runner_path_args=str(runner_path_args))) os.chmod(args.script_output_path, 0750) diff --git a/fuchsia/exe_runner.py b/fuchsia/exe_runner.py new file mode 100755 index 000000000..1eb0fb6e0 --- /dev/null +++ b/fuchsia/exe_runner.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Packages a user.bootfs for a Fuchsia boot image, pulling in the runtime +dependencies of a binary, and then uses either QEMU from the Fuchsia SDK +to run, or starts the bootserver to allow running on a hardware device.""" + +import argparse +import os +import sys + +from runner_common import RunFuchsia, BuildBootfs, ReadRuntimeDeps + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--dry-run', '-n', action='store_true', default=False, + help='Just print commands, don\'t execute them.') + parser.add_argument('--output-directory', + type=os.path.realpath, + help=('Path to the directory in which build files are' + ' located (must include build type).')) + parser.add_argument('--runtime-deps-path', + type=os.path.realpath, + help='Runtime data dependency file from GN.') + parser.add_argument('--exe-name', + type=os.path.realpath, + help='Name of the the binary executable.') + parser.add_argument('-d', '--device', action='store_true', default=False, + help='Run on hardware device instead of QEMU.') + args, child_args = parser.parse_known_args() + + bootfs = BuildBootfs( + args.output_directory, + ReadRuntimeDeps(args.runtime_deps_path, args.output_directory), + args.exe_name, child_args, args.device, args.dry_run) + if not bootfs: + return 2 + + return RunFuchsia(bootfs, args.exe_name, args.device, args.dry_run) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/fuchsia/runner_common.py b/fuchsia/runner_common.py new file mode 100755 index 000000000..e170f03be --- /dev/null +++ b/fuchsia/runner_common.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python +# +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Packages a user.bootfs for a Fuchsia boot image, pulling in the runtime +dependencies of a binary, and then uses either QEMU from the Fuchsia SDK +to run, or starts the bootserver to allow running on a hardware device.""" + +import argparse +import multiprocessing +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile + + +DIR_SOURCE_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) +SDK_ROOT = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'fuchsia-sdk') +SYMBOLIZATION_TIMEOUT_SECS = 10 + + +def _RunAndCheck(dry_run, args): + if dry_run: + print 'Run:', ' '.join(args) + return 0 + else: + try: + subprocess.check_call(args) + return 0 + except subprocess.CalledProcessError as e: + return e.returncode + + +def _DumpFile(dry_run, name, description): + """Prints out the contents of |name| if |dry_run|.""" + if not dry_run: + return + print + print 'Contents of %s (for %s)' % (name, description) + print '-' * 80 + with open(name) as f: + sys.stdout.write(f.read()) + print '-' * 80 + + +def MakeTargetImageName(common_prefix, output_directory, location): + """Generates the relative path name to be used in the file system image. + common_prefix: a prefix of both output_directory and location that + be removed. + output_directory: an optional prefix on location that will also be removed. + location: the file path to relativize. + + .so files will be stored into the lib subdirectory to be able to be found by + default by the loader. + + Examples: + + >>> MakeTargetImageName(common_prefix='/work/cr/src/', + ... output_directory='/work/cr/src/out/fuch', + ... location='/work/cr/src/base/test/data/xyz.json') + 'base/test/data/xyz.json' + + >>> MakeTargetImageName(common_prefix='/work/cr/src/', + ... output_directory='/work/cr/src/out/fuch', + ... location='/work/cr/src/out/fuch/icudtl.dat') + 'icudtl.dat' + + >>> MakeTargetImageName(common_prefix='/work/cr/src/', + ... output_directory='/work/cr/src/out/fuch', + ... location='/work/cr/src/out/fuch/libbase.so') + 'lib/libbase.so' + """ + assert output_directory.startswith(common_prefix) + output_dir_no_common_prefix = output_directory[len(common_prefix):] + assert location.startswith(common_prefix) + loc = location[len(common_prefix):] + if loc.startswith(output_dir_no_common_prefix): + loc = loc[len(output_dir_no_common_prefix)+1:] + # TODO(fuchsia): The requirements for finding/loading .so are in flux, so this + # ought to be reconsidered at some point. See https://crbug.com/732897. + if location.endswith('.so'): + loc = 'lib/' + loc + return loc + + +def _AddToManifest(manifest_file, target_name, source, mapper): + """Appends |source| to the given |manifest_file| (a file object) in a format + suitable for consumption by mkbootfs. + + If |source| is a file it's directly added. If |source| is a directory, its + contents are recursively added. + + |source| must exist on disk at the time this function is called. + """ + if os.path.isdir(source): + files = [os.path.join(dp, f) for dp, dn, fn in os.walk(source) for f in fn] + for f in files: + # We pass None as the mapper because this should never recurse a 2nd time. + _AddToManifest(manifest_file, mapper(f), f, None) + elif os.path.exists(source): + manifest_file.write('%s=%s\n' % (target_name, source)) + else: + raise Exception('%s does not exist' % source) + + +def ReadRuntimeDeps(deps_path, output_dir): + return [os.path.abspath(os.path.join(output_dir, x.strip())) + for x in open(deps_path)] + + +def _StripBinary(dry_run, bin_path): + strip_path = bin_path + '_stripped'; + shutil.copyfile(bin_path, strip_path) + _RunAndCheck(dry_run, ['/usr/bin/strip', strip_path]) + return strip_path + + +def BuildBootfs(output_directory, runtime_deps, bin_name, child_args, + device, dry_run): + locations_to_add = [os.path.abspath(os.path.join(output_directory, x.strip())) + for x in runtime_deps] + + common_prefix = '/' + if len(locations_to_add) > 1: + common_prefix = os.path.commonprefix(locations_to_add) + target_source_pairs = zip( + [MakeTargetImageName(common_prefix, output_directory, loc) + for loc in locations_to_add], + locations_to_add) + + # Stage the stripped binary in the boot image, keeping the original binary's + # name for symbolization purposes. + bin_path = os.path.abspath(os.path.join(output_directory, bin_name)) + stripped_bin_path = _StripBinary(dry_run, bin_path) + target_source_pairs.append( + [MakeTargetImageName(common_prefix, output_directory, bin_path), + stripped_bin_path]) + + # Generate a script that runs the binaries and shuts down QEMU (if used). + autorun_file = tempfile.NamedTemporaryFile() + autorun_file.write('#!/bin/sh\n') + autorun_file.write('echo Executing ' + os.path.basename(bin_name) + ' ' + + ' '.join(child_args) + '\n') + autorun_file.write('/system/' + os.path.basename(bin_name)) + for arg in child_args: + autorun_file.write(' "%s"' % arg); + autorun_file.write('\n') + autorun_file.write('echo Process terminated.\n') + + if not device: + # If shutdown of QEMU happens too soon after the program finishes, log + # statements from the end of the run will be lost, so sleep for a bit before + # shutting down. When running on device don't power off so the output and + # system can be inspected. + autorun_file.write('msleep 3000\n') + autorun_file.write('dm poweroff\n') + + autorun_file.flush() + os.chmod(autorun_file.name, 0750) + _DumpFile(dry_run, autorun_file.name, 'autorun') + target_source_pairs.append(('autorun', autorun_file.name)) + + manifest_file = tempfile.NamedTemporaryFile() + bootfs_name = bin_name + '.bootfs' + + for target, source in target_source_pairs: + _AddToManifest(manifest_file.file, target, source, + lambda x: MakeTargetImageName( + common_prefix, output_directory, x)) + + mkbootfs_path = os.path.join(SDK_ROOT, 'tools', 'mkbootfs') + + manifest_file.flush() + _DumpFile(dry_run, manifest_file.name, 'manifest') + if _RunAndCheck( + dry_run, + [mkbootfs_path, '-o', bootfs_name, + '--target=boot', os.path.join(SDK_ROOT, 'bootdata.bin'), + '--target=system', manifest_file.name]) != 0: + return None + + return bootfs_name + + +def _SymbolizeEntry(entry): + addr2line_output = subprocess.check_output( + ['addr2line', '-Cipf', '--exe=' + entry[1], entry[2]]) + prefix = '#%s: ' % entry[0] + # addr2line outputs a second line for inlining information, offset + # that to align it properly after the frame index. + addr2line_filtered = addr2line_output.strip().replace( + '(inlined', ' ' * len(prefix) + '(inlined') + if '??' in addr2line_filtered: + addr2line_filtered = "%s+%s" % (os.path.basename(entry[1]), entry[2]) + return '%s%s' % (prefix, addr2line_filtered) + + +def _ParallelSymbolizeBacktrace(backtrace): + # Disable handling of SIGINT during sub-process creation, to prevent + # sub-processes from consuming Ctrl-C signals, rather than the parent + # process doing so. + saved_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) + p = multiprocessing.Pool(multiprocessing.cpu_count()) + + # Restore the signal handler for the parent process. + signal.signal(signal.SIGINT, saved_sigint_handler) + + symbolized = [] + try: + result = p.map_async(_SymbolizeEntry, backtrace) + symbolized = result.get(SYMBOLIZATION_TIMEOUT_SECS) + if not symbolized: + return [] + except multiprocessing.TimeoutError: + return ['(timeout error occurred during symbolization)'] + except KeyboardInterrupt: # SIGINT + p.terminate() + + return symbolized + + +def RunFuchsia(bootfs, exe_name, use_device, dry_run): + kernel_path = os.path.join(SDK_ROOT, 'kernel', 'magenta.bin') + + if use_device: + # TODO(fuchsia): This doesn't capture stdout as there's no way to do so + # currently. See https://crbug.com/749242. + bootserver_path = os.path.join(SDK_ROOT, 'tools', 'bootserver') + bootserver_command = [bootserver_path, '-1', kernel_path, bootfs] + return _RunAndCheck(dry_run, bootserver_command) + + qemu_path = os.path.join(SDK_ROOT, 'qemu', 'bin', 'qemu-system-x86_64') + qemu_command = [qemu_path, + '-m', '2048', + '-nographic', + '-net', 'none', + '-smp', '4', + '-machine', 'q35', + '-kernel', kernel_path, + '-initrd', bootfs, + + # Use stdio for the guest OS only; don't attach the QEMU interactive + # monitor. + '-serial', 'stdio', + '-monitor', 'none', + + # TERM=dumb tells the guest OS to not emit ANSI commands that trigger + # noisy ANSI spew from the user's terminal emulator. + '-append', 'TERM=dumb kernel.halt_on_panic=true'] + + if int(os.environ.get('CHROME_HEADLESS', 0)) == 0: + qemu_command += ['-enable-kvm', '-cpu', 'host,migratable=no'] + else: + qemu_command += ['-cpu', 'Haswell,+smap,-check'] + + if dry_run: + print 'Run:', ' '.join(qemu_command) + return 0 + + # Set up backtrace-parsing regexps. + prefix = r'^.*> ' + bt_end_re = re.compile(prefix + '(bt)?#(\d+):? end') + bt_with_offset_re = re.compile( + prefix + 'bt#(\d+): pc 0x[0-9a-f]+ sp (0x[0-9a-f]+) ' + + '\((\S+),(0x[0-9a-f]+)\)$') + in_process_re = re.compile(prefix + + '#(\d+) 0x[0-9a-f]+ \S+\+(0x[0-9a-f]+)$') + + # We pass a separate stdin stream to qemu. Sharing stdin across processes + # leads to flakiness due to the OS prematurely killing the stream and the + # Python script panicking and aborting. + # The precise root cause is still nebulous, but this fix works. + # See crbug.com/741194 . + qemu_popen = subprocess.Popen( + qemu_command, stdout=subprocess.PIPE, stdin=open(os.devnull)) + + # A buffer of backtrace entries awaiting symbolization, stored as tuples. + # Element #0: backtrace frame number (starting at 0). + # Element #1: path to executable code corresponding to the current frame. + # Element #2: memory offset within the executable. + bt_entries = [] + + success = False + while True: + line = qemu_popen.stdout.readline().strip() + if not line: + break + if 'SUCCESS: all tests passed.' in line: + success = True + + # Check for an end-of-backtrace marker. + if bt_end_re.match(line): + if bt_entries: + print '----- start symbolized stack' + for processed in _ParallelSymbolizeBacktrace(bt_entries): + print processed + print '----- end symbolized stack' + bt_entries = [] + continue + + # Try to parse this as a Fuchsia system backtrace. + m = bt_with_offset_re.match(line) + if m: + bt_entries.append((m.group(1), exe_name, m.group(4))) + continue + + # Try to parse the line as an in-process backtrace entry. + m = in_process_re.match(line) + if m: + bt_entries.append((m.group(1), exe_name, m.group(2))) + continue + + # Some other line, so print it. Back-traces should not be interleaved with + # other output, so while this may re-order lines we see, it should actually + # make things more readable. + print line + + qemu_popen.wait() + + return 0 if success else 1 + diff --git a/fuchsia/test_runner.py b/fuchsia/test_runner.py index 0ba1b4cfa..a9f72d5ac 100755 --- a/fuchsia/test_runner.py +++ b/fuchsia/test_runner.py @@ -9,207 +9,11 @@ dependencies of a test binary, and then uses either QEMU from the Fuchsia SDK to run, or starts the bootserver to allow running on a hardware device.""" import argparse -import multiprocessing import os -import re -import signal -import subprocess import sys -import tempfile - -DIR_SOURCE_ROOT = os.path.abspath( - os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -SDK_ROOT = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'fuchsia-sdk') -SYMBOLIZATION_TIMEOUT_SECS = 10 - - -def RunAndCheck(dry_run, args): - if dry_run: - print 'Run:', args - else: - try: - subprocess.check_call(args) - return 0 - except subprocess.CalledProcessError as e: - return e.returncode - - -def DumpFile(dry_run, name, description): - """Prints out the contents of |name| if |dry_run|.""" - if not dry_run: - return - print - print 'Contents of %s (for %s)' % (name, description) - print '-' * 80 - with open(name) as f: - sys.stdout.write(f.read()) - print '-' * 80 - - -def MakeTargetImageName(common_prefix, output_directory, location): - """Generates the relative path name to be used in the file system image. - common_prefix: a prefix of both output_directory and location that - be removed. - output_directory: an optional prefix on location that will also be removed. - location: the file path to relativize. - - .so files will be stored into the lib subdirectory to be able to be found by - default by the loader. - - Examples: - - >>> MakeTargetImageName(common_prefix='/work/cr/src/', - ... output_directory='/work/cr/src/out/fuch', - ... location='/work/cr/src/base/test/data/xyz.json') - 'base/test/data/xyz.json' - - >>> MakeTargetImageName(common_prefix='/work/cr/src/', - ... output_directory='/work/cr/src/out/fuch', - ... location='/work/cr/src/out/fuch/icudtl.dat') - 'icudtl.dat' - - >>> MakeTargetImageName(common_prefix='/work/cr/src/', - ... output_directory='/work/cr/src/out/fuch', - ... location='/work/cr/src/out/fuch/libbase.so') - 'lib/libbase.so' - """ - assert output_directory.startswith(common_prefix) - output_dir_no_common_prefix = output_directory[len(common_prefix):] - assert location.startswith(common_prefix) - loc = location[len(common_prefix):] - if loc.startswith(output_dir_no_common_prefix): - loc = loc[len(output_dir_no_common_prefix)+1:] - # TODO(fuchsia): The requirements for finding/loading .so are in flux, so this - # ought to be reconsidered at some point. See https://crbug.com/732897. - if location.endswith('.so'): - loc = 'lib/' + loc - return loc - - -def AddToManifest(manifest_file, target_name, source, mapper): - """Appends |source| to the given |manifest_file| (a file object) in a format - suitable for consumption by mkbootfs. - - If |source| is a file it's directly added. If |source| is a directory, its - contents are recursively added. - - |source| must exist on disk at the time this function is called. - """ - if os.path.isdir(source): - files = [os.path.join(dp, f) for dp, dn, fn in os.walk(source) for f in fn] - for f in files: - # We pass None as the mapper because this should never recurse a 2nd time. - AddToManifest(manifest_file, mapper(f), f, None) - elif os.path.exists(source): - manifest_file.write('%s=%s\n' % (target_name, source)) - else: - raise Exception('%s does not exist' % source) - - -def BuildBootfs(output_directory, runtime_deps_path, test_name, child_args, - test_launcher_filter_file, device, dry_run): - with open(runtime_deps_path) as f: - lines = f.readlines() - - locations_to_add = [os.path.abspath(os.path.join(output_directory, x.strip())) - for x in lines] - locations_to_add.append( - os.path.abspath(os.path.join(output_directory, test_name))) - - common_prefix = os.path.commonprefix(locations_to_add) - target_source_pairs = zip( - [MakeTargetImageName(common_prefix, output_directory, loc) - for loc in locations_to_add], - locations_to_add) - - if test_launcher_filter_file: - test_launcher_filter_file = os.path.normpath( - os.path.join(output_directory, test_launcher_filter_file)) - filter_file_on_device = MakeTargetImageName( - common_prefix, output_directory, test_launcher_filter_file) - child_args.append('--test-launcher-filter-file=/system/' + - filter_file_on_device) - target_source_pairs.append( - [filter_file_on_device, test_launcher_filter_file]) - - # Generate a little script that runs the test binaries and then shuts down - # QEMU. - autorun_file = tempfile.NamedTemporaryFile() - autorun_file.write('#!/bin/sh\n') - autorun_file.write('/system/' + os.path.basename(test_name)) - - for arg in child_args: - autorun_file.write(' "%s"' % arg); - - autorun_file.write('\n') - if not device: - # If shutdown of QEMU happens too soon after the test completion, log - # statements from the end of the run will be lost, so sleep for a bit before - # shutting down. When running on device don't power off so the output and - # system can be inspected. - autorun_file.write('msleep 3000\n') - autorun_file.write('dm poweroff\n') - autorun_file.flush() - os.chmod(autorun_file.name, 0750) - DumpFile(dry_run, autorun_file.name, 'autorun') - target_source_pairs.append(('autorun', autorun_file.name)) - - manifest_file = tempfile.NamedTemporaryFile() - bootfs_name = runtime_deps_path + '.bootfs' - - for target, source in target_source_pairs: - AddToManifest(manifest_file.file, target, source, - lambda x: MakeTargetImageName( - common_prefix, output_directory, x)) - - mkbootfs_path = os.path.join(SDK_ROOT, 'tools', 'mkbootfs') - - manifest_file.flush() - DumpFile(dry_run, manifest_file.name, 'manifest') - RunAndCheck(dry_run, - [mkbootfs_path, '-o', bootfs_name, - '--target=boot', os.path.join(SDK_ROOT, 'bootdata.bin'), - '--target=system', manifest_file.name, - ]) - return bootfs_name - - -def SymbolizeEntry(entry): - addr2line_output = subprocess.check_output( - ['addr2line', '-Cipf', '--exe=' + entry[1], entry[2]]) - prefix = '#%s: ' % entry[0] - # addr2line outputs a second line for inlining information, offset - # that to align it properly after the frame index. - addr2line_filtered = addr2line_output.strip().replace( - '(inlined', ' ' * len(prefix) + '(inlined') - if '??' in addr2line_filtered: - addr2line_filtered = "%s+%s" % (os.path.basename(entry[1]), entry[2]) - return '%s%s' % (prefix, addr2line_filtered) - - -def ParallelSymbolizeBacktrace(backtrace): - # Disable handling of SIGINT during sub-process creation, to prevent - # sub-processes from consuming Ctrl-C signals, rather than the parent - # process doing so. - saved_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) - p = multiprocessing.Pool(multiprocessing.cpu_count()) - - # Restore the signal handler for the parent process. - signal.signal(signal.SIGINT, saved_sigint_handler) - - symbolized = [] - try: - result = p.map_async(SymbolizeEntry, backtrace) - symbolized = result.get(SYMBOLIZATION_TIMEOUT_SECS) - if not symbolized: - return [] - except multiprocessing.TimeoutError: - return ['(timeout error occurred during symbolization)'] - except KeyboardInterrupt: # SIGINT - p.terminate() - - return symbolized +from runner_common import MakeTargetImageName, RunFuchsia, BuildBootfs, \ + ReadRuntimeDeps def main(): @@ -223,7 +27,7 @@ def main(): parser.add_argument('--runtime-deps-path', type=os.path.realpath, help='Runtime data dependency file from GN.') - parser.add_argument('--test-name', + parser.add_argument('--exe-name', type=os.path.realpath, help='Name of the the test') parser.add_argument('--gtest_filter', @@ -277,109 +81,27 @@ def main(): if args.child_args: child_args.extend(args.child_args) - bootfs = BuildBootfs(args.output_directory, args.runtime_deps_path, - args.test_name, child_args, - args.test_launcher_filter_file, args.device, - args.dry_run) + runtime_deps = ReadRuntimeDeps(args.runtime_deps_path, args.output_directory) - kernel_path = os.path.join(SDK_ROOT, 'kernel', 'magenta.bin') + if args.test_launcher_filter_file: + # Bundle the filter file in the runtime deps and compose the command-line + # flag which references it. + test_launcher_filter_file = os.path.normpath( + os.path.join(args.output_directory, args.test_launcher_filter_file)) + common_prefix = os.path.commonprefix(runtime_deps) + filter_device_path = MakeTargetImageName(common_prefix, + args.output_directory, + test_launcher_filter_file) + runtime_deps.append(args.test_launcher_filter_file) + child_args.append('--test-launcher-filter-file=/system/' + + filter_device_path) - if args.device: - # TODO(fuchsia): This doesn't capture stdout as there's no way to do so - # currently. See https://crbug.com/749242. - bootserver_path = os.path.join(SDK_ROOT, 'tools', 'bootserver') - bootserver_command = [bootserver_path, '-1', kernel_path, bootfs] - return RunAndCheck(args.dry_run, bootserver_command) + bootfs = BuildBootfs(args.output_directory, runtime_deps, args.exe_name, + child_args, args.device, args.dry_run) + if not bootfs: + return 2 - qemu_path = os.path.join(SDK_ROOT, 'qemu', 'bin', 'qemu-system-x86_64') - qemu_command = [qemu_path, - '-m', '2048', - '-nographic', - '-net', 'none', - '-smp', '4', - '-machine', 'q35', - '-kernel', kernel_path, - '-initrd', bootfs, - - # Use stdio for the guest OS only; don't attach the QEMU interactive - # monitor. - '-serial', 'stdio', - '-monitor', 'none', - - # TERM=dumb tells the guest OS to not emit ANSI commands that trigger - # noisy ANSI spew from the user's terminal emulator. - '-append', 'TERM=dumb kernel.halt_on_panic=true'] - - if int(os.environ.get('CHROME_HEADLESS', 0)) == 0: - qemu_command += ['-enable-kvm', '-cpu', 'host,migratable=no'] - else: - qemu_command += ['-cpu', 'Haswell,+smap,-check'] - - if args.dry_run: - print 'Run:', qemu_command - return 0 - - # Set up backtrace-parsing regexps. - prefix = r'^.*> ' - bt_end_re = re.compile(prefix + '(bt)?#(\d+):? end') - bt_with_offset_re = re.compile( - prefix + 'bt#(\d+): pc 0x[0-9a-f]+ sp (0x[0-9a-f]+) ' + - '\((\S+),(0x[0-9a-f]+)\)$') - in_process_re = re.compile(prefix + - '#(\d+) 0x[0-9a-f]+ \S+\+(0x[0-9a-f]+)$') - - # We pass a separate stdin stream to qemu. Sharing stdin across processes - # leads to flakiness due to the OS prematurely killing the stream and the - # Python script panicking and aborting. - # The precise root cause is still nebulous, but this fix works. - # See crbug.com/741194 . - qemu_popen = subprocess.Popen( - qemu_command, stdout=subprocess.PIPE, stdin=open(os.devnull)) - - # A buffer of backtrace entries awaiting symbolization, stored as tuples. - # Element #0: backtrace frame number (starting at 0). - # Element #1: path to executable code corresponding to the current frame. - # Element #2: memory offset within the executable. - bt_entries = [] - - success = False - while True: - line = qemu_popen.stdout.readline().strip() - if not line: - break - if 'SUCCESS: all tests passed.' in line: - success = True - - # Check for an end-of-backtrace marker. - if bt_end_re.match(line): - if bt_entries: - print '----- start symbolized stack' - for processed in ParallelSymbolizeBacktrace(bt_entries): - print processed - print '----- end symbolized stack' - bt_entries = [] - continue - - # Try to parse this as a Fuchsia system backtrace. - m = bt_with_offset_re.match(line) - if m: - bt_entries.append((m.group(1), args.test_name, m.group(4))) - continue - - # Try to parse the line as an in-process backtrace entry. - m = in_process_re.match(line) - if m: - bt_entries.append((m.group(1), args.test_name, m.group(2))) - continue - - # Some other line, so print it. Back-traces should not be interleaved with - # other output, so while this may re-order lines we see, it should actually - # make things more readable. - print line - - qemu_popen.wait() - - return 0 if success else 1 + return RunFuchsia(bootfs, args.exe_name, args.device, args.dry_run) if __name__ == '__main__':