From f472b1cdccca57c7a59824006f4a192730eca603 Mon Sep 17 00:00:00 2001 From: bratell Date: Mon, 3 Jul 2017 04:14:37 -0700 Subject: [PATCH] Scripts for unity/jumbo (default disabled) compilation. To speed up compilation times, jumbo allows files to be compiled together. This is a well known method ("unity builds") to both compile faster and create a poor man's "full program optimization". For Chromium we are only interested in the compile times. This patch includes the basic scripts that do the source file merging and changes Blink Core to use those scripts. If the gn configuration includes: use_jumbo_build = true then Blink Core will use jumbo compile. Otherwise it will compile as usual. The expected speedup from using Jumbo on Blink Core (and nothing else) is about 17% of the content_shell+blink_tests compilation CPU time. This is about half an hour for people building with an ordinary computer, but less both in percentage and minutes if using some kind of build accelerator like goma. More information in https://docs.google.com/document/d/19jGsZxh7DX8jkAKbL1nYBa5rcByUL2EeidnYsoXfsYQ/edit# BUG=713137 Review-Url: https://codereview.chromium.org/2963733003 Cr-Original-Commit-Position: refs/heads/master@{#483986} Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src Cr-Mirrored-Commit: 45a1ceb8309b6076aea7d2885b7a62f8d3c549c9 --- config/jumbo.gni | 146 ++++++++++++++++++++++++++++++++++++++ config/merge_for_jumbo.py | 62 ++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 config/jumbo.gni create mode 100755 config/merge_for_jumbo.py diff --git a/config/jumbo.gni b/config/jumbo.gni new file mode 100644 index 000000000..baf19f1b6 --- /dev/null +++ b/config/jumbo.gni @@ -0,0 +1,146 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/split_static_library.gni") # When someone uses that target_type + +declare_args() { + # If true, use a jumbo build (files compiled together) to speed up + # compilation. + use_jumbo_build = false + + # A target to exclude from jumbo builds, for optimal round trip time + # when frequently changing a single cpp file. + jumbo_build_excluded = "" + + # How many files to group at most. Smaller numbers give more + # parallellism, higher numbers give less total CPU usage. Higher + # numbers also give longer single-file recompilation times. + # + # Recommendations: + # Higher numbers than 200 does not reduce wall clock compile times + # for 4 cores or less. + # 200 uses 8% less total CPU than 100 when compiling content and 10% + # less wall clock when compiling with 4 cores. + jumbo_file_merge_limit = 200 +} + +# Use this to generate a target which merges sources if possible to +# compile much faster. +# +# Special values. +# +# target_type +# The kind of target to build. For example the string +# "static_library". +# +# always_build_jumbo +# If set and set to true, then use jumbo compile even when it is +# globally disabled. Otherwise it has no effect. +# +# never_build_jumbo +# If set and set to true, then do not jumbo compile even if it is +# globally enabled. Otherwise it has no effect. +# +# jumbo_excluded_sources +# If set to a list of files, those files will not be merged with +# the rest. This can be necessary if merging the files causes +# compilation issues and fixing the issues is impractical. +template("jumbo_target") { + use_jumbo_build_for_target = use_jumbo_build + if (defined(invoker.always_build_jumbo) && invoker.always_build_jumbo) { + use_jumbo_build_for_target = true + } + if (defined(invoker.never_build_jumbo) && invoker.never_build_jumbo) { + use_jumbo_build_for_target = false + } + if (target_name == jumbo_build_excluded) { + use_jumbo_build_for_target = false + } + + excluded_sources = [] + if (defined(invoker.jumbo_excluded_sources)) { + excluded_sources += invoker.jumbo_excluded_sources + } + + invoker_sources = invoker.sources + gen_target_dir = get_path_info(invoker_sources[0], "gen_dir") + assert(excluded_sources != [] || true) # Prevent "unused variable". + assert(gen_target_dir != "") # Prevent "unused variable". + + if (use_jumbo_build_for_target) { + jumbo_files = [] + + # Split the sources list into chunks that are not excessively large + files_per_chunk = jumbo_file_merge_limit + current_file_index = 0 + next_chunk_start = 0 + next_chunk_number = 1 + foreach(source_file, invoker.sources) { + if (current_file_index == next_chunk_start) { + jumbo_files += [ "$gen_target_dir/" + target_name + "_jumbo_" + + next_chunk_number + ".cc" ] + next_chunk_number += 1 + next_chunk_start += files_per_chunk + } + current_file_index += 1 + } + + merge_action_name = target_name + "__jumbo_merge" + + # Create an action that calls a script that merges all the source files. + action(merge_action_name) { + script = "//build/config/merge_for_jumbo.py" + response_file_contents = + rebase_path(invoker.sources - excluded_sources, gen_target_dir) + outputs = jumbo_files + args = [ "--outputs" ] + rebase_path(outputs, root_build_dir) + + [ "--file-list={{response_file_name}}" ] + } + } + + target_type = invoker.target_type + if (use_jumbo_build_for_target && target_type == "split_static_library") { + # Meaningless and also impossible if split_count > len(jumbo_files) + target_type = "static_library" + + # Prevent "unused variable" warning. + assert(!defined(invoker.split_count) || invoker.split_count > 0) + } + + # Perform the actual operation, either on the original sources or + # the sources post-jumbo merging. + target(target_type, target_name) { + deps = [] + if (defined(invoker.deps)) { + deps += invoker.deps + } + + # Take everything else not handled above from the invoker. + variables_to_not_forward = [ "deps" ] + if (use_jumbo_build_for_target) { + deps += [ ":" + merge_action_name ] + variables_to_not_forward += [ "sources" ] + assert(jumbo_files != []) + sources = jumbo_files + excluded_sources + + # Need to keep the headers in sources so that dependency checks + # work, and we need to keep Objective-C code since they + # cannot be merged into a cc file (FIXME). + foreach(source_file, invoker.sources) { + source_ext = get_path_info(source_file, "extension") + if (source_ext == "h" || source_ext == "mm") { + sources += [ source_file ] + } + } + } + forward_variables_from(invoker, "*", variables_to_not_forward) + } +} + +set_defaults("jumbo_target") { + # This sets the default list of configs when the content_source_set target + # is defined. The default_compiler_configs comes from BUILDCONFIG.gn and + # is the list normally applied to static libraries and source sets. + configs = default_compiler_configs +} diff --git a/config/merge_for_jumbo.py b/config/merge_for_jumbo.py new file mode 100755 index 000000000..f6eb87dce --- /dev/null +++ b/config/merge_for_jumbo.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This script creates a "jumbo" file which merges all incoming files +for compiling. + +""" + +from __future__ import print_function + +import argparse + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--outputs", nargs="+", required=True, + help='List of output files to split input into') + parser.add_argument("--file-list", required=True) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + + output_count = len(args.outputs) + + lines = [] + # If written with gn |write_file| each file is on its own line. + with open(args.file_list) as file_list_file: + lines = [line.strip() for line in file_list_file if line.strip()] + # If written with gn |response_file_contents| the files are space separated. + inputs = [] + for line in lines: + inputs.extend(line.split()) + input_count = len(inputs) + + written_inputs = 0 + for output_index, output_file in enumerate(args.outputs): + # TODO: Check if the file is right already and then do not update it. + with open(output_file, "w") as out: + out.write("/* This is a Jumbo file. Don't edit. */\n\n") + out.write("/* Generated with merge_for_jumbo.py. */\n\n") + input_limit = (output_index + 1) * input_count / output_count + while written_inputs < input_limit: + filename = inputs[written_inputs] + written_inputs += 1 + # The source list includes headers which should not be + # compiled, and Objective C files which will be special cased + # later since they will not compile correctly if included in a + # C++ file. We will just skip them here for now. + if filename.endswith((".h", ".mm")): + continue + + out.write("#include \"%s\"\n" % filename) + + if args.verbose: + print("Generated %s (%d files) based on %s" % (str(args.outputs), + written_inputs, + args.file_list)) + +if __name__ == "__main__": + main()