Scripts for unity/jumbo (default disabled) compilation.

To speed up compilation times, jumbo allows files to be compiled
together. This is a well known method ("unity builds") to both
compile faster and create a poor man's "full program optimization".
For Chromium we are only interested in the compile times.

This patch includes the basic scripts that do the source file merging
and changes Blink Core to use those scripts. If the gn configuration
includes: use_jumbo_build = true then Blink Core will use jumbo
compile. Otherwise it will compile as usual.

The expected speedup from using Jumbo on Blink Core (and nothing else)
is about 17% of the content_shell+blink_tests compilation CPU
time. This is about half an hour for people building with an ordinary
computer, but less both in percentage and minutes if using some kind
of build accelerator like goma.

More information in
https://docs.google.com/document/d/19jGsZxh7DX8jkAKbL1nYBa5rcByUL2EeidnYsoXfsYQ/edit#

BUG=713137

Review-Url: https://codereview.chromium.org/2963733003
Cr-Original-Commit-Position: refs/heads/master@{#483986}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 45a1ceb8309b6076aea7d2885b7a62f8d3c549c9

config/jumbo.gni

@@ -0,0 +1,146 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/split_static_library.gni")  # When someone uses that target_type
+
+declare_args() {
+  # If true, use a jumbo build (files compiled together) to speed up
+  # compilation.
+  use_jumbo_build = false
+
+  # A target to exclude from jumbo builds, for optimal round trip time
+  # when frequently changing a single cpp file.
+  jumbo_build_excluded = ""
+
+  # How many files to group at most. Smaller numbers give more
+  # parallellism, higher numbers give less total CPU usage. Higher
+  # numbers also give longer single-file recompilation times.
+  #
+  # Recommendations:
+  # Higher numbers than 200 does not reduce wall clock compile times
+  # for 4 cores or less.
+  # 200 uses 8% less total CPU than 100 when compiling content and 10%
+  # less wall clock when compiling with 4 cores.
+  jumbo_file_merge_limit = 200
+}
+
+# Use this to generate a target which merges sources if possible to
+# compile much faster.
+#
+# Special values.
+#
+#   target_type
+#      The kind of target to build. For example the string
+#      "static_library".
+#
+#   always_build_jumbo
+#      If set and set to true, then use jumbo compile even when it is
+#      globally disabled. Otherwise it has no effect.
+#
+#   never_build_jumbo
+#      If set and set to true, then do not jumbo compile even if it is
+#      globally enabled. Otherwise it has no effect.
+#
+#   jumbo_excluded_sources
+#      If set to a list of files, those files will not be merged with
+#      the rest. This can be necessary if merging the files causes
+#      compilation issues and fixing the issues is impractical.
+template("jumbo_target") {
+  use_jumbo_build_for_target = use_jumbo_build
+  if (defined(invoker.always_build_jumbo) && invoker.always_build_jumbo) {
+    use_jumbo_build_for_target = true
+  }
+  if (defined(invoker.never_build_jumbo) && invoker.never_build_jumbo) {
+    use_jumbo_build_for_target = false
+  }
+  if (target_name == jumbo_build_excluded) {
+    use_jumbo_build_for_target = false
+  }
+
+  excluded_sources = []
+  if (defined(invoker.jumbo_excluded_sources)) {
+    excluded_sources += invoker.jumbo_excluded_sources
+  }
+
+  invoker_sources = invoker.sources
+  gen_target_dir = get_path_info(invoker_sources[0], "gen_dir")
+  assert(excluded_sources != [] || true)  # Prevent "unused variable".
+  assert(gen_target_dir != "")  # Prevent "unused variable".
+
+  if (use_jumbo_build_for_target) {
+    jumbo_files = []
+
+    # Split the sources list into chunks that are not excessively large
+    files_per_chunk = jumbo_file_merge_limit
+    current_file_index = 0
+    next_chunk_start = 0
+    next_chunk_number = 1
+    foreach(source_file, invoker.sources) {
+      if (current_file_index == next_chunk_start) {
+        jumbo_files += [ "$gen_target_dir/" + target_name + "_jumbo_" +
+                         next_chunk_number + ".cc" ]
+        next_chunk_number += 1
+        next_chunk_start += files_per_chunk
+      }
+      current_file_index += 1
+    }
+
+    merge_action_name = target_name + "__jumbo_merge"
+
+    # Create an action that calls a script that merges all the source files.
+    action(merge_action_name) {
+      script = "//build/config/merge_for_jumbo.py"
+      response_file_contents =
+          rebase_path(invoker.sources - excluded_sources, gen_target_dir)
+      outputs = jumbo_files
+      args = [ "--outputs" ] + rebase_path(outputs, root_build_dir) +
+             [ "--file-list={{response_file_name}}" ]
+    }
+  }
+
+  target_type = invoker.target_type
+  if (use_jumbo_build_for_target && target_type == "split_static_library") {
+    # Meaningless and also impossible if split_count > len(jumbo_files)
+    target_type = "static_library"
+
+    # Prevent "unused variable" warning.
+    assert(!defined(invoker.split_count) || invoker.split_count > 0)
+  }
+
+  # Perform the actual operation, either on the original sources or
+  # the sources post-jumbo merging.
+  target(target_type, target_name) {
+    deps = []
+    if (defined(invoker.deps)) {
+      deps += invoker.deps
+    }
+
+    # Take everything else not handled above from the invoker.
+    variables_to_not_forward = [ "deps" ]
+    if (use_jumbo_build_for_target) {
+      deps += [ ":" + merge_action_name ]
+      variables_to_not_forward += [ "sources" ]
+      assert(jumbo_files != [])
+      sources = jumbo_files + excluded_sources
+
+      # Need to keep the headers in sources so that dependency checks
+      # work, and we need to keep Objective-C code since they
+      # cannot be merged into a cc file (FIXME).
+      foreach(source_file, invoker.sources) {
+        source_ext = get_path_info(source_file, "extension")
+        if (source_ext == "h" || source_ext == "mm") {
+          sources += [ source_file ]
+        }
+      }
+    }
+    forward_variables_from(invoker, "*", variables_to_not_forward)
+  }
+}
+
+set_defaults("jumbo_target") {
+  # This sets the default list of configs when the content_source_set target
+  # is defined. The default_compiler_configs comes from BUILDCONFIG.gn and
+  # is the list normally applied to static libraries and source sets.
+  configs = default_compiler_configs
+}

config/merge_for_jumbo.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This script creates a "jumbo" file which merges all incoming files
+for compiling.
+
+"""
+
+from __future__ import print_function
+
+import argparse
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--outputs", nargs="+", required=True,
+                      help='List of output files to split input into')
+  parser.add_argument("--file-list", required=True)
+  parser.add_argument("--verbose", action="store_true")
+  args = parser.parse_args()
+
+  output_count = len(args.outputs)
+
+  lines = []
+  # If written with gn |write_file| each file is on its own line.
+  with open(args.file_list) as file_list_file:
+    lines = [line.strip() for line in file_list_file if line.strip()]
+  # If written with gn |response_file_contents| the files are space separated.
+  inputs = []
+  for line in lines:
+    inputs.extend(line.split())
+  input_count = len(inputs)
+
+  written_inputs = 0
+  for output_index, output_file in enumerate(args.outputs):
+    # TODO: Check if the file is right already and then do not update it.
+    with open(output_file, "w") as out:
+      out.write("/* This is a Jumbo file. Don't edit. */\n\n")
+      out.write("/* Generated with merge_for_jumbo.py. */\n\n")
+      input_limit = (output_index + 1) * input_count / output_count
+      while written_inputs < input_limit:
+        filename = inputs[written_inputs]
+        written_inputs += 1
+        # The source list includes headers which should not be
+        # compiled, and Objective C files which will be special cased
+        # later since they will not compile correctly if included in a
+        # C++ file. We will just skip them here for now.
+        if filename.endswith((".h", ".mm")):
+          continue
+
+        out.write("#include \"%s\"\n" % filename)
+
+  if args.verbose:
+      print("Generated %s (%d files) based on %s" % (str(args.outputs),
+                                                     written_inputs,
+                                                     args.file_list))
+
+if __name__ == "__main__":
+    main()