Add depfile support to android build scripts

This adds the --depfile option to a bunch of android's build scripts
(the ones used by gn). It also adds a simple function in build_utils to
calculate the paths of the imported python files.

Currently, the written depfiles just contain that list of python files
(which is much more accurate than the way this is handled in gyp).

BUG=359249

Review URL: https://codereview.chromium.org/341823003

git-svn-id: http://src.chromium.org/svn/trunk/src/build@279546 4ff67af0-8c30-449e-8e8b-ad334ec8d88c

android/PRESUBMIT.py

@@ -38,15 +38,15 @@ def CommonChecks(input_api, output_api):
   output.extend(input_api.canned_checks.RunPylint(
       input_api,
       output_api,
-      black_list=[r'pylib/symbols/.*\.py$', r'gyp/.*\.py$'],
+      black_list=[r'pylib/symbols/.*\.py$', r'gyp/.*\.py$', r'gn/.*\.py'],
       extra_paths_list=[
           J(), J('..', '..', 'third_party', 'android_testrunner'),
           J('buildbot')]))
   output.extend(input_api.canned_checks.RunPylint(
       input_api,
       output_api,
-      white_list=[r'gyp/.*\.py$'],
-      extra_paths_list=[J('gyp')]))
+      white_list=[r'gyp/.*\.py$', r'gn/.*\.py'],
+      extra_paths_list=[J('gyp'), J('gn')]))
 
   output.extend(input_api.canned_checks.RunUnitTestsInDirectory(
       input_api, output_api, J('buildbot', 'tests')))

android/gn/zip.py

@@ -13,6 +13,9 @@ import os
 import sys
 import zipfile
 
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, 'gyp'))
+from util import build_utils
+
 def DoZip(inputs, output, base_dir):
   with zipfile.ZipFile(output, 'w') as outfile:
     for f in inputs:
@@ -20,6 +23,8 @@ def DoZip(inputs, output, base_dir):
 
 def main():
   parser = optparse.OptionParser()
+  build_utils.AddDepfileOption(parser)
+
   parser.add_option('--inputs', help='List of files to archive.')
   parser.add_option('--output', help='Path to output archive.')
   parser.add_option('--base-dir',
@@ -34,6 +39,11 @@ def main():
 
   DoZip(inputs, output, base_dir)
 
+  if options.depfile:
+    build_utils.WriteDepfile(
+        options.depfile,
+        build_utils.GetPythonDependencies())
+
 
 if __name__ == '__main__':
   sys.exit(main())

android/gyp/gcc_preprocess.py

@@ -31,6 +31,8 @@ def DoGcc(options):
 
 def main():
   parser = optparse.OptionParser()
+  build_utils.AddDepfileOption(parser)
+
   parser.add_option('--include-path', help='Include path for gcc.')
   parser.add_option('--template', help='Path to template.')
   parser.add_option('--output', help='Path for generated file.')
@@ -41,6 +43,11 @@ def main():
 
   DoGcc(options)
 
+  if options.depfile:
+    build_utils.WriteDepfile(
+        options.depfile,
+        build_utils.GetPythonDependencies())
+
   if options.stamp:
     build_utils.Touch(options.stamp)
 

android/gyp/jar_toc.py

@@ -92,6 +92,8 @@ def DoJarToc(options):
 
 def main():
   parser = optparse.OptionParser()
+  build_utils.AddDepfileOption(parser)
+
   parser.add_option('--jar-path', help='Input .jar path.')
   parser.add_option('--toc-path', help='Output .jar.TOC path.')
   parser.add_option('--stamp', help='Path to touch on success.')
@@ -100,6 +102,11 @@ def main():
 
   DoJarToc(options)
 
+  if options.depfile:
+    build_utils.WriteDepfile(
+        options.depfile,
+        build_utils.GetPythonDependencies())
+
   if options.stamp:
     build_utils.Touch(options.stamp)
 

android/gyp/javac.py

@@ -111,6 +111,8 @@ def main():
   colorama.init()
 
   parser = optparse.OptionParser()
+  build_utils.AddDepfileOption(parser)
+
   parser.add_option(
       '--src-gendirs',
       help='Directories containing generated java files.')
@@ -165,6 +167,11 @@ def main():
       build_utils.DeleteDirectory(options.classes_dir)
       shutil.copytree(classes_dir, options.classes_dir)
 
+  if options.depfile:
+    build_utils.WriteDepfile(
+        options.depfile,
+        build_utils.GetPythonDependencies())
+
   if options.stamp:
     build_utils.Touch(options.stamp)
 

android/gyp/util/build_utils.py

@@ -14,8 +14,9 @@ import sys
 import tempfile
 import zipfile
 
-CHROMIUM_SRC = os.path.join(os.path.dirname(__file__),
-                            os.pardir, os.pardir, os.pardir, os.pardir)
+CHROMIUM_SRC = os.path.normpath(
+    os.path.join(os.path.dirname(__file__),
+                 os.pardir, os.pardir, os.pardir, os.pardir))
 COLORAMA_ROOT = os.path.join(CHROMIUM_SRC,
                              'third_party', 'colorama', 'src')
 
@@ -82,6 +83,7 @@ def CheckOptions(options, parser, required=None):
     if getattr(options, option_name) is None:
       parser.error('--%s is required' % option_name.replace('_', '-'))
 
+
 def WriteJson(obj, path, only_if_changed=False):
   old_dump = None
   if os.path.exists(path):
@@ -94,6 +96,7 @@ def WriteJson(obj, path, only_if_changed=False):
     with open(path, 'w') as outfile:
       outfile.write(new_dump)
 
+
 def ReadJson(path):
   with open(path, 'r') as jsonfile:
     return json.load(jsonfile)
@@ -212,3 +215,40 @@ def PrintBigWarning(message):
   print '*****     ' * 8
   PrintWarning(message)
   print '*****     ' * 8
+
+
+def GetPythonDependencies():
+  """Gets the paths of imported non-system python modules.
+
+  A path is assumed to be a "system" import if it is outside of chromium's
+  src/. The paths will be relative to the current directory.
+  """
+  module_paths = (m.__file__ for m in sys.modules.itervalues()
+                  if m is not None and hasattr(m, '__file__'))
+
+  abs_module_paths = map(os.path.abspath, module_paths)
+
+  non_system_module_paths = [
+      p for p in abs_module_paths if p.startswith(CHROMIUM_SRC)]
+  def ConvertPycToPy(s):
+    if s.endswith('.pyc'):
+      return s[:-1]
+    return s
+
+  non_system_module_paths = map(ConvertPycToPy, non_system_module_paths)
+  non_system_module_paths = map(os.path.relpath, non_system_module_paths)
+  return sorted(set(non_system_module_paths))
+
+
+def AddDepfileOption(parser):
+  parser.add_option('--depfile',
+                    help='Path to depfile. This must be specified as the '
+                    'action\'s first output.')
+
+
+def WriteDepfile(path, dependencies):
+  with open(path, 'w') as depfile:
+    depfile.write(path)
+    depfile.write(': ')
+    depfile.write(' '.join(dependencies))
+    depfile.write('\n')

config/android/internal_rules.gni

@@ -9,9 +9,11 @@ template("zip") {
   rebase_output = rebase_path(invoker.output)
   action(target_name) {
     script = "//build/android/gn/zip.py"
+    depfile = "$target_gen_dir/$target_name.d"
     source_prereqs = invoker.inputs
-    outputs = [invoker.output]
+    outputs = [depfile, invoker.output]
     args = [
+      "--depfile", rebase_path(depfile, root_build_dir),
       "--inputs=$rebase_inputs",
       "--output=$rebase_output",
     ]

config/android/rules.gni

@@ -36,13 +36,16 @@ template("generate_jni") {
   foreach_target_name = "${target_name}__jni_gen"
   action_foreach(foreach_target_name) {
     script = "//base/android/jni_generator/jni_generator.py"
+    depfile = "$target_gen_dir/$target_name.{{source_name_part}}.d"
     sources = invoker.sources
     source_prereqs = [ jni_generator_include ]
     outputs = [
+      depfile,
       "${jni_output_dir}/{{source_name_part}}_jni.h"
     ]
 
     args = [
+      "--depfile", rebase_path(depfile, root_build_dir),
       "--input_file={{source}}",
       "--optimize_generation=1",
       "--ptr_type=long",
@@ -123,16 +126,19 @@ template("generate_jar_jni") {
     jni_target_name = "${target_name}__jni_${classname}"
     jni_actions += [ ":$jni_target_name" ]
     action(jni_target_name) {
+      depfile = "$target_gen_dir/$target_name.d"
       script = "//base/android/jni_generator/jni_generator.py"
       sources = [
         jni_generator_include,
         jar_file,
       ]
       outputs = [
+        depfile,
         "${jni_output_dir}/${classname}_jni.h"
       ]
 
       args = [
+        "--depfile", rebase_path(depfile, root_build_dir),
         "--jar_file", rebase_path(jar_file, root_build_dir),
         "--input_file", class,
         "--optimize_generation=1",
@@ -205,6 +211,7 @@ template("java_cpp_template") {
     if (defined(invoker.source_prereqs)) {
       source_prereqs = invoker.source_prereqs + []
     }
+    depfile = "${target_gen_dir}/${target_name}.d"
 
     sources = invoker.sources
 
@@ -212,10 +219,12 @@ template("java_cpp_template") {
     gcc_template_output_pattern = "${gen_dir}/{{source_name_part}}.java"
 
     outputs = [
+      depfile,
       gcc_template_output_pattern
     ]
 
     args = [
+      "--depfile", rebase_path(depfile, root_build_dir),
       "--include-path", rebase_path(include_path, root_build_dir),
       "--output", rebase_path(gen_dir, root_build_dir) + "/{{source_name_part}}.java",
       "--template={{source}}",