Finish converting all scripts from depfile -> .pydeps

Removes option to add pydeps via build_utils.WriteDepfile().

TBR=agrieve  # Refactored method signature

Bug: None
Change-Id: I5fb2d4e15c2739344bcb89ebd1906efb10a7c149
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2265504
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Mohamed Heikal <mheikal@chromium.org>
Cr-Commit-Position: refs/heads/master@{#782541}
GitOrigin-RevId: 5a01ad3dbc90575c701e4e904333a486365ac3a5

android/gyp/aar.pydeps

@@ -1,6 +1,7 @@
 # Generated by running:
 #   build/print_python_deps.py --root build/android/gyp --output build/android/gyp/aar.pydeps build/android/gyp/aar.py
 ../../gn_helpers.py
+../../print_python_deps.py
 aar.py
 util/__init__.py
 util/build_utils.py

android/gyp/apkbuilder.py

@@ -359,8 +359,7 @@ def _MaybeWriteDepAndStampFiles(options, depfile_deps):
       output = options.stamp
     else:
       output = options.output_apk
-    build_utils.WriteDepfile(
-        options.depfile, output, inputs=depfile_deps, add_pydeps=False)
+    build_utils.WriteDepfile(options.depfile, output, inputs=depfile_deps)
 
 
 def main(args):

android/gyp/compile_java.pydeps

@@ -7,6 +7,7 @@
 ../../../third_party/colorama/src/colorama/win32.py
 ../../../third_party/colorama/src/colorama/winterm.py
 ../../gn_helpers.py
+../../print_python_deps.py
 compile_java.py
 util/__init__.py
 util/build_utils.py

android/gyp/compile_resources.pydeps

@@ -46,6 +46,7 @@
 ../../../third_party/protobuf/python/google/protobuf/text_format.py
 ../../../third_party/six/src/six.py
 ../../gn_helpers.py
+../../print_python_deps.py
 compile_resources.py
 proto/Configuration_pb2.py
 proto/Resources_pb2.py

android/gyp/copy_ex.py

@@ -119,8 +119,7 @@ def main(args):
     DoRenaming(options, deps)
 
   if options.depfile:
-    build_utils.WriteDepfile(
-        options.depfile, options.stamp, deps, add_pydeps=False)
+    build_utils.WriteDepfile(options.depfile, options.stamp, deps)
 
   if options.stamp:
     build_utils.Touch(options.stamp)

android/gyp/create_app_bundle_apks.pydeps

@@ -22,6 +22,7 @@
 ../../../third_party/markupsafe/_compat.py
 ../../../third_party/markupsafe/_native.py
 ../../gn_helpers.py
+../../print_python_deps.py
 ../pylib/__init__.py
 ../pylib/utils/__init__.py
 ../pylib/utils/app_bundle_utils.py

android/gyp/create_bundle_wrapper_script.pydeps

@@ -85,6 +85,7 @@
 ../../../third_party/markupsafe/_compat.py
 ../../../third_party/markupsafe/_native.py
 ../../gn_helpers.py
+../../print_python_deps.py
 ../adb_command_line.py
 ../apk_operations.py
 ../convert_dex_profile.py

android/gyp/create_size_info_files.py

@@ -179,11 +179,9 @@ def main(args):
   _MergeResInfoFiles(options.res_info_path, res_inputs)
 
   all_inputs = jar_inputs + pak_inputs + res_inputs
-  build_utils.WriteDepfile(
-      options.depfile,
-      options.jar_info_path,
-      inputs=all_inputs,
-      add_pydeps=False)
+  build_utils.WriteDepfile(options.depfile,
+                           options.jar_info_path,
+                           inputs=all_inputs)
 
 
 if __name__ == '__main__':

android/gyp/create_ui_locale_resources.py

@@ -60,7 +60,6 @@ def main():
   parser = argparse.ArgumentParser(
       description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
 
-  build_utils.AddDepfileOption(parser)
   parser.add_argument(
       '--locale-list',
       required=True,
@@ -83,9 +82,6 @@ def main():
         android_locale = resource_utils.ToAndroidLocaleName(locale)
         _AddLocaleResourceFileToZip(out_zip, android_locale, locale)
 
-  if args.depfile:
-    build_utils.WriteDepfile(args.depfile, args.output_zip)
-
 
 if __name__ == '__main__':
   main()

android/gyp/create_ui_locale_resources.pydeps

@@ -0,0 +1,28 @@
+# Generated by running:
+#   build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_ui_locale_resources.pydeps build/android/gyp/create_ui_locale_resources.py
+../../../third_party/jinja2/__init__.py
+../../../third_party/jinja2/_compat.py
+../../../third_party/jinja2/bccache.py
+../../../third_party/jinja2/compiler.py
+../../../third_party/jinja2/defaults.py
+../../../third_party/jinja2/environment.py
+../../../third_party/jinja2/exceptions.py
+../../../third_party/jinja2/filters.py
+../../../third_party/jinja2/idtracking.py
+../../../third_party/jinja2/lexer.py
+../../../third_party/jinja2/loaders.py
+../../../third_party/jinja2/nodes.py
+../../../third_party/jinja2/optimizer.py
+../../../third_party/jinja2/parser.py
+../../../third_party/jinja2/runtime.py
+../../../third_party/jinja2/tests.py
+../../../third_party/jinja2/utils.py
+../../../third_party/jinja2/visitor.py
+../../../third_party/markupsafe/__init__.py
+../../../third_party/markupsafe/_compat.py
+../../../third_party/markupsafe/_native.py
+../../gn_helpers.py
+create_ui_locale_resources.py
+util/__init__.py
+util/build_utils.py
+util/resource_utils.py

android/gyp/desugar.py

@@ -53,11 +53,9 @@ def main():
       stderr_filter=build_utils.FilterReflectiveAccessJavaWarnings)
 
   if options.depfile:
-    build_utils.WriteDepfile(
-        options.depfile,
-        options.output_jar,
-        inputs=options.bootclasspath + options.classpath,
-        add_pydeps=False)
+    build_utils.WriteDepfile(options.depfile,
+                             options.output_jar,
+                             inputs=options.bootclasspath + options.classpath)
 
 
 if __name__ == '__main__':

android/gyp/dex.pydeps

@@ -1,6 +1,7 @@
 # Generated by running:
 #   build/print_python_deps.py --root build/android/gyp --output build/android/gyp/dex.pydeps build/android/gyp/dex.py
 ../../gn_helpers.py
+../../print_python_deps.py
 ../convert_dex_profile.py
 dex.py
 util/__init__.py

android/gyp/dist_aar.py

@@ -152,8 +152,7 @@ def main(args):
   if options.depfile:
     all_inputs = (options.jars + options.dependencies_res_zips +
                   options.r_text_files + options.proguard_configs)
-    build_utils.WriteDepfile(options.depfile, options.output, all_inputs,
-                             add_pydeps=False)
+    build_utils.WriteDepfile(options.depfile, options.output, all_inputs)
 
 
 if __name__ == '__main__':

android/gyp/gcc_preprocess.py

@@ -47,7 +47,7 @@ def main(args):
   DoGcc(options)
 
   if options.depfile:
-    build_utils.WriteDepfile(options.depfile, options.output, add_pydeps=False)
+    build_utils.WriteDepfile(options.depfile, options.output)
 
 
 if __name__ == '__main__':

android/gyp/java_cpp_enum.py

@@ -417,7 +417,6 @@ ${ENUM_ENTRIES}
 def DoMain(argv):
   usage = 'usage: %prog [options] [output_dir] input_file(s)...'
   parser = optparse.OptionParser(usage=usage)
-  build_utils.AddDepfileOption(parser)
 
   parser.add_option('--srcjar',
                     help='When specified, a .srcjar at the given path is '
@@ -434,9 +433,6 @@ def DoMain(argv):
       for output_path, data in DoGenerate(input_paths):
         build_utils.AddToZipHermetic(srcjar, output_path, data=data)
 
-  if options.depfile:
-    build_utils.WriteDepfile(options.depfile, options.srcjar, add_pydeps=False)
-
 
 if __name__ == '__main__':
   DoMain(sys.argv[1:])

android/gyp/lint.py

@@ -335,10 +335,7 @@ def main():
   build_utils.Touch(args.stamp)
 
   if args.depfile:
-    build_utils.WriteDepfile(args.depfile,
-                             args.stamp,
-                             depfile_deps,
-                             add_pydeps=False)  # pydeps listed in GN.
+    build_utils.WriteDepfile(args.depfile, args.stamp, depfile_deps)
 
 
 if __name__ == '__main__':

android/gyp/main_dex_list.py

@@ -132,11 +132,9 @@ def main():
     f.write(main_dex_list)
 
   if args.depfile:
-    build_utils.WriteDepfile(
-        args.depfile,
-        args.main_dex_list_path,
-        inputs=args.class_inputs_filearg,
-        add_pydeps=False)
+    build_utils.WriteDepfile(args.depfile,
+                             args.main_dex_list_path,
+                             inputs=args.class_inputs_filearg)
 
 
 if __name__ == '__main__':

android/gyp/merge_manifest.py

@@ -137,8 +137,7 @@ def main(argv):
 
   if args.depfile:
     inputs = extras + classpath.split(':')
-    build_utils.WriteDepfile(args.depfile, args.output, inputs=inputs,
-                             add_pydeps=False)
+    build_utils.WriteDepfile(args.depfile, args.output, inputs=inputs)
 
 
 if __name__ == '__main__':

android/gyp/prepare_resources.pydeps

@@ -22,6 +22,7 @@
 ../../../third_party/markupsafe/_compat.py
 ../../../third_party/markupsafe/_native.py
 ../../gn_helpers.py
+../../print_python_deps.py
 prepare_resources.py
 util/__init__.py
 util/build_utils.py

android/gyp/proguard.py

@@ -512,8 +512,7 @@ def _MaybeWriteStampAndDepFile(options, inputs):
     build_utils.Touch(options.stamp)
     output = options.stamp
   if options.depfile:
-    build_utils.WriteDepfile(
-        options.depfile, output, inputs=inputs, add_pydeps=False)
+    build_utils.WriteDepfile(options.depfile, output, inputs=inputs)
 
 
 def main():

android/gyp/turbine.pydeps

@@ -1,6 +1,7 @@
 # Generated by running:
 #   build/print_python_deps.py --root build/android/gyp --output build/android/gyp/turbine.pydeps build/android/gyp/turbine.py
 ../../gn_helpers.py
+../../print_python_deps.py
 turbine.py
 util/__init__.py
 util/build_utils.py

android/gyp/util/build_utils.py

@@ -541,49 +541,6 @@ def GetSortedTransitiveDependencies(top, deps_func):
   return list(deps_map)
 
 
-def ComputePythonDependencies():
-  """Gets the paths of imported non-system python modules.
-
-  A path is assumed to be a "system" import if it is outside of chromium's
-  src/. The paths will be relative to the current directory.
-  """
-  _ForceLazyModulesToLoad()
-  module_paths = (m.__file__ for m in sys.modules.values()
-                  if m is not None and hasattr(m, '__file__'))
-  abs_module_paths = map(os.path.abspath, module_paths)
-
-  abs_dir_source_root = os.path.abspath(DIR_SOURCE_ROOT)
-  non_system_module_paths = [
-      p for p in abs_module_paths if p.startswith(abs_dir_source_root)
-  ]
-
-  def ConvertPycToPy(s):
-    if s.endswith('.pyc'):
-      return s[:-1]
-    return s
-
-  non_system_module_paths = map(ConvertPycToPy, non_system_module_paths)
-  non_system_module_paths = map(os.path.relpath, non_system_module_paths)
-  return sorted(set(non_system_module_paths))
-
-
-def _ForceLazyModulesToLoad():
-  """Forces any lazily imported modules to fully load themselves.
-
-  Inspecting the modules' __file__ attribute causes lazily imported modules
-  (e.g. from email) to get fully imported and update sys.modules. Iterate
-  over the values until sys.modules stabilizes so that no modules are missed.
-  """
-  while True:
-    num_modules_before = len(sys.modules.keys())
-    for m in sys.modules.values():
-      if m is not None and hasattr(m, '__file__'):
-        _ = m.__file__
-    num_modules_after = len(sys.modules.keys())
-    if num_modules_before == num_modules_after:
-      break
-
-
 def InitLogging(enabling_env):
   logging.basicConfig(
       level=logging.DEBUG if os.environ.get(enabling_env) else logging.WARNING,
@@ -611,12 +568,10 @@ def AddDepfileOption(parser):
        help='Path to depfile (refer to `gn help depfile`)')
 
 
-def WriteDepfile(depfile_path, first_gn_output, inputs=None, add_pydeps=True):
+def WriteDepfile(depfile_path, first_gn_output, inputs=None):
   assert depfile_path != first_gn_output  # http://crbug.com/646165
   assert not isinstance(inputs, string_types)  # Easy mistake to make
   inputs = inputs or []
-  if add_pydeps:
-    inputs = ComputePythonDependencies() + inputs
   MakeDirectory(os.path.dirname(depfile_path))
   # Ninja does not support multiple outputs in depfiles.
   with open(depfile_path, 'w') as depfile:

android/gyp/util/md5_check.py

@@ -14,6 +14,9 @@ import zipfile
 
 from util import build_utils
 
+sys.path.insert(1, os.path.join(build_utils.DIR_SOURCE_ROOT, 'build'))
+import print_python_deps
+
 # When set and a difference is detected, a diff of what changed is printed.
 PRINT_EXPLANATIONS = int(os.environ.get('PRINT_BUILD_EXPLANATIONS', 0))
 
@@ -48,7 +51,7 @@ def CallAndWriteDepfileIfStale(on_stale_md5,
   input_strings = list(input_strings or [])
   output_paths = list(output_paths or [])
 
-  input_paths += build_utils.ComputePythonDependencies()
+  input_paths += print_python_deps.ComputePythonDependencies()
 
   CallAndRecordIfStale(
       on_stale_md5,
@@ -64,8 +67,7 @@ def CallAndWriteDepfileIfStale(on_stale_md5,
   # on bots that build with & without patch, and the patch changes the depfile
   # location.
   if hasattr(options, 'depfile') and options.depfile:
-    build_utils.WriteDepfile(
-        options.depfile, output_paths[0], depfile_deps, add_pydeps=False)
+    build_utils.WriteDepfile(options.depfile, output_paths[0], depfile_deps)
 
 
 def CallAndRecordIfStale(function,

android/gyp/write_build_config.py

@@ -1911,8 +1911,7 @@ def main(argv):
   build_utils.WriteJson(config, options.build_config, only_if_changed=True)
 
   if options.depfile:
-    build_utils.WriteDepfile(options.depfile, options.build_config, all_inputs,
-                             add_pydeps=False)  # pydeps listed in GN.
+    build_utils.WriteDepfile(options.depfile, options.build_config, all_inputs)
 
 
 if __name__ == '__main__':

android/gyp/write_native_libraries_java.py

@@ -102,8 +102,7 @@ def main():
     assert options.native_libraries_list
     build_utils.WriteDepfile(options.depfile,
                              options.output,
-                             inputs=[options.native_libraries_list],
-                             add_pydeps=False)
+                             inputs=[options.native_libraries_list])
 
 
 if __name__ == '__main__':

android/gyp/zip.py

@@ -63,8 +63,9 @@ def main(args):
 
   # Depfile used only by dist_jar().
   if options.depfile:
-    build_utils.WriteDepfile(
-        options.depfile, options.output, inputs=depfile_deps, add_pydeps=False)
+    build_utils.WriteDepfile(options.depfile,
+                             options.output,
+                             inputs=depfile_deps)
 
 
 if __name__ == '__main__':

android/test_runner.pydeps

@@ -131,6 +131,7 @@
 ../../tools/swarming_client/libs/logdog/streamname.py
 ../../tools/swarming_client/libs/logdog/varint.py
 ../gn_helpers.py
+../print_python_deps.py
 ../skia_gold_common/__init__.py
 ../skia_gold_common/skia_gold_properties.py
 ../skia_gold_common/skia_gold_session.py

config/android/rules.gni

@@ -524,17 +524,12 @@ if (enable_java_templates) {
       # The sources aren't compiled so don't check their dependencies.
       check_includes = false
       script = "//build/android/gyp/java_cpp_enum.py"
-      depfile = "$target_gen_dir/$target_name.d"
 
       _srcjar_path = "${target_gen_dir}/${target_name}.srcjar"
       _rebased_srcjar_path = rebase_path(_srcjar_path, root_build_dir)
       _rebased_sources = rebase_path(invoker.sources, root_build_dir)
 
-      args = [
-               "--depfile",
-               rebase_path(depfile, root_build_dir),
-               "--srcjar=$_rebased_srcjar_path",
-             ] + _rebased_sources
+      args = [ "--srcjar=$_rebased_srcjar_path" ] + _rebased_sources
       outputs = [ _srcjar_path ]
     }
   }
@@ -5112,17 +5107,11 @@ template("generate_ui_locale_resources") {
   _output_zip = "${root_out_dir}/resource_zips/${_rebased_output_zip_path}/" +
                 "${target_name}.zip"
 
-  _locales = invoker.ui_locales
-  _depfile = "$target_gen_dir/$target_name.d"
-
-  action(_generating_target_name) {
+  action_with_pydeps(_generating_target_name) {
     script = "//build/android/gyp/create_ui_locale_resources.py"
-    depfile = _depfile
     outputs = [ _output_zip ]
     args = [
-      "--locale-list=$_locales",
-      "--depfile",
-      rebase_path(_depfile, root_build_dir),
+      "--locale-list=${invoker.ui_locales}",
       "--output-zip",
       rebase_path(_output_zip, root_build_dir),
     ]

print_python_deps.py

@@ -22,7 +22,7 @@ import sys
 _SRC_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
 
 
-def _ComputePythonDependencies():
+def ComputePythonDependencies():
   """Gets the paths of imported non-system python modules.
 
   A path is assumed to be a "system" import if it is outside of chromium's
@@ -179,7 +179,7 @@ def main():
     sys.stderr.write('python={}\n'.format(sys.executable))
     raise
 
-  paths_set = _ComputePythonDependencies()
+  paths_set = ComputePythonDependencies()
   for path in options.whitelists:
     paths_set.update(os.path.abspath(p) for p in _FindPythonInDirectory(path))
 

protoc_java.py

@@ -77,8 +77,7 @@ def main(argv):
   if options.depfile:
     assert options.srcjar
     deps = args + [options.protoc]
-    build_utils.WriteDepfile(options.depfile, options.srcjar, deps,
-                             add_pydeps=False)
+    build_utils.WriteDepfile(options.depfile, options.srcjar, deps)
 
   if options.stamp:
     build_utils.Touch(options.stamp)