Speed up diff calculations in dex.py (~200ms)

Explicitly tell md5_check,py which zip files to track changes in so that
it stops looking at the massive r8.jar when doing diff calculations.

Brings the md5_check time for dex.py from ~150ms -> 1ms

Bug: 906803
Change-Id: I52435c2de3fdc34a23e2e9e4b717bf8fae239c1f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1842013
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Sam Maier <smaier@chromium.org>
Cr-Original-Commit-Position: refs/heads/master@{#703567}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 11190da8c381ede4cfa5f01760c38c3449ae3471

android/gyp/dex.py

@@ -402,7 +402,8 @@ def main(args):
       output_paths=output_paths,
       input_paths=input_paths,
       input_strings=dex_cmd + [bool(options.incremental_dir)],
-      pass_changes=True)
+      pass_changes=True,
+      track_subpaths_whitelist=options.class_inputs)
 
 
 if __name__ == '__main__':

android/gyp/util/build_utils.py

@@ -642,6 +642,7 @@ def CallAndWriteDepfileIfStale(on_stale_md5,
                                output_paths=None,
                                force=False,
                                pass_changes=False,
+                               track_subpaths_whitelist=None,
                                depfile_deps=None):
   """Wraps md5_check.CallAndRecordIfStale() and writes a depfile if applicable.
 
@@ -669,7 +670,8 @@ def CallAndWriteDepfileIfStale(on_stale_md5,
       input_strings=input_strings,
       output_paths=output_paths,
       force=force,
-      pass_changes=pass_changes)
+      pass_changes=pass_changes,
+      track_subpaths_whitelist=track_subpaths_whitelist)
 
   # Write depfile even when inputs have not changed to ensure build correctness
   # on bots that build with & without patch, and the patch changes the depfile

android/gyp/util/md5_check.py

@@ -20,9 +20,14 @@ PRINT_EXPLANATIONS = int(os.environ.get('PRINT_BUILD_EXPLANATIONS', 0))
 _FORCE_REBUILD = int(os.environ.get('FORCE_REBUILD', 0))
 
 
-def CallAndRecordIfStale(
-    function, record_path=None, input_paths=None, input_strings=None,
-    output_paths=None, force=False, pass_changes=False):
+def CallAndRecordIfStale(function,
+                         record_path=None,
+                         input_paths=None,
+                         input_strings=None,
+                         output_paths=None,
+                         force=False,
+                         pass_changes=False,
+                         track_subpaths_whitelist=None):
   """Calls function if outputs are stale.
 
   Outputs are considered stale if:
@@ -43,6 +48,8 @@ def CallAndRecordIfStale(
     force: Whether to treat outputs as missing regardless of whether they
       actually are.
     pass_changes: Whether to pass a Changes instance to |function|.
+    track_subpaths_whitelist: Relevant only when pass_changes=True. List of .zip
+      files from |input_paths| to make subpath information available for.
   """
   assert record_path or output_paths
   input_paths = input_paths or []
@@ -57,10 +64,11 @@ def CallAndRecordIfStale(
   new_metadata = _Metadata(track_entries=pass_changes or PRINT_EXPLANATIONS)
   new_metadata.AddStrings(input_strings)
 
+  zip_whitelist = set(track_subpaths_whitelist or [])
   for path in input_paths:
     # It's faster to md5 an entire zip file than it is to just locate & hash
     # its central directory (which is what this used to do).
-    if pass_changes and _IsZipFile(path):
+    if path in zip_whitelist:
       entries = _ExtractZipEntries(path)
       new_metadata.AddZipFile(path, entries)
     else:
@@ -391,14 +399,6 @@ def _ComputeInlineMd5(iterable):
   return md5.hexdigest()
 
 
-def _IsZipFile(path):
-  """Returns whether to treat the given file as a zip file."""
-  # ijar doesn't set the CRC32 field.
-  if path.endswith('.interface.jar'):
-    return False
-  return path[-4:] in ('.zip', '.apk', '.jar') or path.endswith('.srcjar')
-
-
 def _ExtractZipEntries(path):
   """Returns a list of (path, CRC32) of all files within |path|."""
   entries = []

android/gyp/util/md5_check_test.py

@@ -36,12 +36,18 @@ class TestMd5Check(unittest.TestCase):
     # Test out empty zip file to start.
     _WriteZipFile(input_file2.name, [])
     input_files = [input_file1.name, input_file2.name]
+    zip_paths = [input_file2.name]
 
     record_path = tempfile.NamedTemporaryFile(suffix='.stamp')
 
-    def CheckCallAndRecord(should_call, message, force=False,
-                           outputs_specified=False, outputs_missing=False,
-                           expected_changes=None, added_or_modified_only=None):
+    def CheckCallAndRecord(should_call,
+                           message,
+                           force=False,
+                           outputs_specified=False,
+                           outputs_missing=False,
+                           expected_changes=None,
+                           added_or_modified_only=None,
+                           track_subentries=False):
       output_paths = None
       if outputs_specified:
         output_file1 = tempfile.NamedTemporaryFile()
@@ -66,7 +72,8 @@ class TestMd5Check(unittest.TestCase):
           input_strings=input_strings,
           output_paths=output_paths,
           force=force,
-          pass_changes=(expected_changes or added_or_modified_only) is not None)
+          pass_changes=(expected_changes or added_or_modified_only) is not None,
+          track_subpaths_whitelist=zip_paths if track_subentries else None)
       self.assertEqual(should_call, self.called, message)
       if expected_changes:
         description = self.changes.DescribeDifference()
@@ -80,7 +87,6 @@ class TestMd5Check(unittest.TestCase):
     CheckCallAndRecord(True, 'should call when record doesn\'t exist',
                        expected_changes='Previous stamp file not found.',
                        added_or_modified_only=False)
-    CheckCallAndRecord(True, 'pass_changes changed')
     CheckCallAndRecord(False, 'should not call when nothing changed')
     input_files = input_files[::-1]
     CheckCallAndRecord(False, 'reordering of inputs shouldn\'t trigger call')
@@ -130,15 +136,21 @@ class TestMd5Check(unittest.TestCase):
         added_or_modified_only=False)
 
     _WriteZipFile(input_file2.name, [('path/1.txt', '1')])
-    CheckCallAndRecord(True, 'added subpath should trigger call',
-                       expected_changes='*Modified: %s*Subpath added: %s' % (
-                                        input_file2.name, 'path/1.txt'),
-                       added_or_modified_only=True)
+    CheckCallAndRecord(
+        True,
+        'added subpath should trigger call',
+        expected_changes='*Modified: %s*Subpath added: %s' % (input_file2.name,
+                                                              'path/1.txt'),
+        added_or_modified_only=True,
+        track_subentries=True)
     _WriteZipFile(input_file2.name, [('path/1.txt', '2')])
-    CheckCallAndRecord(True, 'changed subpath should trigger call',
-                       expected_changes='*Modified: %s*Subpath modified: %s' % (
-                                        input_file2.name, 'path/1.txt'),
-                       added_or_modified_only=True)
+    CheckCallAndRecord(
+        True,
+        'changed subpath should trigger call',
+        expected_changes='*Modified: %s*Subpath modified: %s' %
+        (input_file2.name, 'path/1.txt'),
+        added_or_modified_only=True,
+        track_subentries=True)
 
     _WriteZipFile(input_file2.name, [])
     CheckCallAndRecord(True, 'removed subpath should trigger call',