(reland) Optimize check_gn_headers.py for speed

- Process the output of 'ninja' live through the pipe rather than
  waiting for it to finish.
- Use multiprocessing.

Benchmark results using pypy:
Before:
17.10user 14.38system 0:20.05elapsed 157%CPU (0avgtext+0avgdata 1708036maxresident)k
18.06user 15.63system 0:21.92elapsed 153%CPU (0avgtext+0avgdata 1710912maxresident)k
17.94user 13.72system 0:19.91elapsed 158%CPU (0avgtext+0avgdata 1172128maxresident)k

After:
15.28user 7.11system 0:06.25elapsed 357%CPU (0avgtext+0avgdata 319236maxresident)k
16.03user 7.74system 0:06.84elapsed 347%CPU (0avgtext+0avgdata 319396maxresident)k
15.24user 8.16system 0:06.36elapsed 367%CPU (0avgtext+0avgdata 318272maxresident)k

BUG=661774

Review-Url: https://codereview.chromium.org/2842513003
Review-Url: https://codereview.chromium.org/2846473002
Cr-Original-Commit-Position: refs/heads/master@{#467604}
Cr-Original-Original-Commit-Position: refs/heads/master@{#466900}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: ef74ec9973e071da2dc5813f0b93662a0f1497bf

check_gn_headers.py

@@ -15,12 +15,27 @@ import os
 import re
 import subprocess
 import sys
+from multiprocessing import Process, Queue
 
 
-def GetHeadersFromNinja(out_dir):
+def GetHeadersFromNinja(out_dir, q):
   """Return all the header files from ninja_deps"""
-  ninja_out = subprocess.check_output(['ninja', '-C', out_dir, '-t', 'deps'])
-  return ParseNinjaDepsOutput(ninja_out)
+
+  def NinjaSource():
+    cmd = ['ninja', '-C', out_dir, '-t', 'deps']
+    # A negative bufsize means to use the system default, which usually
+    # means fully buffered.
+    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1)
+    for line in iter(popen.stdout.readline, ''):
+      yield line.rstrip()
+
+    popen.stdout.close()
+    return_code = popen.wait()
+    if return_code:
+      raise subprocess.CalledProcessError(return_code, cmd)
+
+  ninja_out = NinjaSource()
+  q.put(ParseNinjaDepsOutput(ninja_out))
 
 
 def ParseNinjaDepsOutput(ninja_out):
@@ -30,7 +45,7 @@ def ParseNinjaDepsOutput(ninja_out):
   prefix = '..' + os.sep + '..' + os.sep
 
   is_valid = False
-  for line in ninja_out.split('\n'):
+  for line in ninja_out:
     if line.startswith('    '):
       if not is_valid:
         continue
@@ -49,11 +64,11 @@ def ParseNinjaDepsOutput(ninja_out):
   return all_headers
 
 
-def GetHeadersFromGN(out_dir):
+def GetHeadersFromGN(out_dir, q):
   """Return all the header files from GN"""
   subprocess.check_call(['gn', 'gen', out_dir, '--ide=json', '-q'])
   gn_json = json.load(open(os.path.join(out_dir, 'project.json')))
-  return ParseGNProjectJSON(gn_json)
+  q.put(ParseGNProjectJSON(gn_json))
 
 
 def ParseGNProjectJSON(gn):
@@ -70,7 +85,7 @@ def ParseGNProjectJSON(gn):
   return all_headers
 
 
-def GetDepsPrefixes():
+def GetDepsPrefixes(q):
   """Return all the folders controlled by DEPS file"""
   gclient_out = subprocess.check_output(
       ['gclient', 'recurse', '--no-progress', '-j1',
@@ -80,7 +95,7 @@ def GetDepsPrefixes():
     if i.startswith('src/'):
       i = i[4:]
       prefixes.add(i)
-  return prefixes
+  q.put(prefixes)
 
 
 def ParseWhiteList(whitelist):
@@ -101,13 +116,29 @@ def main():
 
   args, _extras = parser.parse_known_args()
 
-  d = GetHeadersFromNinja(args.out_dir)
-  gn = GetHeadersFromGN(args.out_dir)
+  d_q = Queue()
+  d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir, d_q,))
+  d_p.start()
+
+  gn_q = Queue()
+  gn_p = Process(target=GetHeadersFromGN, args=(args.out_dir, gn_q,))
+  gn_p.start()
+
+  deps_q = Queue()
+  deps_p = Process(target=GetDepsPrefixes, args=(deps_q,))
+  deps_p.start()
+
+  d = d_q.get()
+  gn = gn_q.get()
   missing = d - gn
 
-  deps = GetDepsPrefixes()
+  deps = deps_q.get()
   missing = {m for m in missing if not any(m.startswith(d) for d in deps)}
 
+  d_p.join()
+  gn_p.join()
+  deps_p.join()
+
   if args.whitelist:
     whitelist = ParseWhiteList(open(args.whitelist).read())
     missing -= whitelist

check_gn_headers_unittest.py

@@ -58,7 +58,7 @@ a/b/c
 
 class CheckGnHeadersTest(unittest.TestCase):
   def testNinja(self):
-    headers = check_gn_headers.ParseNinjaDepsOutput(ninja_input)
+    headers = check_gn_headers.ParseNinjaDepsOutput(ninja_input.split('\n'))
     expected = set([
         'dir/path/b.h',
         'c.hh',
@@ -71,7 +71,8 @@ class CheckGnHeadersTest(unittest.TestCase):
     old_sep = os.sep
     os.sep = '\\'
 
-    headers = check_gn_headers.ParseNinjaDepsOutput(ninja_input_win)
+    headers = check_gn_headers.ParseNinjaDepsOutput(
+        ninja_input_win.split('\n'))
     expected = set([
         'dir\\path\\b.h',
         'c.hh',