Bug 1529691 - use futures in order to process in parallel clang-format. r=bbouvier

Parts of this patch were taken from the original work of :bbouvier in Bug 1521772.
We needed to revert Bug 1521772 since it broken Windows compatibility.

Differential Revision: https://phabricator.services.mozilla.com/D23100

--HG--
extra : moz-landing-system : lando

python/mozbuild/mozbuild/mach_commands.py

@@ -75,6 +75,15 @@ warning heuristic.
 '''
 
 
+# Function used to run clang-format on a batch of files. It is a helper function
+# in order to integrate into the futures ecosystem clang-format.
+def run_one_clang_format_batch(args):
+    try:
+        subprocess.check_output(args)
+    except subprocess.CalledProcessError as e:
+        return e
+
+
 class StoreDebugParamsAndWarnAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         sys.stderr.write('The --debugparams argument is deprecated. Please ' +
@@ -2871,13 +2880,10 @@ class StaticAnalysis(MachCommandBase):
 
         print("Processing %d file(s)..." % len(path_list))
 
-        batchsize = 200
         if show:
-            batchsize = 1
+            for i in range(0, len(path_list)):
+                l = path_list[i: (i + 1)]
 
-        for i in range(0, len(path_list), batchsize):
-            l = path_list[i: (i + batchsize)]
-            if show:
                 # Copy the files into a temp directory
                 # and run clang-format on the temp directory
                 # and show the diff
@@ -2890,15 +2896,14 @@ class StaticAnalysis(MachCommandBase):
                 shutil.copy(l[0], faketmpdir)
                 l[0] = target_file
 
-            # Run clang-format on the list
-            try:
-                check_output(args + l)
-            except CalledProcessError as e:
-                # Something wrong happend
-                print("clang-format: An error occured while running clang-format.")
-                return e.returncode
+                # Run clang-format on the list
+                try:
+                    check_output(args + l)
+                except CalledProcessError as e:
+                    # Something wrong happend
+                    print("clang-format: An error occured while running clang-format.")
+                    return e.returncode
 
-            if show:
                 # show the diff
                 diff_command = ["diff", "-u", original_path, target_file]
                 try:
@@ -2909,8 +2914,37 @@ class StaticAnalysis(MachCommandBase):
                     # there is a diff to show
                     if e.output:
                         print(e.output)
-        if show:
+
             shutil.rmtree(tmpdir)
+            return 0
+
+        # Run clang-format in parallel trying to saturate all of the available cores.
+        import concurrent.futures
+        import multiprocessing
+        import math
+
+        max_workers = multiprocessing.cpu_count()
+        batchsize = int(math.ceil(float(len(path_list)) / max_workers))
+
+        batches = []
+        for i in range(0, len(path_list), batchsize):
+            batches.append(args + path_list[i: (i + batchsize)])
+
+        error_code = None
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = []
+            for batch in batches:
+                futures.append(executor.submit(run_one_clang_format_batch, batch))
+
+            for future in concurrent.futures.as_completed(futures):
+                # Wait for every task to finish
+                ret_val = future.result()
+                if ret_val is not None:
+                    error_code = ret_val
+
+            if error_code is not None:
+                return error_code
         return 0
 
 @CommandProvider