Android: Parallelize javac post-processing
We do a number of tasks in javac.py after the javac command completes:
- Move annotation processor generated files to out/Default/gen.
- Scan all java files (including aforesaid generated ones) to
construct .apk.jar.info files.
- Pack all .class (and other files?) into a .jar file for future build
steps.
This CL splits out these tasks and does them in parallel, in particular
also processing java files in parallel.
Estimated incremental build savings: ~1.2s
Bug: 906803
Change-Id: Ic7a4d4e4cee998ea15f90a1c435de34e60780a3a
Reviewed-on: https://chromium-review.googlesource.com/c/1352806
Commit-Queue: Peter Wen <wnwen@chromium.org>
Reviewed-by: agrieve <agrieve@chromium.org>
Cr-Original-Commit-Position: refs/heads/master@{#613227}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: d0df151404f2c6d421121690b073b088d8ab1464
This commit is contained in:
Peter Wen
Commit Bot
Родитель
a2fb01732c
Коммит
8a0a1a308d
|
|
@ -63,10 +63,11 @@ def Jar(class_files, classes_dir, jar_path, manifest_file=None,
|
|||
|
||||
def JarDirectory(classes_dir, jar_path, manifest_file=None, predicate=None,
|
||||
provider_configurations=None, additional_files=None):
|
||||
all_files = sorted(build_utils.FindInDirectory(classes_dir, '*'))
|
||||
all_files = build_utils.FindInDirectory(classes_dir, '*')
|
||||
if predicate:
|
||||
all_files = [
|
||||
f for f in all_files if predicate(os.path.relpath(f, classes_dir))]
|
||||
all_files.sort()
|
||||
|
||||
Jar(all_files, classes_dir, jar_path, manifest_file=manifest_file,
|
||||
provider_configurations=provider_configurations,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@
|
|||
|
||||
import distutils.spawn
|
||||
import itertools
|
||||
import logging
|
||||
import multiprocessing
|
||||
import optparse
|
||||
import os
|
||||
import shutil
|
||||
|
|
@ -174,6 +176,7 @@ def _ExtractClassFiles(jar_path, dest_dir, java_files):
|
|||
partial_java_path = path_without_suffix + '.java'
|
||||
return not any(p.endswith(partial_java_path) for p in java_files)
|
||||
|
||||
logging.info('Extracting class files from %s', jar_path)
|
||||
build_utils.ExtractAll(jar_path, path=dest_dir, predicate=extract_predicate)
|
||||
for path in build_utils.FindInDirectory(dest_dir, '*.class'):
|
||||
shutil.copystat(jar_path, path)
|
||||
|
|
@ -226,7 +229,39 @@ def _CheckPathMatchesClassName(java_file, package_name, class_name):
|
|||
(java_file, expected_path_suffix))
|
||||
|
||||
|
||||
def _CreateInfoFile(java_files, options, srcjar_files, javac_generated_sources):
|
||||
def _MoveGeneratedJavaFilesToGenDir(classes_dir, generated_java_dir):
|
||||
# Move any Annotation Processor-generated .java files into $out/gen
|
||||
# so that codesearch can find them.
|
||||
javac_generated_sources = []
|
||||
for src_path in build_utils.FindInDirectory(classes_dir, '*.java'):
|
||||
dst_path = os.path.join(generated_java_dir,
|
||||
os.path.relpath(src_path, classes_dir))
|
||||
build_utils.MakeDirectory(os.path.dirname(dst_path))
|
||||
shutil.move(src_path, dst_path)
|
||||
javac_generated_sources.append(dst_path)
|
||||
return javac_generated_sources
|
||||
|
||||
|
||||
def _ProcessJavaFileForInfo(java_file):
|
||||
package_name, class_names = _ParsePackageAndClassNames(java_file)
|
||||
return java_file, package_name, class_names
|
||||
|
||||
|
||||
def _ProcessInfo(java_file, package_name, class_names, source, chromium_code):
|
||||
for class_name in class_names:
|
||||
yield '{}.{}'.format(package_name, class_name)
|
||||
# Skip aidl srcjars since they don't indent code correctly.
|
||||
if '_aidl.srcjar' in source:
|
||||
continue
|
||||
assert not chromium_code or len(class_names) == 1, (
|
||||
'Chromium java files must only have one class: {}'.format(source))
|
||||
if chromium_code:
|
||||
# This check is necessary to ensure JMake works.
|
||||
_CheckPathMatchesClassName(java_file, package_name, class_names[0])
|
||||
|
||||
|
||||
def _CreateInfoFile(java_files, jar_path, chromium_code, srcjar_files,
|
||||
classes_dir, generated_java_dir):
|
||||
"""Writes a .jar.info file.
|
||||
|
||||
This maps fully qualified names for classes to either the java file that they
|
||||
|
|
@ -235,27 +270,48 @@ def _CreateInfoFile(java_files, options, srcjar_files, javac_generated_sources):
|
|||
For apks this also produces a coalesced .apk.jar.info file combining all the
|
||||
.jar.info files of its transitive dependencies.
|
||||
"""
|
||||
info_data = dict()
|
||||
for java_file in itertools.chain(java_files, javac_generated_sources):
|
||||
package_name, class_names = _ParsePackageAndClassNames(java_file)
|
||||
for class_name in class_names:
|
||||
fully_qualified_name = '{}.{}'.format(package_name, class_name)
|
||||
info_data[fully_qualified_name] = java_file
|
||||
# Skip aidl srcjars since they don't indent code correctly.
|
||||
output_path = jar_path + '.info'
|
||||
logging.info('Start creating info file: %s', output_path)
|
||||
javac_generated_sources = _MoveGeneratedJavaFilesToGenDir(
|
||||
classes_dir, generated_java_dir)
|
||||
logging.info('Finished moving generated java files: %s', output_path)
|
||||
# 2 processes saves ~0.9s, 3 processes saves ~1.2s, 4 processes saves ~1.2s.
|
||||
pool = multiprocessing.Pool(processes=3)
|
||||
results = pool.imap_unordered(
|
||||
_ProcessJavaFileForInfo,
|
||||
itertools.chain(java_files, javac_generated_sources),
|
||||
chunksize=10)
|
||||
pool.close()
|
||||
all_info_data = {}
|
||||
for java_file, package_name, class_names in results:
|
||||
source = srcjar_files.get(java_file, java_file)
|
||||
if '_aidl.srcjar' in source:
|
||||
continue
|
||||
assert not options.chromium_code or len(class_names) == 1, (
|
||||
'Chromium java files must only have one class: {}'.format(source))
|
||||
if options.chromium_code:
|
||||
_CheckPathMatchesClassName(java_file, package_name, class_names[0])
|
||||
for fully_qualified_name in _ProcessInfo(
|
||||
java_file, package_name, class_names, source, chromium_code):
|
||||
all_info_data[fully_qualified_name] = java_file
|
||||
logging.info('Writing info file: %s', output_path)
|
||||
with build_utils.AtomicOutput(output_path) as f:
|
||||
jar_info_utils.WriteJarInfoFile(f.name, all_info_data, srcjar_files)
|
||||
logging.info('Completed info file: %s', output_path)
|
||||
|
||||
with build_utils.AtomicOutput(options.jar_path + '.info') as f:
|
||||
jar_info_utils.WriteJarInfoFile(f.name, info_data, srcjar_files)
|
||||
|
||||
def _CreateJarFile(jar_path, provider_configurations, additional_jar_files,
|
||||
classes_dir):
|
||||
logging.info('Start creating jar file: %s', jar_path)
|
||||
with build_utils.AtomicOutput(jar_path) as f:
|
||||
jar.JarDirectory(
|
||||
classes_dir,
|
||||
f.name,
|
||||
# Avoid putting generated java files into the jar since
|
||||
# _MoveGeneratedJavaFilesToGenDir has not completed yet
|
||||
predicate=lambda name: not name.endswith('.java'),
|
||||
provider_configurations=provider_configurations,
|
||||
additional_files=additional_jar_files)
|
||||
logging.info('Completed jar file: %s', jar_path)
|
||||
|
||||
|
||||
def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
|
||||
classpath):
|
||||
logging.info('Starting _OnStaleMd5')
|
||||
# Don't bother enabling incremental compilation for non-chromium code.
|
||||
incremental = options.incremental and options.chromium_code
|
||||
|
||||
|
|
@ -308,6 +364,7 @@ def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
|
|||
|
||||
srcjar_files = {}
|
||||
if srcjars:
|
||||
logging.info('Extracting srcjars to %s', generated_java_dir)
|
||||
build_utils.MakeDirectory(generated_java_dir)
|
||||
jar_srcs = []
|
||||
for srcjar in options.java_srcjars:
|
||||
|
|
@ -323,6 +380,7 @@ def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
|
|||
srcjar_files[path] = '{}/{}'.format(
|
||||
srcjar, os.path.relpath(path, generated_java_dir))
|
||||
jar_srcs.extend(extracted_files)
|
||||
logging.info('Done extracting srcjars')
|
||||
java_files.extend(jar_srcs)
|
||||
if changed_paths:
|
||||
# Set the mtime of all sources to 0 since we use the absence of .class
|
||||
|
|
@ -366,6 +424,7 @@ def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
|
|||
if md5_check.PRINT_EXPLANATIONS:
|
||||
stdout_filter = None
|
||||
|
||||
logging.debug('Build command %s', cmd)
|
||||
attempt_build = lambda: build_utils.CheckOutput(
|
||||
cmd,
|
||||
print_stdout=options.chromium_code,
|
||||
|
|
@ -378,24 +437,28 @@ def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
|
|||
if ('project database corrupted' not in e.output
|
||||
and 'jmake: internal Java exception' not in e.output):
|
||||
raise
|
||||
print ('Applying work-around for jmake project database corrupted '
|
||||
'(http://crbug.com/551449).')
|
||||
logging.error(
|
||||
'Applying work-around for jmake project database corrupted '
|
||||
'(http://crbug.com/551449).')
|
||||
os.unlink(pdb_path)
|
||||
attempt_build()
|
||||
logging.info('Finished build command')
|
||||
|
||||
if options.incremental or save_outputs:
|
||||
# Creating the jar file takes the longest, start it first on a separate
|
||||
# process to unblock the rest of the post-processing steps.
|
||||
jar_file_worker = multiprocessing.Process(
|
||||
target=_CreateJarFile,
|
||||
args=(options.jar_path, options.provider_configurations,
|
||||
options.additional_jar_files, classes_dir))
|
||||
jar_file_worker.start()
|
||||
else:
|
||||
jar_file_worker = None
|
||||
build_utils.Touch(options.jar_path)
|
||||
|
||||
if save_outputs:
|
||||
# Move any Annotation Processor-generated .java files into $out/gen
|
||||
# so that codesearch can find them.
|
||||
javac_generated_sources = []
|
||||
for src_path in build_utils.FindInDirectory(classes_dir, '*.java'):
|
||||
dst_path = os.path.join(generated_java_dir,
|
||||
os.path.relpath(src_path, classes_dir))
|
||||
build_utils.MakeDirectory(os.path.dirname(dst_path))
|
||||
shutil.move(src_path, dst_path)
|
||||
javac_generated_sources.append(dst_path)
|
||||
|
||||
_CreateInfoFile(java_files, options, srcjar_files,
|
||||
javac_generated_sources)
|
||||
_CreateInfoFile(java_files, options.jar_path, options.chromium_code,
|
||||
srcjar_files, classes_dir, generated_java_dir)
|
||||
else:
|
||||
build_utils.Touch(options.jar_path + '.info')
|
||||
|
||||
|
|
@ -403,15 +466,9 @@ def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
|
|||
# Make sure output exists.
|
||||
build_utils.Touch(pdb_path)
|
||||
|
||||
if options.incremental or save_outputs:
|
||||
with build_utils.AtomicOutput(options.jar_path) as f:
|
||||
jar.JarDirectory(
|
||||
classes_dir,
|
||||
f.name,
|
||||
provider_configurations=options.provider_configurations,
|
||||
additional_files=options.additional_jar_files)
|
||||
else:
|
||||
build_utils.Touch(options.jar_path)
|
||||
if jar_file_worker:
|
||||
jar_file_worker.join()
|
||||
logging.info('Completed all steps in _OnStaleMd5')
|
||||
|
||||
|
||||
def _ParseAndFlattenGnLists(gn_lists):
|
||||
|
|
@ -538,6 +595,9 @@ def _ParseOptions(argv):
|
|||
|
||||
|
||||
def main(argv):
|
||||
logging.basicConfig(
|
||||
level=logging.INFO if os.environ.get('_JAVAC_DEBUG') else logging.WARNING,
|
||||
format='%(levelname).1s %(relativeCreated)6d %(message)s')
|
||||
colorama.init()
|
||||
|
||||
argv = build_utils.ExpandFileArgs(argv)
|
||||
|
|
@ -632,6 +692,7 @@ def main(argv):
|
|||
force=force,
|
||||
pass_changes=True,
|
||||
add_pydeps=False)
|
||||
logging.info('Script complete: %s', __file__)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче