From 6cb89e2fb0c3d7e17e95321eef281e526015936e Mon Sep 17 00:00:00 2001 From: jbudorick Date: Tue, 25 Aug 2015 10:03:49 -0700 Subject: [PATCH] [Android] Add gyp support for multidex. Note that this does not enable multidex builds yet. BUG=272790 Review URL: https://codereview.chromium.org/1278573002 Cr-Original-Commit-Position: refs/heads/master@{#345357} Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src Cr-Mirrored-Commit: e40294eca50d57360a2bb0718adcd5cde7810fbd --- android/ant/apk-package.xml | 33 +++++++- android/apkbuilder_action.gypi | 5 ++ android/gyp/dex.py | 52 +++++++++++- android/gyp/main_dex_list.py | 84 ++++++++++++++++++++ android/main_dex_action.gypi | 44 ++++++++++ android/main_dex_classes.flags | 3 + android/pylib/constants/__init__.py | 3 + config/android/rules.gni | 3 + java.gypi | 10 +++ java_apk.gypi | 52 +++++++++++- secondary/third_party/android_tools/BUILD.gn | 5 ++ 11 files changed, 286 insertions(+), 8 deletions(-) create mode 100755 android/gyp/main_dex_list.py create mode 100644 android/main_dex_action.gypi create mode 100644 android/main_dex_classes.flags diff --git a/android/ant/apk-package.xml b/android/ant/apk-package.xml index e8b76f7e4..cb7956091 100644 --- a/android/ant/apk-package.xml +++ b/android/ant/apk-package.xml @@ -54,6 +54,9 @@ + + + @@ -70,13 +73,32 @@ hascode="${HAS_CODE}" previousBuildType="/" buildType="${build.is.packaging.debug}/${build.is.signing.debug}"> - + + + + + + + + + + + @@ -89,7 +111,14 @@ - + + + + + + + + diff --git a/android/apkbuilder_action.gypi b/android/apkbuilder_action.gypi index 27807d861..e073e9bdb 100644 --- a/android/apkbuilder_action.gypi +++ b/android/apkbuilder_action.gypi @@ -56,6 +56,11 @@ '-DDEX_FILE_PATH=<(dex_path)', ] }], + ['enable_multidex == 1', { + 'action': [ + '-DMULTIDEX_ENABLED=1', + ] + }] ], 'action': [ 'python', '<(DEPTH)/build/android/gyp/ant.py', diff --git a/android/gyp/dex.py b/android/gyp/dex.py index c26d23a61..3b4141f2f 100755 --- a/android/gyp/dex.py +++ b/android/gyp/dex.py @@ -4,21 +4,57 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import logging import optparse import os import sys +import tempfile +import zipfile from util import build_utils from util import md5_check -def DoDex(options, paths): +def DoMultiDex(options, paths): + main_dex_list = [] + main_dex_list_files = build_utils.ParseGypList(options.main_dex_list_paths) + for m in main_dex_list_files: + with open(m) as main_dex_list_file: + main_dex_list.extend(l for l in main_dex_list_file if l) + + with tempfile.NamedTemporaryFile(suffix='.txt') as combined_main_dex_list: + combined_main_dex_list.write('\n'.join(main_dex_list)) + combined_main_dex_list.flush() + + dex_args = [ + '--multi-dex', + '--minimal-main-dex', + '--main-dex-list=%s' % combined_main_dex_list.name + ] + + DoDex(options, paths, dex_args=dex_args) + + if options.dex_path.endswith('.zip'): + iz = zipfile.ZipFile(options.dex_path, 'r') + tmp_dex_path = '%s.tmp.zip' % options.dex_path + oz = zipfile.ZipFile(tmp_dex_path, 'w', zipfile.ZIP_DEFLATED) + for i in iz.namelist(): + if i.endswith('.dex'): + oz.writestr(i, iz.read(i)) + os.remove(options.dex_path) + os.rename(tmp_dex_path, options.dex_path) + + +def DoDex(options, paths, dex_args=None): dx_binary = os.path.join(options.android_sdk_tools, 'dx') # See http://crbug.com/272064 for context on --force-jumbo. dex_cmd = [dx_binary, '--dex', '--force-jumbo', '--output', options.dex_path] if options.no_locals != '0': dex_cmd.append('--no-locals') + if dex_args: + dex_cmd += dex_args + dex_cmd += paths record_path = '%s.md5.stamp' % options.dex_path @@ -54,9 +90,14 @@ def main(): 'is enabled.')) parser.add_option('--no-locals', help='Exclude locals list from the dex file.') + parser.add_option('--multi-dex', default=False, action='store_true', + help='Create multiple dex files.') parser.add_option('--inputs', help='A list of additional input paths.') parser.add_option('--excluded-paths', help='A list of paths to exclude from the dex file.') + parser.add_option('--main-dex-list-paths', + help='A list of paths containing a list of the classes to ' + 'include in the main dex.') options, paths = parser.parse_args(args) @@ -76,7 +117,14 @@ def main(): paths = [p for p in paths if not os.path.relpath(p, options.output_directory) in exclude_paths] - DoDex(options, paths) + if options.multi_dex and options.main_dex_list_paths: + DoMultiDex(options, paths) + else: + if options.multi_dex: + logging.warning('--multi-dex is unused without --main-dex-list-paths') + elif options.main_dex_list_paths: + logging.warning('--main-dex-list-paths is unused without --multi-dex') + DoDex(options, paths) if options.depfile: build_utils.WriteDepfile( diff --git a/android/gyp/main_dex_list.py b/android/gyp/main_dex_list.py new file mode 100755 index 000000000..64420019e --- /dev/null +++ b/android/gyp/main_dex_list.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import os +import sys +import tempfile + +from util import build_utils + +sys.path.append(os.path.abspath(os.path.join( + os.path.dirname(__file__), os.pardir))) +from pylib import constants + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--android-sdk-tools', required=True, + help='Android sdk build tools directory.') + parser.add_argument('--main-dex-rules-path', action='append', default=[], + dest='main_dex_rules_paths', + help='A file containing a list of proguard rules to use ' + 'in determining the class to include in the ' + 'main dex.') + parser.add_argument('--main-dex-list-path', required=True, + help='The main dex list file to generate.') + parser.add_argument('paths', nargs='+', + help='JARs for which a main dex list should be ' + 'generated.') + + args = parser.parse_args() + + with open(args.main_dex_list_path, 'w') as main_dex_list_file: + + shrinked_android_jar = os.path.abspath( + os.path.join(args.android_sdk_tools, 'lib', 'shrinkedAndroid.jar')) + dx_jar = os.path.abspath( + os.path.join(args.android_sdk_tools, 'lib', 'dx.jar')) + paths_arg = ':'.join(args.paths) + rules_file = os.path.abspath( + os.path.join(args.android_sdk_tools, 'mainDexClasses.rules')) + + with tempfile.NamedTemporaryFile(suffix='.jar') as temp_jar: + proguard_cmd = [ + constants.PROGUARD_SCRIPT_PATH, + '-forceprocessing', + '-dontwarn', '-dontoptimize', '-dontobfuscate', '-dontpreverify', + '-injars', paths_arg, + '-outjars', temp_jar.name, + '-libraryjars', shrinked_android_jar, + '-include', rules_file, + ] + for m in args.main_dex_rules_paths: + proguard_cmd.extend(['-include', m]) + + main_dex_list = '' + try: + build_utils.CheckOutput(proguard_cmd) + + java_cmd = [ + 'java', '-cp', dx_jar, + 'com.android.multidex.MainDexListBuilder', + temp_jar.name, paths_arg + ] + main_dex_list = build_utils.CheckOutput(java_cmd) + except build_utils.CalledProcessError as e: + if 'output jar is empty' in e.output: + pass + elif "input doesn't contain any classes" in e.output: + pass + else: + raise + + main_dex_list_file.write(main_dex_list) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/android/main_dex_action.gypi b/android/main_dex_action.gypi new file mode 100644 index 000000000..06717ddcf --- /dev/null +++ b/android/main_dex_action.gypi @@ -0,0 +1,44 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This file is meant to be included into an action to provide a rule that +# generates a list of classes that must be kept in the main dex file. +# +# To use this, create a gyp target with the following form: +# { +# 'action_name': 'some name for the action' +# 'actions': [ +# 'variables': { +# 'jar_path': 'path to jar', +# 'output_path': 'output path' +# }, +# 'includes': [ 'relative/path/to/main_dex_action.gypi' ], +# ], +# }, +# + +{ + 'message': 'Generating main dex classes list for <(jar_path)', + 'variables': { + 'jar_path%': '', + 'output_path%': '', + 'main_dex_list_script': '<(DEPTH)/build/android/gyp/main_dex_list.py', + 'main_dex_rules_path': '<(DEPTH)/build/android/main_dex_classes.flags', + }, + 'inputs': [ + '<(jar_path)', + '<(main_dex_list_script)', + '<(main_dex_rules_path)', + ], + 'outputs': [ + '<(output_path)', + ], + 'action': [ + 'python', '<(main_dex_list_script)', + '--main-dex-list-path', '<(output_path)', + '--android-sdk-tools', '<(android_sdk_tools)', + '--main-dex-rules-path', '<(main_dex_rules_path)', + '<(jar_path)', + ] +} diff --git a/android/main_dex_classes.flags b/android/main_dex_classes.flags new file mode 100644 index 000000000..d9689b206 --- /dev/null +++ b/android/main_dex_classes.flags @@ -0,0 +1,3 @@ +-keep @**.MainDex class * { + *; +} diff --git a/android/pylib/constants/__init__.py b/android/pylib/constants/__init__.py index 46171192f..43c0a6d09 100644 --- a/android/pylib/constants/__init__.py +++ b/android/pylib/constants/__init__.py @@ -189,6 +189,9 @@ ANDROID_SDK_TOOLS = os.path.join(ANDROID_SDK_ROOT, ANDROID_NDK_ROOT = os.path.join(DIR_SOURCE_ROOT, 'third_party/android_tools/ndk') +PROGUARD_SCRIPT_PATH = os.path.join( + ANDROID_SDK_ROOT, 'tools', 'proguard', 'bin', 'proguard.sh') + EMULATOR_SDK_ROOT = os.environ.get('ANDROID_EMULATOR_SDK_ROOT', os.path.join(DIR_SOURCE_ROOT, 'android_emulator_sdk')) diff --git a/config/android/rules.gni b/config/android/rules.gni index 4c6c4bbd3..ffb2b42b8 100644 --- a/config/android/rules.gni +++ b/config/android/rules.gni @@ -940,6 +940,9 @@ template("java_library") { # proguard_preprocess: If true, proguard preprocessing will be run. This can # be used to remove unwanted parts of the library. # proguard_config: Path to the proguard config for preprocessing. +# supports_android: If true, Android targets (android_library, android_apk) +# may depend on this target. Note: if true, this target must only use the +# subset of Java available on Android. # # Example # java_prebuilt("foo_java") { diff --git a/java.gypi b/java.gypi index 614d50a10..dc642b94f 100644 --- a/java.gypi +++ b/java.gypi @@ -64,6 +64,7 @@ 'instr_stamp': '<(intermediate_dir)/instr.stamp', 'additional_input_paths': [], 'dex_path': '<(PRODUCT_DIR)/lib.java/<(_target_name).dex.jar', + 'main_dex_list_path': '<(intermediate_dir)/main_dex_list.txt', 'generated_src_dirs': ['>@(generated_R_dirs)'], 'generated_R_dirs': [], 'has_java_resources%': 0, @@ -113,6 +114,7 @@ 'variables': { 'input_jars_paths': ['<(jar_final_path)'], 'library_dexed_jars_paths': ['<(dex_path)'], + 'main_dex_list_paths': ['<(main_dex_list_path)'], }, }, }], @@ -308,6 +310,14 @@ '<@(extra_args)', ] }, + { + 'action_name': 'main_dex_list_for_<(_target_name)', + 'variables': { + 'jar_path': '<(javac_jar_path)', + 'output_path': '<(main_dex_list_path)', + }, + 'includes': [ 'android/main_dex_action.gypi' ], + }, { 'action_name': 'instr_jar_<(_target_name)', 'message': 'Instrumenting <(_target_name) jar', diff --git a/java_apk.gypi b/java_apk.gypi index 6dbb85876..52902a0ce 100644 --- a/java_apk.gypi +++ b/java_apk.gypi @@ -70,11 +70,14 @@ 'variables': { 'tested_apk_obfuscated_jar_path%': '/', 'tested_apk_dex_path%': '/', + 'tested_apk_is_multidex%': 0, 'additional_input_paths': [], 'create_density_splits%': 0, 'language_splits': [], 'input_jars_paths': [], 'library_dexed_jars_paths': [], + 'main_dex_list_path': '<(intermediate_dir)/main_dex_list.txt', + 'main_dex_list_paths': ['<(main_dex_list_path)'], 'additional_src_dirs': [], 'generated_src_dirs': [], 'app_manifest_version_name%': '<(android_app_version_name)', @@ -131,7 +134,7 @@ 'jar_path': '<(PRODUCT_DIR)/lib.java/<(jar_name)', 'obfuscated_jar_path': '<(intermediate_dir)/obfuscated.jar', 'test_jar_path': '<(PRODUCT_DIR)/test.lib.java/<(apk_name).jar', - 'dex_path': '<(intermediate_dir)/classes.dex', + 'enable_multidex%': 0, 'emma_device_jar': '<(android_sdk_root)/tools/lib/emma_device.jar', 'android_manifest_path%': '<(java_in_dir)/AndroidManifest.xml', 'split_android_manifest_path': '<(intermediate_dir)/split-manifests/<(android_app_abi)/AndroidManifest.xml', @@ -160,6 +163,7 @@ 'unsigned_apk_path': '<(intermediate_dir)/<(apk_name)-unsigned.apk', 'unsigned_abi_split_apk_path': '<(intermediate_dir)/<(apk_name)-abi-<(android_app_abi)-unsigned.apk', 'create_abi_split%': 0, + 'enable_multidex%': 0, }, 'unsigned_apk_path': '<(unsigned_apk_path)', 'unsigned_abi_split_apk_path': '<(unsigned_abi_split_apk_path)', @@ -193,6 +197,11 @@ }, { 'managed_input_apk_path': '<(unsigned_apk_path)', }], + ['enable_multidex == 1', { + 'dex_path': '<(intermediate_dir)/classes.dex.zip', + }, { + 'dex_path': '<(intermediate_dir)/classes.dex', + }], ], }, 'native_lib_target%': '', @@ -213,6 +222,7 @@ 'native_lib_placeholder_stamp': '<(apk_package_native_libs_dir)/<(android_app_abi)/native_lib_placeholder.stamp', 'native_lib_placeholders': [], 'main_apk_name': '<(apk_name)', + 'dex_path': '<(dex_path)', 'enable_errorprone%': '0', 'errorprone_exe_path': '<(PRODUCT_DIR)/bin.java/chromium_errorprone', }, @@ -231,6 +241,7 @@ 'apk_output_jar_path': '<(jar_path)', 'tested_apk_obfuscated_jar_path': '<(obfuscated_jar_path)', 'tested_apk_dex_path': '<(dex_path)', + 'tested_apk_is_multidex': '<(enable_multidex)', }, }, 'conditions': [ @@ -752,8 +763,7 @@ ], }, ], - }, - ] + }], ], 'dependencies': [ '<(DEPTH)/tools/android/md5sum/md5sum.gyp:md5sum', @@ -884,6 +894,14 @@ '>@(java_sources)', ], }, + { + 'action_name': 'main_dex_list_for_<(_target_name)', + 'variables': { + 'jar_path': '<(javac_jar_path)', + 'output_path': '<(main_dex_list_path)', + }, + 'includes': [ 'android/main_dex_action.gypi' ], + }, { 'action_name': 'instr_jar_<(_target_name)', 'message': 'Instrumenting <(_target_name) jar', @@ -1005,14 +1023,40 @@ { 'action_name': 'dex_<(_target_name)', 'variables': { + 'dex_additional_options': [], 'dex_input_paths': [ - '>@(library_dexed_jars_paths)', '<(jar_path)', ], 'output_path': '<(dex_path)', 'proguard_enabled_input_path': '<(obfuscated_jar_path)', }, + 'conditions': [ + ['enable_multidex == 1', { + 'variables': { + 'dex_additional_options': [ + '--multi-dex', + '--main-dex-list-paths', '>@(main_dex_list_paths)', + ], + }, + 'inputs': [ + '>@(main_dex_list_paths)', + ], + }] + ], 'target_conditions': [ + ['enable_multidex == 1 or tested_apk_is_multidex == 1', { + 'variables': { + 'dex_input_paths': [ + '>@(input_jars_paths)', + ], + }, + }, { + 'variables': { + 'dex_input_paths': [ + '>@(library_dexed_jars_paths)', + ], + }, + }], ['emma_instrument != 0', { 'variables': { 'dex_no_locals': 1, diff --git a/secondary/third_party/android_tools/BUILD.gn b/secondary/third_party/android_tools/BUILD.gn index a6b375893..175409b49 100644 --- a/secondary/third_party/android_tools/BUILD.gn +++ b/secondary/third_party/android_tools/BUILD.gn @@ -49,6 +49,11 @@ android_java_prebuilt("android_support_design_java") { jar_path = "$android_sdk_root/extras/android/support/design/libs/android-support-design.jar" } +java_prebuilt("android_support_multidex_java") { + supports_android = true + jar_path = "$android_sdk_root/extras/android/support/multidex/library/libs/android-support-multidex.jar" +} + android_java_prebuilt("android_support_v13_java") { jar_path = "$android_sdk_root/extras/android/support/v13/android-support-v13.jar"