[Android] Add gyp support for multidex.

Note that this does not enable multidex builds yet.

BUG=272790

Review URL: https://codereview.chromium.org/1278573002

Cr-Original-Commit-Position: refs/heads/master@{#345357}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: e40294eca50d57360a2bb0718adcd5cde7810fbd

android/ant/apk-package.xml

@@ -54,6 +54,9 @@
   <property name="resource.package.file.name" value="${RESOURCE_PACKAGED_APK_NAME}" />
 
   <property name="intermediate.dex.file" location="${DEX_FILE_PATH}" />
+  <condition property="multidex.enabled" value="true">
+    <equals arg1="${MULTIDEX_ENABLED}" arg2="1"/>
+  </condition>
 
   <!-- Macro that enables passing a variable list of external jar files
        to ApkBuilder. -->
@@ -77,6 +80,25 @@
     </sequential>
   </macrodef>
 
+  <macrodef name="multidex-package-helper">
+    <element name="extra-jars" optional="yes" />
+    <sequential>
+      <apkbuilder
+          outfolder="${out.absolute.dir}"
+          resourcefile="${resource.package.file.name}"
+          apkfilepath="${out.packaged.file}"
+          debugpackaging="${build.is.packaging.debug}"
+          debugsigning="${build.is.signing.debug}"
+          verbose="${verbose}"
+          hascode="false"
+          previousBuildType="/"
+          buildType="${build.is.packaging.debug}/${build.is.signing.debug}">
+        <zip path="${intermediate.dex.file}" />
+        <nativefolder path="${native.libs.absolute.dir}" />
+        <extra-jars/>
+      </apkbuilder>
+    </sequential>
+  </macrodef>
 
   <!-- Packages the application. -->
   <target name="-package">
@@ -88,9 +110,16 @@
           </extra-jars>
         </package-helper>
       </then>
+      <else>
+        <if condition="${multidex.enabled}">
+          <then>
+            <multidex-package-helper />
+          </then>
           <else>
             <package-helper />
           </else>
         </if>
+      </else>
+    </if>
   </target>
 </project>

android/apkbuilder_action.gypi

@@ -56,6 +56,11 @@
         '-DDEX_FILE_PATH=<(dex_path)',
       ]
     }],
+    ['enable_multidex == 1', {
+      'action': [
+        '-DMULTIDEX_ENABLED=1',
+      ]
+    }]
   ],
   'action': [
     'python', '<(DEPTH)/build/android/gyp/ant.py',

android/gyp/dex.py

@@ -4,21 +4,57 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import logging
 import optparse
 import os
 import sys
+import tempfile
+import zipfile
 
 from util import build_utils
 from util import md5_check
 
 
-def DoDex(options, paths):
+def DoMultiDex(options, paths):
+  main_dex_list = []
+  main_dex_list_files = build_utils.ParseGypList(options.main_dex_list_paths)
+  for m in main_dex_list_files:
+    with open(m) as main_dex_list_file:
+      main_dex_list.extend(l for l in main_dex_list_file if l)
+
+  with tempfile.NamedTemporaryFile(suffix='.txt') as combined_main_dex_list:
+    combined_main_dex_list.write('\n'.join(main_dex_list))
+    combined_main_dex_list.flush()
+
+    dex_args = [
+      '--multi-dex',
+      '--minimal-main-dex',
+      '--main-dex-list=%s' % combined_main_dex_list.name
+    ]
+
+    DoDex(options, paths, dex_args=dex_args)
+
+  if options.dex_path.endswith('.zip'):
+    iz = zipfile.ZipFile(options.dex_path, 'r')
+    tmp_dex_path = '%s.tmp.zip' % options.dex_path
+    oz = zipfile.ZipFile(tmp_dex_path, 'w', zipfile.ZIP_DEFLATED)
+    for i in iz.namelist():
+      if i.endswith('.dex'):
+        oz.writestr(i, iz.read(i))
+    os.remove(options.dex_path)
+    os.rename(tmp_dex_path, options.dex_path)
+
+
+def DoDex(options, paths, dex_args=None):
   dx_binary = os.path.join(options.android_sdk_tools, 'dx')
   # See http://crbug.com/272064 for context on --force-jumbo.
   dex_cmd = [dx_binary, '--dex', '--force-jumbo', '--output', options.dex_path]
   if options.no_locals != '0':
     dex_cmd.append('--no-locals')
 
+  if dex_args:
+    dex_cmd += dex_args
+
   dex_cmd += paths
 
   record_path = '%s.md5.stamp' % options.dex_path
@@ -54,9 +90,14 @@ def main():
                           'is enabled.'))
   parser.add_option('--no-locals',
                     help='Exclude locals list from the dex file.')
+  parser.add_option('--multi-dex', default=False, action='store_true',
+                    help='Create multiple dex files.')
   parser.add_option('--inputs', help='A list of additional input paths.')
   parser.add_option('--excluded-paths',
                     help='A list of paths to exclude from the dex file.')
+  parser.add_option('--main-dex-list-paths',
+                    help='A list of paths containing a list of the classes to '
+                         'include in the main dex.')
 
   options, paths = parser.parse_args(args)
 
@@ -76,6 +117,13 @@ def main():
     paths = [p for p in paths if not
              os.path.relpath(p, options.output_directory) in exclude_paths]
 
+  if options.multi_dex and options.main_dex_list_paths:
+    DoMultiDex(options, paths)
+  else:
+    if options.multi_dex:
+      logging.warning('--multi-dex is unused without --main-dex-list-paths')
+    elif options.main_dex_list_paths:
+      logging.warning('--main-dex-list-paths is unused without --multi-dex')
     DoDex(options, paths)
 
   if options.depfile:

android/gyp/main_dex_list.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+#
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import os
+import sys
+import tempfile
+
+from util import build_utils
+
+sys.path.append(os.path.abspath(os.path.join(
+    os.path.dirname(__file__), os.pardir)))
+from pylib import constants
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--android-sdk-tools', required=True,
+                      help='Android sdk build tools directory.')
+  parser.add_argument('--main-dex-rules-path', action='append', default=[],
+                      dest='main_dex_rules_paths',
+                      help='A file containing a list of proguard rules to use '
+                           'in determining the class to include in the '
+                           'main dex.')
+  parser.add_argument('--main-dex-list-path', required=True,
+                      help='The main dex list file to generate.')
+  parser.add_argument('paths', nargs='+',
+                      help='JARs for which a main dex list should be '
+                           'generated.')
+
+  args = parser.parse_args()
+
+  with open(args.main_dex_list_path, 'w') as main_dex_list_file:
+
+    shrinked_android_jar = os.path.abspath(
+        os.path.join(args.android_sdk_tools, 'lib', 'shrinkedAndroid.jar'))
+    dx_jar = os.path.abspath(
+        os.path.join(args.android_sdk_tools, 'lib', 'dx.jar'))
+    paths_arg = ':'.join(args.paths)
+    rules_file = os.path.abspath(
+        os.path.join(args.android_sdk_tools, 'mainDexClasses.rules'))
+
+    with tempfile.NamedTemporaryFile(suffix='.jar') as temp_jar:
+      proguard_cmd = [
+        constants.PROGUARD_SCRIPT_PATH,
+        '-forceprocessing',
+        '-dontwarn', '-dontoptimize', '-dontobfuscate', '-dontpreverify',
+        '-injars', paths_arg,
+        '-outjars', temp_jar.name,
+        '-libraryjars', shrinked_android_jar,
+        '-include', rules_file,
+      ]
+      for m in args.main_dex_rules_paths:
+        proguard_cmd.extend(['-include', m])
+
+      main_dex_list = ''
+      try:
+        build_utils.CheckOutput(proguard_cmd)
+
+        java_cmd = [
+          'java', '-cp', dx_jar,
+          'com.android.multidex.MainDexListBuilder',
+          temp_jar.name, paths_arg
+        ]
+        main_dex_list = build_utils.CheckOutput(java_cmd)
+      except build_utils.CalledProcessError as e:
+        if 'output jar is empty' in e.output:
+          pass
+        elif "input doesn't contain any classes" in e.output:
+          pass
+        else:
+          raise
+
+      main_dex_list_file.write(main_dex_list)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())
+

android/main_dex_action.gypi

@@ -0,0 +1,44 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file is meant to be included into an action to provide a rule that
+# generates a list of classes that must be kept in the main dex file.
+#
+# To use this, create a gyp target with the following form:
+#  {
+#    'action_name': 'some name for the action'
+#    'actions': [
+#      'variables': {
+#        'jar_path': 'path to jar',
+#        'output_path': 'output path'
+#      },
+#      'includes': [ 'relative/path/to/main_dex_action.gypi' ],
+#    ],
+#  },
+#
+
+{
+  'message': 'Generating main dex classes list for <(jar_path)',
+  'variables': {
+    'jar_path%': '',
+    'output_path%': '',
+    'main_dex_list_script': '<(DEPTH)/build/android/gyp/main_dex_list.py',
+    'main_dex_rules_path': '<(DEPTH)/build/android/main_dex_classes.flags',
+  },
+  'inputs': [
+    '<(jar_path)',
+    '<(main_dex_list_script)',
+    '<(main_dex_rules_path)',
+  ],
+  'outputs': [
+    '<(output_path)',
+  ],
+  'action': [
+    'python', '<(main_dex_list_script)',
+    '--main-dex-list-path', '<(output_path)',
+    '--android-sdk-tools', '<(android_sdk_tools)',
+    '--main-dex-rules-path', '<(main_dex_rules_path)',
+    '<(jar_path)',
+  ]
+}

android/main_dex_classes.flags

@@ -0,0 +1,3 @@
+-keep @**.MainDex class * {
+  *;
+}

android/pylib/constants/__init__.py

@@ -189,6 +189,9 @@ ANDROID_SDK_TOOLS = os.path.join(ANDROID_SDK_ROOT,
 ANDROID_NDK_ROOT = os.path.join(DIR_SOURCE_ROOT,
                                 'third_party/android_tools/ndk')
 
+PROGUARD_SCRIPT_PATH = os.path.join(
+    ANDROID_SDK_ROOT, 'tools', 'proguard', 'bin', 'proguard.sh')
+
 EMULATOR_SDK_ROOT = os.environ.get('ANDROID_EMULATOR_SDK_ROOT',
                                    os.path.join(DIR_SOURCE_ROOT,
                                                 'android_emulator_sdk'))

config/android/rules.gni

@@ -940,6 +940,9 @@ template("java_library") {
 #   proguard_preprocess: If true, proguard preprocessing will be run. This can
 #     be used to remove unwanted parts of the library.
 #   proguard_config: Path to the proguard config for preprocessing.
+#   supports_android: If true, Android targets (android_library, android_apk)
+#     may depend on this target. Note: if true, this target must only use the
+#     subset of Java available on Android.
 #
 # Example
 #   java_prebuilt("foo_java") {

java.gypi

@@ -64,6 +64,7 @@
     'instr_stamp': '<(intermediate_dir)/instr.stamp',
     'additional_input_paths': [],
     'dex_path': '<(PRODUCT_DIR)/lib.java/<(_target_name).dex.jar',
+    'main_dex_list_path': '<(intermediate_dir)/main_dex_list.txt',
     'generated_src_dirs': ['>@(generated_R_dirs)'],
     'generated_R_dirs': [],
     'has_java_resources%': 0,
@@ -113,6 +114,7 @@
         'variables': {
           'input_jars_paths': ['<(jar_final_path)'],
           'library_dexed_jars_paths': ['<(dex_path)'],
+          'main_dex_list_paths': ['<(main_dex_list_path)'],
         },
       },
     }],
@@ -308,6 +310,14 @@
         '<@(extra_args)',
       ]
     },
+    {
+      'action_name': 'main_dex_list_for_<(_target_name)',
+      'variables': {
+        'jar_path': '<(javac_jar_path)',
+        'output_path': '<(main_dex_list_path)',
+      },
+      'includes': [ 'android/main_dex_action.gypi' ],
+    },
     {
       'action_name': 'instr_jar_<(_target_name)',
       'message': 'Instrumenting <(_target_name) jar',

java_apk.gypi

@@ -70,11 +70,14 @@
   'variables': {
     'tested_apk_obfuscated_jar_path%': '/',
     'tested_apk_dex_path%': '/',
+    'tested_apk_is_multidex%': 0,
     'additional_input_paths': [],
     'create_density_splits%': 0,
     'language_splits': [],
     'input_jars_paths': [],
     'library_dexed_jars_paths': [],
+    'main_dex_list_path': '<(intermediate_dir)/main_dex_list.txt',
+    'main_dex_list_paths': ['<(main_dex_list_path)'],
     'additional_src_dirs': [],
     'generated_src_dirs': [],
     'app_manifest_version_name%': '<(android_app_version_name)',
@@ -131,7 +134,7 @@
     'jar_path': '<(PRODUCT_DIR)/lib.java/<(jar_name)',
     'obfuscated_jar_path': '<(intermediate_dir)/obfuscated.jar',
     'test_jar_path': '<(PRODUCT_DIR)/test.lib.java/<(apk_name).jar',
-    'dex_path': '<(intermediate_dir)/classes.dex',
+    'enable_multidex%': 0,
     'emma_device_jar': '<(android_sdk_root)/tools/lib/emma_device.jar',
     'android_manifest_path%': '<(java_in_dir)/AndroidManifest.xml',
     'split_android_manifest_path': '<(intermediate_dir)/split-manifests/<(android_app_abi)/AndroidManifest.xml',
@@ -160,6 +163,7 @@
         'unsigned_apk_path': '<(intermediate_dir)/<(apk_name)-unsigned.apk',
         'unsigned_abi_split_apk_path': '<(intermediate_dir)/<(apk_name)-abi-<(android_app_abi)-unsigned.apk',
         'create_abi_split%': 0,
+        'enable_multidex%': 0,
       },
       'unsigned_apk_path': '<(unsigned_apk_path)',
       'unsigned_abi_split_apk_path': '<(unsigned_abi_split_apk_path)',
@@ -193,6 +197,11 @@
         }, {
           'managed_input_apk_path': '<(unsigned_apk_path)',
         }],
+        ['enable_multidex == 1', {
+          'dex_path': '<(intermediate_dir)/classes.dex.zip',
+        }, {
+          'dex_path': '<(intermediate_dir)/classes.dex',
+        }],
       ],
     },
     'native_lib_target%': '',
@@ -213,6 +222,7 @@
     'native_lib_placeholder_stamp': '<(apk_package_native_libs_dir)/<(android_app_abi)/native_lib_placeholder.stamp',
     'native_lib_placeholders': [],
     'main_apk_name': '<(apk_name)',
+    'dex_path': '<(dex_path)',
     'enable_errorprone%': '0',
     'errorprone_exe_path': '<(PRODUCT_DIR)/bin.java/chromium_errorprone',
   },
@@ -231,6 +241,7 @@
       'apk_output_jar_path': '<(jar_path)',
       'tested_apk_obfuscated_jar_path': '<(obfuscated_jar_path)',
       'tested_apk_dex_path': '<(dex_path)',
+      'tested_apk_is_multidex': '<(enable_multidex)',
     },
   },
   'conditions': [
@@ -752,8 +763,7 @@
           ],
         },
       ],
-    },
-    ]
+    }],
   ],
   'dependencies': [
     '<(DEPTH)/tools/android/md5sum/md5sum.gyp:md5sum',
@@ -884,6 +894,14 @@
         '>@(java_sources)',
       ],
     },
+    {
+      'action_name': 'main_dex_list_for_<(_target_name)',
+      'variables': {
+        'jar_path': '<(javac_jar_path)',
+        'output_path': '<(main_dex_list_path)',
+      },
+      'includes': [ 'android/main_dex_action.gypi' ],
+    },
     {
       'action_name': 'instr_jar_<(_target_name)',
       'message': 'Instrumenting <(_target_name) jar',
@@ -1005,14 +1023,40 @@
     {
       'action_name': 'dex_<(_target_name)',
       'variables': {
+        'dex_additional_options': [],
         'dex_input_paths': [
-          '>@(library_dexed_jars_paths)',
           '<(jar_path)',
         ],
         'output_path': '<(dex_path)',
         'proguard_enabled_input_path': '<(obfuscated_jar_path)',
       },
+      'conditions': [
+        ['enable_multidex == 1', {
+          'variables': {
+            'dex_additional_options': [
+              '--multi-dex',
+              '--main-dex-list-paths', '>@(main_dex_list_paths)',
+            ],
+          },
+          'inputs': [
+            '>@(main_dex_list_paths)',
+          ],
+        }]
+      ],
       'target_conditions': [
+        ['enable_multidex == 1 or tested_apk_is_multidex == 1', {
+          'variables': {
+            'dex_input_paths': [
+              '>@(input_jars_paths)',
+            ],
+          },
+        }, {
+          'variables': {
+            'dex_input_paths': [
+              '>@(library_dexed_jars_paths)',
+            ],
+          },
+        }],
         ['emma_instrument != 0', {
           'variables': {
             'dex_no_locals': 1,

secondary/third_party/android_tools/BUILD.gn

@@ -49,6 +49,11 @@ android_java_prebuilt("android_support_design_java") {
   jar_path = "$android_sdk_root/extras/android/support/design/libs/android-support-design.jar"
 }
 
+java_prebuilt("android_support_multidex_java") {
+  supports_android = true
+  jar_path = "$android_sdk_root/extras/android/support/multidex/library/libs/android-support-multidex.jar"
+}
+
 android_java_prebuilt("android_support_v13_java") {
   jar_path =
       "$android_sdk_root/extras/android/support/v13/android-support-v13.jar"