Bug 1508976 - Produce a multi-architecture GeckoView "fat AAR". r=glandium

This follows the model set down for EME artifacts:

- a new tier is added that uses a new Python build action to fetch and
  artifacts
- the action unpacks the fetched artifacts and moves specific inputs
  into places expected by the build and packager
- in automation, MOZ_ARTIFACT_TASK* is used to ensure the artifacts
  come from the correct tasks

In this case, the artifact fetching is done entirely in a new Python
build action that internally uses `mach artifact install --job ...`.
The action also verifies that the fetched artifacts are compatible and
that we're not assembling a fat AAR that is nonsensical.  The specific
inputs are not used in the Fennec APK that is produced; they're only
used in the GeckoView AAR that is produced.

The artifact fetching itself required tweaking to fetch only
`target.maven.zip` artifacts and to not unpack them.

The specific inputs used are the native libraries (libs/$ARCH/*.so)
and the architecture-specific preference files ($ARCH/greprefs.js and
defaults/pref/$ARCH/geckoview-prefs.js).  None of these inputs are
impacted by l10n.

Differential Revision: https://phabricator.services.mozilla.com/D31572

--HG--
extra : moz-landing-system : lando

Makefile.in

@@ -164,6 +164,11 @@ recurse_win32-artifact:
 	mv $(DIST)/i686/bin/* $(DIST)/i686
 endif
 
+ifdef MOZ_ANDROID_FAT_AAR_ARCHITECTURES
+recurse_android-fat-aar-artifact:
+	$(call py_action,fat_aar,$(MOZ_ANDROID_FAT_AAR_ARCHITECTURES) --distdir $(abspath $(DIST)/fat-aar))
+endif # MOZ_ANDROID_FAT_AAR_ARCHITECTURES
+
 ifdef MOZ_WIDGET_TOOLKIT
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites

config/baseconfig.mk

@@ -42,7 +42,7 @@ endif # WINNT
 ifndef INCLUDED_AUTOCONF_MK
 default::
 else
-TIERS := $(if $(MOZ_ARTIFACT_BUILDS),artifact )$(if $(MOZ_EME_WIN32_ARTIFACT),win32-artifact )pre-export export $(if $(COMPILE_ENVIRONMENT),compile )misc libs tools$(if $(filter check recurse_check,$(MAKECMDGOALS)), check)
+TIERS := $(if $(MOZ_ARTIFACT_BUILDS),artifact )$(if $(MOZ_EME_WIN32_ARTIFACT),win32-artifact )$(if $(MOZ_ANDROID_FAT_AAR_ARCHITECTURES),android-fat-aar-artifact )pre-export export $(if $(COMPILE_ENVIRONMENT),compile )misc libs tools$(if $(filter check recurse_check,$(MAKECMDGOALS)), check)
 endif
 
 # These defines are used to support the twin-topsrcdir model for comm-central.

mobile/android/app/moz.build

@@ -52,11 +52,17 @@ JS_PREFERENCE_PP_FILES += [
      'mobile.js',
 ]
 
-# Equivalent to JS_PREFERENCE_PP_FILES[CONFIG['ANDROID_CPU_ARCH']],
-# which isn't supported out of the box.
-FINAL_TARGET_PP_FILES.defaults.pref[CONFIG['ANDROID_CPU_ARCH']] += [
-     'geckoview-prefs.js',
-]
+if not CONFIG['MOZ_ANDROID_FAT_AAR_ARCHITECTURES']:
+    # Equivalent to JS_PREFERENCE_PP_FILES[CONFIG['ANDROID_CPU_ARCH']],
+    # which isn't supported out of the box.
+    FINAL_TARGET_PP_FILES.defaults.pref[CONFIG['ANDROID_CPU_ARCH']] += [
+        'geckoview-prefs.js',
+    ]
+else:
+    for arch in CONFIG['MOZ_ANDROID_FAT_AAR_ARCHITECTURES']:
+        FINAL_TARGET_FILES.defaults.pref[arch] += [
+             '!/dist/fat-aar/output/defaults/pref/{arch}/geckoview-prefs.js'.format(arch=arch),
+        ]
 
 FINAL_TARGET_PP_FILES += [
   'ua-update.json.in',

mobile/android/gradle/with_gecko_binaries.gradle

@@ -74,7 +74,12 @@ ext.configureVariantWithGeckoBinaries = { variant ->
 
         android.sourceSets."${variant.name}".assets.srcDir syncOmnijarFromDistDir.destinationDir
         android.sourceSets."${variant.name}".assets.srcDir syncAssetsFromDistDir.destinationDir
-        android.sourceSets."${variant.name}".jniLibs.srcDir syncLibsFromDistDir.destinationDir
+
+        if (!mozconfig.substs.MOZ_ANDROID_FAT_AAR_ARCHITECTURES) {
+            android.sourceSets."${variant.name}".jniLibs.srcDir syncLibsFromDistDir.destinationDir
+        } else {
+            android.sourceSets."${variant.name}".jniLibs.srcDir "${topobjdir}/dist/fat-aar/output/jni"
+        }
     }
 }
 

mobile/android/installer/Makefile.in

@@ -57,6 +57,10 @@ DEFINES += -DMOZ_GECKOVIEW_JAR=1
 MOZ_PKG_DIR = geckoview
 endif
 
+ifdef MOZ_ANDROID_FAT_AAR_ARCHITECTURES
+DEFINES += -DMOZ_ANDROID_FAT_AAR_ARCHITECTURES=1
+endif
+
 MOZ_PACKAGER_MINIFY=1
 
 include $(topsrcdir)/toolkit/mozapps/installer/packager.mk

mobile/android/installer/allowed-dupes.mn

@@ -50,3 +50,14 @@ chrome/chrome/searchplugins/bolcom-fy-NL.xml
 chrome/chrome/searchplugins/bolcom-nl.xml
 chrome/chrome/searchplugins/ddg.xml
 chrome/chrome/searchplugins/duckduckgo.xml
+
+#ifdef MOZ_ANDROID_FAT_AAR_ARCHITECTURES
+defaults/pref/arm64-v8a/geckoview-prefs.js
+defaults/pref/armeabi-v7a/geckoview-prefs.js
+defaults/pref/x86/geckoview-prefs.js
+defaults/pref/x86_64/geckoview-prefs.js
+arm64-v8a/greprefs.js
+armeabi-v7a/greprefs.js
+x86/greprefs.js
+x86_64/greprefs.js
+#endif # MOZ_ANDROID_FAT_AAR_ARCHITECTURES

mobile/android/installer/package-manifest.in

@@ -158,13 +158,19 @@
 ; [Default Preferences]
 ; All the pref files must be part of base to prevent migration bugs
 #ifdef MOZ_GECKOVIEW_JAR
+#ifndef MOZ_ANDROID_FAT_AAR_ARCHITECTURES
+@BINPATH@/@ANDROID_CPU_ARCH@/greprefs.js
 @BINPATH@/@PREF_DIR@/@ANDROID_CPU_ARCH@/geckoview-prefs.js
 #else
+@BINPATH@/*/greprefs.js
+@BINPATH@/@PREF_DIR@/*/geckoview-prefs.js
+#endif # !MOZ_ANDROID_FAT_AAR_ARCHITECTURES
+#else
+@BINPATH@/@ANDROID_CPU_ARCH@/greprefs.js
 @BINPATH@/@PREF_DIR@/mobile.js
-#endif
+#endif # MOZ_GECKOVIEW_JAR
 @BINPATH@/@PREF_DIR@/channel-prefs.js
 @BINPATH@/ua-update.json
-@BINPATH@/@ANDROID_CPU_ARCH@/greprefs.js
 @BINPATH@/defaults/autoconfig/prefcalls.js
 
 ; [Layout Engine Resources]

mobile/android/moz.configure

@@ -188,3 +188,10 @@ def check_android_gcm(android_gcm,
         if not google_play_services:
             die('You must specify --with-google-play-services when'
                 ' building with MOZ_ANDROID_GCM=1')
+
+# Automation will set this via the TC environment.
+option(env='MOZ_ANDROID_FAT_AAR_ARCHITECTURES',
+       nargs='*', choices=('armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64'),
+       help='Comma-separated list of Android CPU architectures like "armeabi-v7a,arm64-v8a,x86,x86_64"')
+
+set_config('MOZ_ANDROID_FAT_AAR_ARCHITECTURES', depends('MOZ_ANDROID_FAT_AAR_ARCHITECTURES')(lambda x: x))

modules/libpref/moz.build

@@ -49,9 +49,17 @@ DEFINES['MOZ_WIDGET_TOOLKIT'] = CONFIG['MOZ_WIDGET_TOOLKIT']
 if CONFIG['MOZ_ENABLE_WEBRENDER']:
     DEFINES['MOZ_ENABLE_WEBRENDER'] = True
 
-grepref_location = FINAL_TARGET_PP_FILES
 if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'android':
-    grepref_location = grepref_location[CONFIG['ANDROID_CPU_ARCH']]
-grepref_location += [
-    'greprefs.js',
-]
+    if not CONFIG['MOZ_ANDROID_FAT_AAR_ARCHITECTURES']:
+        FINAL_TARGET_PP_FILES[CONFIG['ANDROID_CPU_ARCH']] += [
+            'greprefs.js',
+        ]
+    else:
+        for arch in CONFIG['MOZ_ANDROID_FAT_AAR_ARCHITECTURES']:
+            FINAL_TARGET_FILES[arch] += [
+                 '!/dist/fat-aar/output/{arch}/greprefs.js'.format(arch=arch),
+            ]
+else:
+    FINAL_TARGET_PP_FILES += [
+        'greprefs.js',
+    ]

python/mozbuild/mozbuild/action/fat_aar.py

@@ -0,0 +1,203 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Fetch and unpack architecture-specific Maven zips, verify cross-architecture
+compatibility, and ready inputs to an Android multi-architecture fat AAR build.
+'''
+
+from __future__ import absolute_import, unicode_literals, print_function
+
+import argparse
+import buildconfig
+import subprocess
+import sys
+
+from collections import (
+    defaultdict,
+    OrderedDict,
+)
+from hashlib import sha1  # We don't need a strong hash to compare inputs.
+from zipfile import ZipFile
+from io import BytesIO
+
+from mozpack.copier import FileCopier
+from mozpack.files import JarFinder
+from mozpack.mozjar import JarReader
+from mozpack.packager.unpack import UnpackFinder
+import mozpack.path as mozpath
+
+
+def _download_zips(distdir, architectures):
+    # The mapping from Android CPU architecture to TC job is defined here, and the TC index
+    # lookup is mediated by python/mozbuild/mozbuild/artifacts.py and
+    # python/mozbuild/mozbuild/artifact_builds.py.
+    jobs = {
+        'arm64-v8a': 'android-aarch64-opt',
+        'armeabi-v7a': 'android-api-16-opt',
+        'x86': 'android-x86-opt',
+        'x86_64': 'android-x86_64-opt',
+    }
+
+    for arch in architectures:
+        # It's unfortunate that we must couple tightly, but that's the current API for
+        # dispatching.  In automation, MOZ_ARTIFACT_TASK* environment variables will ensure
+        # that the correct tasks are chosen as install sources.
+        subprocess.check_call([sys.executable, mozpath.join(buildconfig.topsrcdir, 'mach'),
+                               'artifact', 'install',
+                               '--job', jobs[arch],
+                               '--distdir', mozpath.join(distdir, 'input', arch),
+                               '--no-tests', '--no-process', '--maven-zip'])
+
+
+def fat_aar(distdir, architectures=[],
+            no_download=False, no_process=False, no_compatibility_check=False,
+            rewrite_old_archives=False):
+    if not no_download:
+        _download_zips(distdir, architectures)
+    else:
+        print('Not downloading architecture-specific artifact Maven zips.')
+
+    if no_process:
+        print('Not processing architecture-specific artifact Maven zips.')
+        return 0
+
+    # Map {filename: {fingerprint: [arch1, arch2, ...]}}.
+    diffs = defaultdict(lambda: defaultdict(list))
+    missing_arch_prefs = set()
+    # Collect multi-architecture inputs to the fat AAR.
+    copier = FileCopier()
+
+    for arch in architectures:
+        # Map old non-architecture-specific path to new architecture-specific path.
+        old_rewrite_map = {
+            'greprefs.js': '{}/greprefs.js'.format(arch),
+            'defaults/pref/geckoview-prefs.js': 'defaults/pref/{}/geckoview-prefs.js'.format(arch),
+        }
+
+        # Architecture-specific preferences files.
+        arch_prefs = set(old_rewrite_map.values())
+        missing_arch_prefs |= set(arch_prefs)
+
+        path = mozpath.join(distdir, 'input', arch, 'target.maven.zip')
+
+        aars = list(JarFinder(path, JarReader(path)).find('**/geckoview-*.aar'))
+        if len(aars) != 1:
+            raise ValueError('Maven zip "{path}" with more than one candidate AAR found: {aars}'
+                             .format(path=path, aars=tuple(sorted(p for p, _ in aars))))
+
+        [aar_path, aar_file] = aars[0]
+
+        jar_finder = JarFinder(aar_file.file.filename, JarReader(fileobj=aar_file.open()))
+        for path, fileobj in UnpackFinder(jar_finder):
+            # Native libraries go straight through.
+            if mozpath.match(path, 'jni/**'):
+                copier.add(path, fileobj)
+
+            elif path in arch_prefs:
+                copier.add(path, fileobj)
+
+            elif rewrite_old_archives and path in old_rewrite_map:
+                # Ease testing during transition by allowing old omnijars that don't have
+                # architecture-specific files yet.
+                new_path = old_rewrite_map[path]
+                print('Rewrote old path "{path}" to new path "{new_path}"'.format(
+                    path=path, new_path=new_path))
+                copier.add(new_path, fileobj)
+
+            elif path in ('classes.jar', 'annotations.zip'):
+                # annotations.zip differs due to timestamps, but the contents should not.
+
+                # `JarReader` fails on the non-standard `classes.jar` produced by Gradle/aapt,
+                # and it's not worth working around, so we use Python's zip functionality
+                # instead.
+                z = ZipFile(BytesIO(fileobj.open().read()))
+                for r in z.namelist():
+                    fingerprint = sha1(z.open(r).read()).hexdigest()
+                    diffs['{}!/{}'.format(path, r)][fingerprint].append(arch)
+
+            else:
+                fingerprint = sha1(fileobj.open().read()).hexdigest()
+                # There's no need to distinguish `target.maven.zip` from `assets/omni.ja` here,
+                # since in practice they will never overlap.
+                diffs[path][fingerprint].append(arch)
+
+            missing_arch_prefs.discard(path)
+
+    # Some differences are allowed across the architecture-specific AARs.  We could allow-list
+    # the actual content, but it's not necessary right now.
+    allow_list = {
+        'AndroidManifest.xml',  # Min SDK version is different for 32- and 64-bit builds.
+        'classes.jar!/org/mozilla/gecko/util/HardwareUtils.class',  # Min SDK as well.
+        'classes.jar!/org/mozilla/geckoview/BuildConfig.class',
+        # Each input captures its CPU architecture.
+        'chrome/toolkit/content/global/buildconfig.html',
+    }
+
+    not_allowed = OrderedDict()
+
+    def format_diffs(ds):
+        # Like '  armeabi-v7a, arm64-v8a -> XXX\n  x86, x86_64 -> YYY'.
+        return '\n'.join(sorted(
+            '  {archs} -> {fingerprint}'.format(archs=', '.join(sorted(archs)),
+                                                fingerprint=fingerprint)
+            for fingerprint, archs in ds.iteritems()))
+
+    for p, ds in sorted(diffs.iteritems()):
+        if len(ds) <= 1:
+            # Only one hash across all inputs: roll on.
+            continue
+
+        if p in allow_list:
+            print('Allowed: Path "{path}" has architecture-specific versions:\n{ds_repr}'.format(
+                path=p, ds_repr=format_diffs(ds)))
+            continue
+
+        not_allowed[p] = ds
+
+    for p, ds in not_allowed.iteritems():
+        print('Disallowed: Path "{path}" has architecture-specific versions:\n{ds_repr}'.format(
+                 path=p, ds_repr=format_diffs(ds)))
+
+    for missing in sorted(missing_arch_prefs):
+        print('Disallowed: Inputs missing expected architecture-specific input: {missing}'.format(
+            missing=missing))
+
+    if not no_compatibility_check and (missing_arch_prefs or not_allowed):
+        return 1
+
+    copier.copy(mozpath.join(distdir, 'output'))
+
+    return 0
+
+
+def main(argv):
+    description = '''Fetch and unpack architecture-specific Maven zips, verify cross-architecture
+compatibility, and ready inputs to an Android multi-architecture fat AAR build.'''
+
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('--no-download', action='store_true',
+                        help='Do not fetch Maven zips.')
+    parser.add_argument('--no-process', action='store_true',
+                        help='Do not process Maven zips.')
+    parser.add_argument('--no-compatibility-check', action='store_true',
+                        help='Do not fail if Maven zips are not compatible.')
+    parser.add_argument('--rewrite-old-archives', action='store_true',
+                        help='Rewrite Maven zips containing omnijars that do not contain '
+                             'architecture-specific preference files.')
+    parser.add_argument('--distdir', required=True)
+    parser.add_argument('architectures', nargs='+',
+                        choices=('armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64'))
+
+    args = parser.parse_args(argv)
+
+    return fat_aar(
+        args.distdir, architectures=args.architectures,
+        no_download=args.no_download, no_process=args.no_process,
+        no_compatibility_check=args.no_compatibility_check,
+        rewrite_old_archives=args.rewrite_old_archives)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

taskcluster/ci/build/android.yml

@@ -1238,3 +1238,70 @@ android-x86_64-gcp/debug:
         - linux64-sccache
         - linux64-nasm
         - linux64-node
+
+android-geckoview-fat-aar/opt:
+    description: "Android GeckoView multi-architecture fat AAR Opt"
+    index:
+        product: mobile
+        job-name: android-geckoview-fat-aar-opt
+    treeherder:
+        platform: android-4-0-geckoview-fat-aar/opt
+        symbol: Bgv
+    worker-type: b-linux
+    dependencies:
+        android-x86-opt: build-android-x86/opt
+        android-x86_64-opt: build-android-x86_64/opt
+        android-api-16-opt: build-android-api-16/opt
+        android-aarch64-opt: build-android-aarch64/opt
+    worker:
+        docker-image: {in-tree: android-build}
+        max-run-time: 7200
+        env:
+            # Online in order to download the per-architecture AARs.
+            GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online"
+            TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"
+            PERFHERDER_EXTRA_OPTIONS: android-geckoview-fat-aar-opt
+            MOZ_ANDROID_FAT_AAR_ARCHITECTURES: 'armeabi-v7a,arm64-v8a,x86,x86_64'
+            USE_ARTIFACT: '1'
+            MOZ_ARTIFACT_TASK: {task-reference: '<android-api-16-opt>'}
+            MOZ_ARTIFACT_TASK_ANDROID_API_16_OPT: {task-reference: '<android-api-16-opt>'}
+            MOZ_ARTIFACT_TASK_ANDROID_AARCH64_OPT: {task-reference: '<android-aarch64-opt>'}
+            MOZ_ARTIFACT_TASK_ANDROID_X86_OPT: {task-reference: '<android-x86-opt>'}
+            MOZ_ARTIFACT_TASK_ANDROID_X86_64_OPT: {task-reference: '<android-x86_64-opt>'}
+        artifacts:
+            - name: public/android/maven
+              # TODO Bug 1433198. Remove the following entry once target.maven.zip is uploaded to a maven repository
+              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/
+              type: directory
+            - name: public/build/target.maven.zip
+              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/target.maven.zip
+              type: file
+            - name: public/build/geckoview-androidTest.apk
+              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk
+              type: file
+            - name: public/build/geckoview_example.apk
+              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk
+              type: file
+            - name: public/build
+              path: /builds/worker/artifacts/
+              type: directory
+    run:
+        using: mozharness
+        actions: [get-secrets, build]
+        config:
+            - builds/releng_base_android_64_builds.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: api-16
+        tooltool-downloads: internal
+    toolchains:
+        - android-gradle-dependencies
+        - android-ndk-linux
+        - android-sdk-linux
+        - linux64-clang
+        - linux64-rust-android
+        - linux64-rust-size
+        - linux64-cbindgen
+        - linux64-sccache
+        - linux64-nasm
+        - linux64-node