Bug 1260241 - Implement Fennec packaging in Python. r=glandium

A few notes:

* This doesn't accommodate general OMNIJAR_NAME definitions.  The
  current name (assets/omni.ja) is baked into the product in a few
  places, and is very unlikely to change, so we just error out if this
  isn't true.

* This makes the package-manifest.in file authoritative for what goes
  into assets/, libs/, and the APK root.  Previously,
  package-manifest.in wrote into assets/ and libs/ but
  upload-files-APK.mk also had a convoluted DIST_FILES filtering
  process to work through before a file actually made it into the APK.

* This is intentional about repackaging.  It simplifies the repackage
  step rather than trying to make unpackage-then-repackage the same as
  just package.  I pretty much never get repackaging correct the first
  time; this should help.  (I've manually tested it.)

* The ALREADY_SZIPPED during repackaging is subsumed by the previous
  check if UNPACKAGE is set.  The custom linker expects stored, not
  deflated, libraries, so there's some small legwork to accommodate
  that in mozjar.

MozReview-Commit-ID: JvVtIUSX685

--HG--
extra : rebase_source : fd8a9cfe3dc364d23b1065995db599f99e676e38
This commit is contained in:
Nick Alexander 2016-03-28 20:01:07 -07:00
Родитель 45f7d697fc
Коммит 9ab9fa5451
22 изменённых файлов: 262 добавлений и 128 удалений

Просмотреть файл

@ -39,11 +39,13 @@
@BINPATH@/@DLL_PREFIX@nspr4@DLL_SUFFIX@
#endif
@BINPATH@/@DLL_PREFIX@lgpllibs@DLL_SUFFIX@
#ifdef MOZ_OMX_PLUGIN
@BINPATH@/@DLL_PREFIX@omxplugin@DLL_SUFFIX@
@BINPATH@/@DLL_PREFIX@omxplugingb@DLL_SUFFIX@
@BINPATH@/@DLL_PREFIX@omxplugingb235@DLL_SUFFIX@
@BINPATH@/@DLL_PREFIX@omxpluginhc@DLL_SUFFIX@
@BINPATH@/@DLL_PREFIX@omxpluginkk@DLL_SUFFIX@
#endif
@BINPATH@/@DLL_PREFIX@xul@DLL_SUFFIX@
@BINPATH@/@DLL_PREFIX@nssckbi@DLL_SUFFIX@

Просмотреть файл

@ -28,6 +28,7 @@ PYTHON_UNIT_TESTS += [
'mozbuild/dumbmake/test/test_dumbmake.py',
'mozbuild/mozbuild/test/action/test_buildlist.py',
'mozbuild/mozbuild/test/action/test_generate_browsersearch.py',
'mozbuild/mozbuild/test/action/test_package_fennec_apk.py',
'mozbuild/mozbuild/test/backend/test_android_eclipse.py',
'mozbuild/mozbuild/test/backend/test_build.py',
'mozbuild/mozbuild/test/backend/test_configenvironment.py',

Просмотреть файл

@ -0,0 +1,133 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
'''
Script to produce an Android package (.apk) for Fennec.
'''
from __future__ import absolute_import, print_function
import argparse
import buildconfig
import os
import subprocess
import sys
from mozpack.copier import Jarrer
from mozpack.files import (
DeflatedFile,
File,
FileFinder,
)
from mozpack.mozjar import JarReader
import mozpack.path as mozpath
def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None,
lib_dirs=[],
assets_dirs=[],
szip_assets_libs_with=None,
root_files=[],
verbose=False):
jarrer = Jarrer(optimize=False)
# First, take input files. The contents of the later files overwrites the
# content of earlier files.
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' % \
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir, find_executables=False)
for p, f in finder.find('**'):
compress = None # Take default from Jarrer.
if p.endswith('.so'):
# Asset libraries are special.
if szip_assets_libs_with:
# We need to szip libraries before packing. The file
# returned by the finder is not yet opened. When it is
# opened, it will "see" the content updated by szip.
subprocess.check_output([szip_assets_libs_with,
mozpath.join(finder.base, p)])
if f.open().read(4) == 'SeZz':
# We need to store (rather than deflate) szipped libraries
# (even if we don't szip them ourselves).
compress = False
add(mozpath.join('assets', p), f, compress=compress)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir, find_executables=False)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
if classes_dex:
add('classes.dex', File(classes_dex))
return jarrer
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--verbose', '-v', default=False, action='store_true',
help='be verbose')
parser.add_argument('--inputs', nargs='+',
help='Input skeleton AP_ or APK file(s).')
parser.add_argument('-o', '--output',
help='Output APK file.')
parser.add_argument('--omnijar', default=None,
help='Optional omni.ja to pack into APK file.')
parser.add_argument('--classes-dex', default=None,
help='Optional classes.dex to pack into APK file.')
parser.add_argument('--lib-dirs', nargs='*', default=[],
help='Optional lib/ dirs to pack into APK file.')
parser.add_argument('--assets-dirs', nargs='*', default=[],
help='Optional assets/ dirs to pack into APK file.')
parser.add_argument('--szip-assets-libs-with', default=None,
help='IN PLACE szip assets/**/*.so BEFORE packing '
'into APK file using the given szip executable.')
parser.add_argument('--root-files', nargs='*', default=[],
help='Optional files to pack into APK file root.')
args = parser.parse_args(args)
if buildconfig.substs.get('OMNIJAR_NAME') != 'assets/omni.ja':
raise ValueError("Don't know how package Fennec APKs when "
" OMNIJAR_NAME is not 'assets/omni.jar'.")
jarrer = package_fennec_apk(inputs=args.inputs,
omni_ja=args.omnijar,
classes_dex=args.classes_dex,
lib_dirs=args.lib_dirs,
assets_dirs=args.assets_dirs,
szip_assets_libs_with=args.szip_assets_libs_with,
root_files=args.root_files,
verbose=args.verbose)
jarrer.copy(args.output)
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))

Просмотреть файл

@ -0,0 +1 @@
assets/asset.txt

Просмотреть файл

@ -0,0 +1 @@
classes.dex

Двоичный файл не отображается.

Просмотреть файл

@ -0,0 +1 @@
input1/res/res.txt

Просмотреть файл

@ -0,0 +1 @@
input1/resources.arsc

Двоичный файл не отображается.

Просмотреть файл

@ -0,0 +1 @@
input2/assets/asset.txt

Просмотреть файл

@ -0,0 +1 @@
input2/assets/omni.ja

Просмотреть файл

@ -0,0 +1 @@
input2/classes.dex

Просмотреть файл

@ -0,0 +1 @@
input2/lib/lib.txt

Просмотреть файл

@ -0,0 +1 @@
input2/res/res.txt

Просмотреть файл

@ -0,0 +1 @@
input/resources.arsc

Просмотреть файл

@ -0,0 +1 @@
input2/root_file.txt

Просмотреть файл

@ -0,0 +1 @@
lib/lib.txt

Просмотреть файл

@ -0,0 +1 @@
omni.ja

Просмотреть файл

@ -0,0 +1 @@
root_file.txt

Просмотреть файл

@ -0,0 +1,70 @@
# -*- coding: utf-8 -*-
# Any copyright is dedicated to the Public Domain.
# http://creativecommons.org/publicdomain/zero/1.0/
from __future__ import unicode_literals
import os
import unittest
import mozunit
from mozbuild.action.package_fennec_apk import (
package_fennec_apk as package,
)
from mozpack.mozjar import JarReader
import mozpack.path as mozpath
test_data_path = mozpath.abspath(mozpath.dirname(__file__))
test_data_path = mozpath.join(test_data_path, 'data', 'package_fennec_apk')
def data(name):
return os.path.join(test_data_path, name)
class TestPackageFennecAPK(unittest.TestCase):
"""
Unit tests for package_fennec_apk.py.
"""
def test_arguments(self):
# Language repacks take updated resources from an ap_ and pack them
# into an apk. Make sure the second input overrides the first.
jarrer = package(inputs=[],
omni_ja=data('omni.ja'),
classes_dex=data('classes.dex'),
assets_dirs=[data('assets')],
lib_dirs=[data('lib')],
root_files=[data('root_file.txt')])
# omni.ja ends up in assets/omni.ja.
self.assertEquals(jarrer['assets/omni.ja'].open().read().strip(), 'omni.ja')
# Everything else is in place.
for name in ('classes.dex',
'assets/asset.txt',
'lib/lib.txt',
'root_file.txt'):
self.assertEquals(jarrer[name].open().read().strip(), name)
def test_inputs(self):
# Language repacks take updated resources from an ap_ and pack them
# into an apk. In this case, the first input is the original package,
# the second input the update ap_. Make sure the second input
# overrides the first.
jarrer = package(inputs=[data('input2.apk'), data('input1.ap_')])
files1 = JarReader(data('input1.ap_')).entries.keys()
files2 = JarReader(data('input2.apk')).entries.keys()
for name in files2:
self.assertTrue(name in files1 or
jarrer[name].open().read().startswith('input2/'))
for name in files1:
self.assertTrue(jarrer[name].open().read().startswith('input1/'))
if __name__ == '__main__':
mozunit.main()

Просмотреть файл

@ -109,7 +109,6 @@ endif
endif
endif
repackage-zip: UNPACKAGE='$(ZIP_IN)'
repackage-zip: ALREADY_SZIPPED=1
repackage-zip: libs-$(AB_CD)
# call a hook for apps to put their uninstall helper.exe into the package
$(UNINSTALLER_PACKAGE_HOOK)

Просмотреть файл

@ -8,59 +8,17 @@
include $(MOZILLA_DIR)/config/android-common.mk
DIST_FILES =
# Place the files in the order they are going to be opened by the linker
ifndef MOZ_FOLD_LIBS
DIST_FILES += \
libnspr4.so \
libplc4.so \
libplds4.so \
libmozsqlite3.so \
libnssutil3.so \
$(NULL)
endif
DIST_FILES += libnss3.so
ifndef MOZ_FOLD_LIBS
DIST_FILES += \
libssl3.so \
libsmime3.so \
$(NULL)
endif
DIST_FILES += \
liblgpllibs.so \
libxul.so \
libnssckbi.so \
libfreebl3.so \
libsoftokn3.so \
resources.arsc \
AndroidManifest.xml \
chrome \
components \
defaults \
modules \
hyphenation \
res \
lib \
extensions \
# Files packed into the APK root. Packing files into the APK root is not
# supported by modern Android build systems, including Gradle, so don't add to
# this list without Android peer approval.
ROOT_FILES := \
application.ini \
package-name.txt \
ua-update.json \
platform.ini \
greprefs.js \
browserconfig.properties \
blocklist.xml \
chrome.manifest \
update.locale \
removed-files \
$(NULL)
NON_DIST_FILES = \
classes.dex \
$(NULL)
DIST_FILES += $(MOZ_CHILD_PROCESS_NAME)
GECKO_APP_AP_PATH = $(topobjdir)/mobile/android/base
ifdef ENABLE_TESTS
@ -161,29 +119,6 @@ else
INNER_MAKE_GECKOLIBS_AAR=echo 'Android geckolibs.aar packaging is disabled'
endif # MOZ_ANDROID_GECKOLIBS_AAR
ifdef MOZ_OMX_PLUGIN
DIST_FILES += libomxplugin.so libomxplugingb.so libomxplugingb235.so \
libomxpluginhc.so libomxpluginkk.so
endif
SO_LIBRARIES := $(filter %.so,$(DIST_FILES))
# These libraries are placed in the assets/$(ANDROID_CPU_ARCH) directory by packager.py.
ASSET_SO_LIBRARIES := $(addprefix assets/$(ANDROID_CPU_ARCH)/,$(filter-out libmozglue.so $(MOZ_CHILD_PROCESS_NAME),$(SO_LIBRARIES)))
DIST_FILES := $(filter-out $(SO_LIBRARIES),$(DIST_FILES))
NON_DIST_FILES += libmozglue.so $(MOZ_CHILD_PROCESS_NAME) $(ASSET_SO_LIBRARIES)
ifdef MOZ_ENABLE_SZIP
# These libraries are szipped in-place in the
# assets/$(ANDROID_CPU_ARCH) directory.
SZIP_LIBRARIES := $(ASSET_SO_LIBRARIES)
endif
ifndef COMPILE_ENVIRONMENT
# Any Fennec binary libraries we download are already szipped.
ALREADY_SZIPPED=1
endif
# Fennec's OMNIJAR_NAME can include a directory; for example, it might
# be "assets/omni.ja". This path specifies where the omni.ja file
# lives in the APK, but should not root the resources it contains
@ -202,67 +137,46 @@ OMNIJAR_NAME := $(notdir $(OMNIJAR_NAME))
# insert additional resources (translated strings) into the ap_
# without the build system's participation. This can do the wrong
# thing if there are resource changes in between build time and
# package time. We try to prevent mismatched resources by erroring
# out if the compiled resource IDs are not the same as the resource
# IDs being packaged. If we're doing a single locale repack, however,
# we don't have a complete object directory, so we can't compare
# resource IDs.
# A note on the res/ directory. We unzip the ap_ during packaging,
# which produces the res/ directory. This directory is then included
# in the final package. When we unpack (during locale repacks), we
# need to remove the res/ directory because these resources confuse
# the l10n packaging script that updates omni.ja: the script tries to
# localize the contents of the res/ directory, which fails. Instead,
# after the l10n packaging script completes, we build the ap_
# described above (which includes freshly localized Android resources)
# and the res/ directory is taken from the ap_ as part of the regular
# packaging.
# package time.
PKG_SUFFIX = .apk
INNER_SZIP_LIBRARIES = \
$(if $(ALREADY_SZIPPED),,$(foreach lib,$(SZIP_LIBRARIES),host/bin/szip $(MOZ_SZIP_FLAGS) $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH)/$(lib) && )) true
INNER_FENNEC_PACKAGE = \
$(MAKE) -C $(GECKO_APP_AP_PATH) gecko-nodeps.ap_ && \
$(PYTHON) -m mozbuild.action.package_fennec_apk \
--verbose \
--inputs \
$(GECKO_APP_AP_PATH)/gecko-nodeps.ap_ \
--omnijar $(STAGEPATH)$(MOZ_PKG_DIR)/$(OMNIJAR_NAME) \
--classes-dex $(GECKO_APP_AP_PATH)/classes.dex \
--lib-dirs $(STAGEPATH)$(MOZ_PKG_DIR)/lib \
--assets-dirs $(STAGEPATH)$(MOZ_PKG_DIR)/assets \
$(if $(COMPILE_ENVIRONMENT),$(if $(MOZ_ENABLE_SZIP),--szip-assets-libs-with $(ABS_DIST)/host/bin/szip)) \
--root-files $(foreach f,$(ROOT_FILES),$(STAGEPATH)$(MOZ_PKG_DIR)/$(f)) \
--output $(PACKAGE:.apk=-unsigned-unaligned.apk) && \
$(call RELEASE_SIGN_ANDROID_APK,$(PACKAGE:.apk=-unsigned-unaligned.apk),$(PACKAGE))
ifdef MOZ_BUILD_MOBILE_ANDROID_WITH_GRADLE
INNER_CHECK_R_TXT=echo 'No R.txt checking for you!'
else
INNER_CHECK_R_TXT=\
((test ! -f $(GECKO_APP_AP_PATH)/R.txt && echo "*** Warning: The R.txt that is being packaged might not agree with the R.txt that was built. This is normal during l10n repacks.") || \
diff $(GECKO_APP_AP_PATH)/R.txt $(GECKO_APP_AP_PATH)/gecko-nodeps/R.txt >/dev/null || \
(echo "*** Error: The R.txt that was built and the R.txt that is being packaged are not the same. Rebuild mobile/android/base and re-package." && exit 1))
endif
# Insert $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH)/classes.dex into
# $(ABS_DIST)/gecko.ap_, producing $(ABS_DIST)/gecko.apk.
INNER_MAKE_APK = \
( cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && \
unzip -o $(ABS_DIST)/gecko.ap_ && \
rm $(ABS_DIST)/gecko.ap_ && \
$(ZIP) -r9D $(ABS_DIST)/gecko.ap_ assets && \
$(ZIP) $(if $(ALREADY_SZIPPED),-0 ,$(if $(MOZ_ENABLE_SZIP),-0 ))$(ABS_DIST)/gecko.ap_ $(ASSET_SO_LIBRARIES) && \
$(ZIP) -r9D $(ABS_DIST)/gecko.ap_ $(DIST_FILES) -x $(NON_DIST_FILES) $(SZIP_LIBRARIES) && \
$(if $(filter-out ./,$(OMNIJAR_DIR)), \
mkdir -p $(OMNIJAR_DIR) && mv $(OMNIJAR_NAME) $(OMNIJAR_DIR) && ) \
$(ZIP) -0 $(ABS_DIST)/gecko.ap_ $(OMNIJAR_DIR)$(OMNIJAR_NAME)) && \
rm -f $(ABS_DIST)/gecko.apk && \
cp $(ABS_DIST)/gecko.ap_ $(ABS_DIST)/gecko.apk && \
$(ZIP) -j0 $(ABS_DIST)/gecko.apk $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH)/classes.dex && \
$(RELEASE_JARSIGNER) $(ABS_DIST)/gecko.apk && \
$(ZIPALIGN) -f -v 4 $(ABS_DIST)/gecko.apk $(PACKAGE)
ifeq ($(MOZ_BUILD_APP),mobile/android)
INNER_MAKE_PACKAGE = \
$(INNER_SZIP_LIBRARIES) && \
make -C $(GECKO_APP_AP_PATH) gecko-nodeps.ap_ && \
cp $(GECKO_APP_AP_PATH)/gecko-nodeps.ap_ $(ABS_DIST)/gecko.ap_ && \
$(INNER_CHECK_R_TXT) && \
$(INNER_MAKE_APK) && \
# Packaging produces many optional artifacts.
package_fennec = \
$(INNER_FENNEC_PACKAGE) && \
$(INNER_ROBOCOP_PACKAGE) && \
$(INNER_INSTALL_BOUNCER_PACKAGE) && \
$(INNER_MAKE_GECKOLIBS_AAR) && \
$(INNER_MAKE_GECKOVIEW_LIBRARY)
endif
# Re-packaging only replaces Android resources and the omnijar before
# (re-)signing.
repackage_fennec = \
$(MAKE) -C $(GECKO_APP_AP_PATH) gecko-nodeps.ap_ && \
$(PYTHON) -m mozbuild.action.package_fennec_apk \
--verbose \
--inputs \
$(UNPACKAGE) \
$(GECKO_APP_AP_PATH)/gecko-nodeps.ap_ \
--omnijar $(STAGEPATH)$(MOZ_PKG_DIR)/$(OMNIJAR_NAME) \
--output $(PACKAGE:.apk=-unsigned-unaligned.apk) && \
$(call RELEASE_SIGN_ANDROID_APK,$(PACKAGE:.apk=-unsigned-unaligned.apk),$(PACKAGE))
INNER_MAKE_PACKAGE = $(if $(UNPACKAGE),$(repackage_fennec),$(package_fennec))
# Language repacks root the resources contained in assets/omni.ja
# under assets/, but the repacks expect them to be rooted at /.
@ -270,10 +184,11 @@ endif
# under the root here, in INNER_UNMAKE_PACKAGE. See comments about
# OMNIJAR_NAME earlier in this file and in configure.in.
INNER_UNMAKE_PACKAGE = \
INNER_UNMAKE_PACKAGE = \
mkdir $(MOZ_PKG_DIR) && \
( cd $(MOZ_PKG_DIR) && \
$(UNZIP) $(UNPACKAGE) && \
rm -rf res \
$(UNZIP) $(UNPACKAGE) $(ROOT_FILES) && \
$(UNZIP) $(UNPACKAGE) $(OMNIJAR_DIR)$(OMNIJAR_NAME) && \
$(if $(filter-out ./,$(OMNIJAR_DIR)), \
&& mv $(OMNIJAR_DIR)$(OMNIJAR_NAME) $(OMNIJAR_NAME)) )
mv $(OMNIJAR_DIR)$(OMNIJAR_NAME) $(OMNIJAR_NAME), \
true) )