Bug 1264836 - Automatically convert the little-endian ICU data file for big-endian builds. r=firefox-build-system-reviewers,rstewart

Differential Revision: https://phabricator.services.mozilla.com/D83857
2020-07-17 20:06:42 +00:00 · 2020-07-17 20:06:42 +00:00 · f1ffe803dd
--- a/config/external/icu/common/moz.build
+++ b/config/external/icu/common/moz.build
@ -23,3 +23,11 @@ if CONFIG['CC_TYPE'] in ('clang', 'gcc'):

 include('../defs.mozbuild')
 include('sources.mozbuild')
+
+SOURCES += sources
+
+if CONFIG['TARGET_ENDIANNESS'] == 'big':
+    HostLibrary('host_icuuc')
+    HOST_DEFINES['U_COMMON_IMPLEMENTATION'] = True
+    HOST_SOURCES += sources
+    HOST_SOURCES += other_sources
--- a/config/external/icu/common/sources.mozbuild
+++ b/config/external/icu/common/sources.mozbuild
@ -1,5 +1,5 @@
 # THIS FILE IS GENERATED BY /intl/icu_sources_data.py DO NOT EDIT
-SOURCES += [
+sources = [
   '/intl/icu/source/common/appendable.cpp',
   '/intl/icu/source/common/bmpset.cpp',
   '/intl/icu/source/common/brkeng.cpp',
@ -170,7 +170,35 @@ SOURCES += [
   '/intl/icu/source/common/uvectr64.cpp',
   '/intl/icu/source/common/wintz.cpp',
 ]
-
+other_sources = [
+   '/intl/icu/source/common/bytestrieiterator.cpp',
+   '/intl/icu/source/common/cstr.cpp',
+   '/intl/icu/source/common/cwchar.cpp',
+   '/intl/icu/source/common/icudataver.cpp',
+   '/intl/icu/source/common/icuplug.cpp',
+   '/intl/icu/source/common/pluralmap.cpp',
+   '/intl/icu/source/common/propsvec.cpp',
+   '/intl/icu/source/common/resbund_cnv.cpp',
+   '/intl/icu/source/common/ubiditransform.cpp',
+   '/intl/icu/source/common/ucat.cpp',
+   '/intl/icu/source/common/ucnv2022.cpp',
+   '/intl/icu/source/common/ucnv_ct.cpp',
+   '/intl/icu/source/common/ucnv_ext.cpp',
+   '/intl/icu/source/common/ucnv_lmb.cpp',
+   '/intl/icu/source/common/ucnv_set.cpp',
+   '/intl/icu/source/common/ucnvdisp.cpp',
+   '/intl/icu/source/common/ucnvhz.cpp',
+   '/intl/icu/source/common/ucnvisci.cpp',
+   '/intl/icu/source/common/ucnvmbcs.cpp',
+   '/intl/icu/source/common/ucnvsel.cpp',
+   '/intl/icu/source/common/uidna.cpp',
+   '/intl/icu/source/common/unorm.cpp',
+   '/intl/icu/source/common/ures_cnv.cpp',
+   '/intl/icu/source/common/usc_impl.cpp',
+   '/intl/icu/source/common/ushape.cpp',
+   '/intl/icu/source/common/ustr_wcs.cpp',
+   '/intl/icu/source/common/util_props.cpp',
+]
 EXPORTS.unicode += [
   '/intl/icu/source/common/unicode/appendable.h',
   '/intl/icu/source/common/unicode/brkiter.h',
--- a/config/external/icu/data/convert_icudata.py
+++ b/config/external/icu/data/convert_icudata.py
@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import subprocess
+import buildconfig
+
+
+def main(output, data_file):
+    output.close()
+    subprocess.run([
+        os.path.join(buildconfig.topobjdir, 'dist', 'host', 'bin', 'icupkg'),
+        '-tb',
+        data_file,
+        output.name,
+    ])
--- a/config/external/icu/data/moz.build
+++ b/config/external/icu/data/moz.build
@ -14,7 +14,16 @@ prefix = ''
 if (CONFIG['OS_ARCH'] == 'WINNT' and CONFIG['CPU_ARCH'] == 'x86') or CONFIG['OS_ARCH'] == 'Darwin':
    prefix = '_'

-DEFINES['ICU_DATA_FILE'] = '"%s/icudt%sl.dat"' % (SRCDIR, CONFIG['MOZ_ICU_VERSION'])
+data_file = {
+    'little': 'icudt%sl.dat' % CONFIG['MOZ_ICU_VERSION'],
+    'big': 'icudt%sb.dat' % CONFIG['MOZ_ICU_VERSION'],
+}
+data_dir = {
+    'little': SRCDIR,
+    'big': OBJDIR,
+}
+DEFINES['ICU_DATA_FILE'] = '"%s/%s"' % (data_dir[CONFIG['TARGET_ENDIANNESS']],
+                                        data_file[CONFIG['TARGET_ENDIANNESS']])
 DEFINES['ICU_DATA_SYMBOL'] = '%sicudt%s_dat' % (prefix, CONFIG['MOZ_ICU_VERSION'])
 SOURCES += [
    'icu_data.S',
@ -22,3 +31,8 @@ SOURCES += [

 if CONFIG['OS_ARCH'] == 'WINNT' and CONFIG['CC_TYPE'] == 'clang-cl':
    USE_INTEGRATED_CLANGCL_AS = True
+
+if CONFIG['TARGET_ENDIANNESS'] == 'big':
+    GeneratedFile(data_file['big'],
+                  script='convert_icudata.py',
+                  inputs=[data_file['little']])
--- a/config/external/icu/defs.mozbuild
+++ b/config/external/icu/defs.mozbuild
@ -56,3 +56,7 @@ if CONFIG['CC_TYPE'] == 'clang-cl':
        '-Wno-macro-redefined',
        '-Wno-microsoft-include',
    ]
+
+for k, v in DEFINES.items():
+    if k != 'UCONFIG_NO_LEGACY_CONVERSION':
+        HOST_DEFINES[k] = v
--- a/config/external/icu/i18n/moz.build
+++ b/config/external/icu/i18n/moz.build
@ -13,3 +13,11 @@ LOCAL_INCLUDES += ['/intl/icu/source/common']

 include('../defs.mozbuild')
 include('sources.mozbuild')
+
+SOURCES += sources
+
+if CONFIG['TARGET_ENDIANNESS'] == 'big':
+    HostLibrary('host_icui18n')
+    HOST_DEFINES['U_I18N_IMPLEMENTATION'] = True
+    HOST_SOURCES += sources
+    HOST_SOURCES += other_sources
--- a/config/external/icu/i18n/sources.mozbuild
+++ b/config/external/icu/i18n/sources.mozbuild
@ -1,5 +1,5 @@
 # THIS FILE IS GENERATED BY /intl/icu_sources_data.py DO NOT EDIT
-SOURCES += [
+sources = [
   '/intl/icu/source/i18n/astro.cpp',
   '/intl/icu/source/i18n/basictz.cpp',
   '/intl/icu/source/i18n/bocsu.cpp',
@ -178,7 +178,64 @@ SOURCES += [
   '/intl/icu/source/i18n/wintzimpl.cpp',
   '/intl/icu/source/i18n/zonemeta.cpp',
 ]
-
+other_sources = [
+   '/intl/icu/source/i18n/alphaindex.cpp',
+   '/intl/icu/source/i18n/anytrans.cpp',
+   '/intl/icu/source/i18n/brktrans.cpp',
+   '/intl/icu/source/i18n/casetrn.cpp',
+   '/intl/icu/source/i18n/cpdtrans.cpp',
+   '/intl/icu/source/i18n/csdetect.cpp',
+   '/intl/icu/source/i18n/csmatch.cpp',
+   '/intl/icu/source/i18n/csr2022.cpp',
+   '/intl/icu/source/i18n/csrecog.cpp',
+   '/intl/icu/source/i18n/csrmbcs.cpp',
+   '/intl/icu/source/i18n/csrsbcs.cpp',
+   '/intl/icu/source/i18n/csrucode.cpp',
+   '/intl/icu/source/i18n/csrutf8.cpp',
+   '/intl/icu/source/i18n/esctrn.cpp',
+   '/intl/icu/source/i18n/fmtable_cnv.cpp',
+   '/intl/icu/source/i18n/funcrepl.cpp',
+   '/intl/icu/source/i18n/gender.cpp',
+   '/intl/icu/source/i18n/inputext.cpp',
+   '/intl/icu/source/i18n/name2uni.cpp',
+   '/intl/icu/source/i18n/nortrans.cpp',
+   '/intl/icu/source/i18n/nultrans.cpp',
+   '/intl/icu/source/i18n/quant.cpp',
+   '/intl/icu/source/i18n/rbt.cpp',
+   '/intl/icu/source/i18n/rbt_data.cpp',
+   '/intl/icu/source/i18n/rbt_pars.cpp',
+   '/intl/icu/source/i18n/rbt_rule.cpp',
+   '/intl/icu/source/i18n/rbt_set.cpp',
+   '/intl/icu/source/i18n/regexcmp.cpp',
+   '/intl/icu/source/i18n/regeximp.cpp',
+   '/intl/icu/source/i18n/regexst.cpp',
+   '/intl/icu/source/i18n/regextxt.cpp',
+   '/intl/icu/source/i18n/rematch.cpp',
+   '/intl/icu/source/i18n/remtrans.cpp',
+   '/intl/icu/source/i18n/repattrn.cpp',
+   '/intl/icu/source/i18n/scientificnumberformatter.cpp',
+   '/intl/icu/source/i18n/strmatch.cpp',
+   '/intl/icu/source/i18n/strrepl.cpp',
+   '/intl/icu/source/i18n/titletrn.cpp',
+   '/intl/icu/source/i18n/tolowtrn.cpp',
+   '/intl/icu/source/i18n/toupptrn.cpp',
+   '/intl/icu/source/i18n/translit.cpp',
+   '/intl/icu/source/i18n/transreg.cpp',
+   '/intl/icu/source/i18n/tridpars.cpp',
+   '/intl/icu/source/i18n/ucsdet.cpp',
+   '/intl/icu/source/i18n/ulocdata.cpp',
+   '/intl/icu/source/i18n/unesctrn.cpp',
+   '/intl/icu/source/i18n/uni2name.cpp',
+   '/intl/icu/source/i18n/uregex.cpp',
+   '/intl/icu/source/i18n/uregexc.cpp',
+   '/intl/icu/source/i18n/uregion.cpp',
+   '/intl/icu/source/i18n/uspoof_build.cpp',
+   '/intl/icu/source/i18n/uspoof_conf.cpp',
+   '/intl/icu/source/i18n/utrans.cpp',
+   '/intl/icu/source/i18n/vzone.cpp',
+   '/intl/icu/source/i18n/zrule.cpp',
+   '/intl/icu/source/i18n/ztrans.cpp',
+]
 EXPORTS.unicode += [
   '/intl/icu/source/i18n/unicode/alphaindex.h',
   '/intl/icu/source/i18n/unicode/basictz.h',
--- a/config/external/icu/icupkg/moz.build
+++ b/config/external/icu/icupkg/moz.build
@ -0,0 +1,27 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+HostProgram('icupkg')
+
+LOCAL_INCLUDES += [
+    '/intl/icu/source/common',
+    '/intl/icu/source/i18n',
+    '/intl/icu/source/tools/toolutil',
+]
+
+include('../defs.mozbuild')
+include('sources.mozbuild')
+
+HOST_SOURCES += sources
+HOST_SOURCES += [
+    '/intl/icu/source/stubdata/stubdata.cpp',
+]
+
+HOST_USE_LIBS += [
+    'host_icui18n',
+    'host_icutoolutil',
+    'host_icuuc',
+]
--- a/config/external/icu/icupkg/sources.mozbuild
+++ b/config/external/icu/icupkg/sources.mozbuild
@ -0,0 +1,4 @@
+# THIS FILE IS GENERATED BY /intl/icu_sources_data.py DO NOT EDIT
+sources = [
+   '/intl/icu/source/tools/icupkg/icupkg.cpp',
+]
--- a/config/external/icu/moz.build
+++ b/config/external/icu/moz.build
@ -14,4 +14,9 @@ else:
        'data',
        'i18n',
    ]
+    if CONFIG['TARGET_ENDIANNESS'] == 'big':
+        DIRS += [
+            'toolutil',
+            'icupkg',
+        ]
    USE_LIBS += ['icudata']
--- a/config/external/icu/toolutil/moz.build
+++ b/config/external/icu/toolutil/moz.build
@ -0,0 +1,20 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+HostLibrary('host_icutoolutil')
+FINAL_LIBRARY = 'icu'
+
+DEFINES['U_TOOLUTIL_IMPLEMENTATION'] = True
+
+LOCAL_INCLUDES += [
+    '/intl/icu/source/common',
+    '/intl/icu/source/i18n',
+]
+
+include('../defs.mozbuild')
+include('sources.mozbuild')
+
+HOST_SOURCES += sources
--- a/config/external/icu/toolutil/sources.mozbuild
+++ b/config/external/icu/toolutil/sources.mozbuild
@ -0,0 +1,27 @@
+# THIS FILE IS GENERATED BY /intl/icu_sources_data.py DO NOT EDIT
+sources = [
+   '/intl/icu/source/tools/toolutil/collationinfo.cpp',
+   '/intl/icu/source/tools/toolutil/dbgutil.cpp',
+   '/intl/icu/source/tools/toolutil/denseranges.cpp',
+   '/intl/icu/source/tools/toolutil/filestrm.cpp',
+   '/intl/icu/source/tools/toolutil/filetools.cpp',
+   '/intl/icu/source/tools/toolutil/flagparser.cpp',
+   '/intl/icu/source/tools/toolutil/package.cpp',
+   '/intl/icu/source/tools/toolutil/pkg_genc.cpp',
+   '/intl/icu/source/tools/toolutil/pkg_gencmn.cpp',
+   '/intl/icu/source/tools/toolutil/pkg_icu.cpp',
+   '/intl/icu/source/tools/toolutil/pkgitems.cpp',
+   '/intl/icu/source/tools/toolutil/ppucd.cpp',
+   '/intl/icu/source/tools/toolutil/swapimpl.cpp',
+   '/intl/icu/source/tools/toolutil/toolutil.cpp',
+   '/intl/icu/source/tools/toolutil/ucbuf.cpp',
+   '/intl/icu/source/tools/toolutil/ucln_tu.cpp',
+   '/intl/icu/source/tools/toolutil/ucm.cpp',
+   '/intl/icu/source/tools/toolutil/ucmstate.cpp',
+   '/intl/icu/source/tools/toolutil/udbgutil.cpp',
+   '/intl/icu/source/tools/toolutil/unewdata.cpp',
+   '/intl/icu/source/tools/toolutil/uoptions.cpp',
+   '/intl/icu/source/tools/toolutil/uparse.cpp',
+   '/intl/icu/source/tools/toolutil/writesrc.cpp',
+   '/intl/icu/source/tools/toolutil/xmlparser.cpp',
+]
--- a/config/recurse.mk
+++ b/config/recurse.mk
@ -181,6 +181,11 @@ toolkit/components/telemetry/export: layout/style/ServoCSSPropList.py
 # currently have a good way of expressing this dependency.
 toolkit/components/updateagent/target: toolkit/mozapps/update/common/target

+ifeq ($(TARGET_ENDIANNESS),big)
+config/external/icu/data/target-objects: config/external/icu/data/icudt$(MOZ_ICU_VERSION)b.dat
+config/external/icu/data/icudt$(MOZ_ICU_VERSION)b.dat: config/external/icu/icupkg/host
+endif
+
 ifdef ENABLE_CLANG_PLUGIN
 # Only target rules use the clang plugin.
 $(filter %/target %/target-objects,$(filter-out config/export config/host build/unix/stdc++compat/% build/clang-plugin/%,$(compile_targets))): build/clang-plugin/host build/clang-plugin/tests/target-objects
--- a/intl/icu_sources_data.py
+++ b/intl/icu_sources_data.py
@ -158,27 +158,33 @@ def list_headers(path):
 def write_sources(mozbuild, sources, headers):
    with open(mozbuild, 'w', newline='\n', encoding='utf-8') as f:
        f.write('# THIS FILE IS GENERATED BY /intl/icu_sources_data.py ' +
-                'DO NOT EDIT\n' +
-                'SOURCES += [\n')
-        f.write(''.join("   '/%s',\n" % s for s in sources))
-        f.write(']\n\n')
-        f.write('EXPORTS.unicode += [\n')
-        f.write(''.join("   '/%s',\n" % s for s in headers))
-        f.write(']\n')
+                'DO NOT EDIT\n')
+        def write_list(name, content):
+            if content:
+                f.write('%s %s [\n' % (name, '=' if name.islower() else '+='))
+                f.write(''.join("   '/%s',\n" % s for s in content))
+                f.write(']\n')
+
+        write_list('sources', [s for s in sources if s not in UNUSED_SOURCES])
+        write_list('other_sources', [s for s in sources if s in UNUSED_SOURCES])
+        write_list('EXPORTS.unicode', headers)


 def update_sources(topsrcdir):
    print('Updating ICU sources lists...')
-    for d in ['common', 'i18n']:
+    for d in ['common', 'i18n', 'tools/toolutil', 'tools/icupkg']:
        base_path = mozpath.join(topsrcdir, 'intl/icu/source/%s' % d)
        makefile = mozpath.join(base_path, 'Makefile.in')
        mozbuild = mozpath.join(topsrcdir,
-                                'config/external/icu/%s/sources.mozbuild' % d)
+                                'config/external/icu/%s/sources.mozbuild' % mozpath.basename(d))
        sources = [mozpath.relpath(s, topsrcdir)
                   for s in get_sources_from_makefile(makefile)]
-        sources = filter(lambda x: x not in UNUSED_SOURCES, sources)
-        headers = [mozpath.normsep(os.path.relpath(s, topsrcdir))
-                   for s in list_headers(mozpath.join(base_path, 'unicode'))]
+        unicode_dir = mozpath.join(base_path, 'unicode')
+        if os.path.exists(unicode_dir):
+            headers = [mozpath.normsep(os.path.relpath(s, topsrcdir))
+                       for s in list_headers(unicode_dir)]
+        else:
+            headers = None
        write_sources(mozbuild, sources, headers)