bug 1447932 - Add support for localized files to fastermake backend. r=mshal

Co-authored by Nick Alexander <nalexander@mozilla.com>

This patch adds support to the fastermake backend for taking localized files
from a locale directory as specified by the combination of --with-l10n-base
and --enable-ui-locale. This allows artifact builds to be built localized
with a different locale.

MozReview-Commit-ID: 1bD9Gy0ewJ2

--HG--
extra : rebase_source : 7e7d52142e48c0e57ead11df4d3d721a3b10b3e0

python/mozbuild/mozbuild/action/l10n_merge.py

@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import shutil
+import sys
+import os
+
+from mozbuild.util import (
+    ensureParentDir,
+)
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(description='Merge l10n files.')
+    parser.add_argument('--output', help='Path to write merged output')
+    parser.add_argument('--ref-file', help='Path to reference file (en-US)')
+    parser.add_argument('--l10n-file', help='Path to locale file')
+
+    args = parser.parse_args(argv)
+
+    from compare_locales.compare import (
+        ContentComparer,
+        Observer,
+    )
+    from compare_locales.paths import (
+        File,
+    )
+    cc = ContentComparer([Observer()])
+    cc.compare(File(args.ref_file, args.ref_file, ''),
+               File(args.l10n_file, args.l10n_file, ''),
+               args.output)
+
+    ensureParentDir(args.output)
+    if not os.path.exists(args.output):
+        src = args.l10n_file
+        if not os.path.exists(args.l10n_file):
+            src = args.ref_file
+        shutil.copy(src, args.output)
+
+    return 0
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

python/mozbuild/mozbuild/backend/common.py

@@ -31,6 +31,8 @@ from mozbuild.frontend.data import (
     HostGeneratedSources,
     HostRustLibrary,
     IPDLCollection,
+    LocalizedPreprocessedFiles,
+    LocalizedFiles,
     RustLibrary,
     SharedLibrary,
     StaticLibrary,
@@ -372,6 +374,27 @@ class CommonBackend(BuildBackend):
             self._write_unified_file(unified_file, source_filenames,
                                      output_directory, poison_windows_h)
 
+    def localized_path(self, relativesrcdir, filename):
+        '''Return the localized path for a file.
+
+        Given ``relativesrcdir``, a path relative to the topsrcdir, return a path to ``filename``
+        from the current locale as specified by ``MOZ_UI_LOCALE``, using ``L10NBASEDIR`` as the
+        parent directory for non-en-US locales.
+        '''
+        ab_cd = self.environment.substs['MOZ_UI_LOCALE'][0]
+        l10nbase = mozpath.join(self.environment.substs['L10NBASEDIR'], ab_cd)
+        # Filenames from LOCALIZED_FILES will start with en-US/.
+        if filename.startswith('en-US/'):
+            e, filename = filename.split('en-US/')
+            assert(not e)
+        if ab_cd == 'en-US':
+            return mozpath.join(self.environment.topsrcdir, relativesrcdir, 'en-US', filename)
+        if mozpath.basename(relativesrcdir) == 'locales':
+            l10nrelsrcdir = mozpath.dirname(relativesrcdir)
+        else:
+            l10nrelsrcdir = relativesrcdir
+        return mozpath.join(l10nbase, l10nrelsrcdir, filename)
+
     def _consume_jar_manifest(self, obj):
         # Ideally, this would all be handled somehow in the emitter, but
         # this would require all the magic surrounding l10n and addons in
@@ -379,15 +402,15 @@ class CommonBackend(BuildBackend):
         # any time soon enough.
         # Notably missing:
         # - DEFINES from config/config.mk
-        # - L10n support
         # - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
         #   moz.build, but it doesn't matter in dist/bin.
         pp = Preprocessor()
         if obj.defines:
             pp.context.update(obj.defines.defines)
         pp.context.update(self.environment.defines)
+        ab_cd = obj.config.substs['MOZ_UI_LOCALE'][0]
         pp.context.update(
-            AB_CD='en-US',
+            AB_CD=ab_cd,
             BUILD_FASTER=1,
         )
         pp.out = JarManifestParser()
@@ -413,6 +436,8 @@ class CommonBackend(BuildBackend):
                 jar_context['DEFINES'] = obj.defines.defines
             files = jar_context['FINAL_TARGET_FILES']
             files_pp = jar_context['FINAL_TARGET_PP_FILES']
+            localized_files = jar_context['LOCALIZED_FILES']
+            localized_files_pp = jar_context['LOCALIZED_PP_FILES']
 
             for e in jarinfo.entries:
                 if e.is_locale:
@@ -429,7 +454,7 @@ class CommonBackend(BuildBackend):
                 if '*' not in e.source and not os.path.exists(src.full_path):
                     if e.is_locale:
                         raise Exception(
-                            '%s: Cannot find %s' % (obj.path, e.source))
+                            '%s: Cannot find %s (tried %s)' % (obj.path, e.source, src.full_path))
                     if e.source.startswith('/'):
                         src = Path(jar_context, '!' + e.source)
                     else:
@@ -449,15 +474,26 @@ class CommonBackend(BuildBackend):
                     if '*' in e.source:
                         raise Exception('%s: Wildcards are not supported with '
                                         'preprocessing' % obj.path)
-                    files_pp[path] += [src]
+                    if e.is_locale:
+                        localized_files_pp[path] += [src]
+                    else:
+                        files_pp[path] += [src]
                 else:
-                    files[path] += [src]
+                    if e.is_locale:
+                        localized_files[path] += [src]
+                    else:
+                        files[path] += [src]
 
             if files:
                 self.consume_object(FinalTargetFiles(jar_context, files))
             if files_pp:
                 self.consume_object(
                     FinalTargetPreprocessedFiles(jar_context, files_pp))
+            if localized_files:
+                self.consume_object(LocalizedFiles(jar_context, localized_files))
+            if localized_files_pp:
+                self.consume_object(
+                    LocalizedPreprocessedFiles(jar_context, localized_files_pp))
 
             for m in jarinfo.chrome_manifests:
                 entry = parse_manifest_line(

python/mozbuild/mozbuild/backend/fastermake.py

@@ -14,6 +14,8 @@ from mozbuild.frontend.data import (
     FinalTargetPreprocessedFiles,
     FinalTargetFiles,
     JARManifest,
+    LocalizedFiles,
+    LocalizedPreprocessedFiles,
     XPIDLFile,
 )
 from mozbuild.makeutil import Makefile
@@ -31,6 +33,7 @@ class FasterMakeBackend(CommonBackend, PartialBackend):
         self._install_manifests = OrderedDefaultDict(InstallManifest)
 
         self._dependencies = OrderedDefaultDict(list)
+        self._l10n_dependencies = OrderedDefaultDict(list)
 
         self._has_xpidl = False
 
@@ -59,13 +62,29 @@ class FasterMakeBackend(CommonBackend, PartialBackend):
         elif isinstance(obj, (FinalTargetFiles,
                               FinalTargetPreprocessedFiles)) and \
                 obj.install_target.startswith('dist/bin'):
+            ab_cd = self.environment.substs['MOZ_UI_LOCALE'][0]
+            localized = isinstance(obj, (LocalizedFiles, LocalizedPreprocessedFiles))
             defines = obj.defines or {}
             if defines:
                 defines = defines.defines
             for path, files in obj.files.walk():
                 for f in files:
+                    # For localized files we need to find the file from the locale directory.
+                    if (localized and not isinstance(f, ObjDirPath) and ab_cd != 'en-US'):
+                        src = self.localized_path(obj.relsrcdir, f)
+
+                        dep_target = 'install-%s' % obj.install_target
+
+                        if '*' not in src:
+                            merge = mozpath.abspath(mozpath.join(self.environment.topobjdir,
+                                                                 'l10n_merge', obj.relsrcdir, f))
+                            self._l10n_dependencies[dep_target].append((merge, f.full_path, src))
+                            src = merge
+                    else:
+                        src = f.full_path
+
                     if isinstance(obj, FinalTargetPreprocessedFiles):
-                        self._add_preprocess(obj, f.full_path, path,
+                        self._add_preprocess(obj, src, path,
                                              target=f.target_basename,
                                              defines=defines)
                     elif '*' in f:
@@ -73,7 +92,7 @@ class FasterMakeBackend(CommonBackend, PartialBackend):
                             for p in mozpath.split(s):
                                 if '*' not in p:
                                     yield p + '/'
-                        prefix = ''.join(_prefix(f.full_path))
+                        prefix = ''.join(_prefix(src))
 
                         if '*' in f.target_basename:
                             target = path
@@ -83,11 +102,11 @@ class FasterMakeBackend(CommonBackend, PartialBackend):
                         self._install_manifests[obj.install_target] \
                             .add_pattern_link(
                                 prefix,
-                                f.full_path[len(prefix):],
+                                src[len(prefix):],
                                 target)
                     else:
                         self._install_manifests[obj.install_target].add_link(
-                            f.full_path,
+                            src,
                             mozpath.join(path, f.target_basename)
                         )
                     if isinstance(f, ObjDirPath):
@@ -151,11 +170,29 @@ class FasterMakeBackend(CommonBackend, PartialBackend):
         mk.add_statement('INSTALL_MANIFESTS = %s'
                          % ' '.join(self._install_manifests.keys()))
 
-        # Add dependencies we infered:
+        # Add dependencies we inferred:
         for target, deps in self._dependencies.iteritems():
             mk.create_rule([target]).add_dependencies(
                 '$(TOPOBJDIR)/%s' % d for d in deps)
 
+
+        # This is not great, but it's better to have some dependencies on these Python files.
+        python_deps = [
+            '$(TOPSRCDIR)/python/mozbuild/mozbuild/action/l10n_merge.py',
+            '$(TOPSRCDIR)/third_party/python/compare-locales/compare_locales/compare.py',
+            '$(TOPSRCDIR)/third_party/python/compare-locales/compare_locales/paths.py',
+        ]
+        # Add l10n dependencies we inferred:
+        for target, deps in self._l10n_dependencies.iteritems():
+            mk.create_rule([target]).add_dependencies(
+                '%s' % d[0] for d in deps)
+            for (merge, ref_file, l10n_file) in deps:
+                rule = mk.create_rule([merge]).add_dependencies(
+                    [ref_file, l10n_file] + python_deps)
+                rule.add_commands(['$(PYTHON) -m mozbuild.action.l10n_merge --output {} --ref-file {} --l10n-file {}'.format(merge, ref_file, l10n_file)])
+                # Add a dummy rule for the l10n file since it might not exist.
+                mk.create_rule([l10n_file])
+
         mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
 
         for base, install_manifest in self._install_manifests.iteritems():

python/mozbuild/mozbuild/test/backend/common.py

@@ -236,6 +236,7 @@ class BackendTester(unittest.TestCase):
         environment is cleaned up automatically when the test finishes.
         """
         config = CONFIGS[name]
+        config['substs']['MOZ_UI_LOCALE'] = 'en-US'
 
         objdir = mkdtemp()
         self.addCleanup(rmtree, objdir)

python/mozbuild/mozbuild/test/backend/test_build.py

@@ -26,6 +26,7 @@ from tempfile import mkdtemp
 
 BASE_SUBSTS = [
     ('PYTHON', mozpath.normsep(sys.executable)),
+    ('MOZ_UI_LOCALE', 'en-US'),
 ]