diff --git a/python/mozbuild/mozbuild/action/l10n_merge.py b/python/mozbuild/mozbuild/action/l10n_merge.py new file mode 100644 index 000000000000..b7f833676e73 --- /dev/null +++ b/python/mozbuild/mozbuild/action/l10n_merge.py @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import shutil +import sys +import os + +from mozbuild.util import ( + ensureParentDir, +) + + +def main(argv): + parser = argparse.ArgumentParser(description='Merge l10n files.') + parser.add_argument('--output', help='Path to write merged output') + parser.add_argument('--ref-file', help='Path to reference file (en-US)') + parser.add_argument('--l10n-file', help='Path to locale file') + + args = parser.parse_args(argv) + + from compare_locales.compare import ( + ContentComparer, + Observer, + ) + from compare_locales.paths import ( + File, + ) + cc = ContentComparer([Observer()]) + cc.compare(File(args.ref_file, args.ref_file, ''), + File(args.l10n_file, args.l10n_file, ''), + args.output) + + ensureParentDir(args.output) + if not os.path.exists(args.output): + src = args.l10n_file + if not os.path.exists(args.l10n_file): + src = args.ref_file + shutil.copy(src, args.output) + + return 0 + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py index 1096e95d9ce3..e562686c1d53 100644 --- a/python/mozbuild/mozbuild/backend/common.py +++ b/python/mozbuild/mozbuild/backend/common.py @@ -31,6 +31,8 @@ from mozbuild.frontend.data import ( HostGeneratedSources, HostRustLibrary, IPDLCollection, + LocalizedPreprocessedFiles, + LocalizedFiles, RustLibrary, SharedLibrary, StaticLibrary, @@ -372,6 +374,27 @@ class CommonBackend(BuildBackend): self._write_unified_file(unified_file, source_filenames, output_directory, poison_windows_h) + def localized_path(self, relativesrcdir, filename): + '''Return the localized path for a file. + + Given ``relativesrcdir``, a path relative to the topsrcdir, return a path to ``filename`` + from the current locale as specified by ``MOZ_UI_LOCALE``, using ``L10NBASEDIR`` as the + parent directory for non-en-US locales. + ''' + ab_cd = self.environment.substs['MOZ_UI_LOCALE'][0] + l10nbase = mozpath.join(self.environment.substs['L10NBASEDIR'], ab_cd) + # Filenames from LOCALIZED_FILES will start with en-US/. + if filename.startswith('en-US/'): + e, filename = filename.split('en-US/') + assert(not e) + if ab_cd == 'en-US': + return mozpath.join(self.environment.topsrcdir, relativesrcdir, 'en-US', filename) + if mozpath.basename(relativesrcdir) == 'locales': + l10nrelsrcdir = mozpath.dirname(relativesrcdir) + else: + l10nrelsrcdir = relativesrcdir + return mozpath.join(l10nbase, l10nrelsrcdir, filename) + def _consume_jar_manifest(self, obj): # Ideally, this would all be handled somehow in the emitter, but # this would require all the magic surrounding l10n and addons in @@ -379,15 +402,15 @@ class CommonBackend(BuildBackend): # any time soon enough. # Notably missing: # - DEFINES from config/config.mk - # - L10n support # - The equivalent of -e when USE_EXTENSION_MANIFEST is set in # moz.build, but it doesn't matter in dist/bin. pp = Preprocessor() if obj.defines: pp.context.update(obj.defines.defines) pp.context.update(self.environment.defines) + ab_cd = obj.config.substs['MOZ_UI_LOCALE'][0] pp.context.update( - AB_CD='en-US', + AB_CD=ab_cd, BUILD_FASTER=1, ) pp.out = JarManifestParser() @@ -413,6 +436,8 @@ class CommonBackend(BuildBackend): jar_context['DEFINES'] = obj.defines.defines files = jar_context['FINAL_TARGET_FILES'] files_pp = jar_context['FINAL_TARGET_PP_FILES'] + localized_files = jar_context['LOCALIZED_FILES'] + localized_files_pp = jar_context['LOCALIZED_PP_FILES'] for e in jarinfo.entries: if e.is_locale: @@ -429,7 +454,7 @@ class CommonBackend(BuildBackend): if '*' not in e.source and not os.path.exists(src.full_path): if e.is_locale: raise Exception( - '%s: Cannot find %s' % (obj.path, e.source)) + '%s: Cannot find %s (tried %s)' % (obj.path, e.source, src.full_path)) if e.source.startswith('/'): src = Path(jar_context, '!' + e.source) else: @@ -449,15 +474,26 @@ class CommonBackend(BuildBackend): if '*' in e.source: raise Exception('%s: Wildcards are not supported with ' 'preprocessing' % obj.path) - files_pp[path] += [src] + if e.is_locale: + localized_files_pp[path] += [src] + else: + files_pp[path] += [src] else: - files[path] += [src] + if e.is_locale: + localized_files[path] += [src] + else: + files[path] += [src] if files: self.consume_object(FinalTargetFiles(jar_context, files)) if files_pp: self.consume_object( FinalTargetPreprocessedFiles(jar_context, files_pp)) + if localized_files: + self.consume_object(LocalizedFiles(jar_context, localized_files)) + if localized_files_pp: + self.consume_object( + LocalizedPreprocessedFiles(jar_context, localized_files_pp)) for m in jarinfo.chrome_manifests: entry = parse_manifest_line( diff --git a/python/mozbuild/mozbuild/backend/fastermake.py b/python/mozbuild/mozbuild/backend/fastermake.py index b029aa10ff0a..977bade62cdd 100644 --- a/python/mozbuild/mozbuild/backend/fastermake.py +++ b/python/mozbuild/mozbuild/backend/fastermake.py @@ -14,6 +14,8 @@ from mozbuild.frontend.data import ( FinalTargetPreprocessedFiles, FinalTargetFiles, JARManifest, + LocalizedFiles, + LocalizedPreprocessedFiles, XPIDLFile, ) from mozbuild.makeutil import Makefile @@ -31,6 +33,7 @@ class FasterMakeBackend(CommonBackend, PartialBackend): self._install_manifests = OrderedDefaultDict(InstallManifest) self._dependencies = OrderedDefaultDict(list) + self._l10n_dependencies = OrderedDefaultDict(list) self._has_xpidl = False @@ -59,13 +62,29 @@ class FasterMakeBackend(CommonBackend, PartialBackend): elif isinstance(obj, (FinalTargetFiles, FinalTargetPreprocessedFiles)) and \ obj.install_target.startswith('dist/bin'): + ab_cd = self.environment.substs['MOZ_UI_LOCALE'][0] + localized = isinstance(obj, (LocalizedFiles, LocalizedPreprocessedFiles)) defines = obj.defines or {} if defines: defines = defines.defines for path, files in obj.files.walk(): for f in files: + # For localized files we need to find the file from the locale directory. + if (localized and not isinstance(f, ObjDirPath) and ab_cd != 'en-US'): + src = self.localized_path(obj.relsrcdir, f) + + dep_target = 'install-%s' % obj.install_target + + if '*' not in src: + merge = mozpath.abspath(mozpath.join(self.environment.topobjdir, + 'l10n_merge', obj.relsrcdir, f)) + self._l10n_dependencies[dep_target].append((merge, f.full_path, src)) + src = merge + else: + src = f.full_path + if isinstance(obj, FinalTargetPreprocessedFiles): - self._add_preprocess(obj, f.full_path, path, + self._add_preprocess(obj, src, path, target=f.target_basename, defines=defines) elif '*' in f: @@ -73,7 +92,7 @@ class FasterMakeBackend(CommonBackend, PartialBackend): for p in mozpath.split(s): if '*' not in p: yield p + '/' - prefix = ''.join(_prefix(f.full_path)) + prefix = ''.join(_prefix(src)) if '*' in f.target_basename: target = path @@ -83,11 +102,11 @@ class FasterMakeBackend(CommonBackend, PartialBackend): self._install_manifests[obj.install_target] \ .add_pattern_link( prefix, - f.full_path[len(prefix):], + src[len(prefix):], target) else: self._install_manifests[obj.install_target].add_link( - f.full_path, + src, mozpath.join(path, f.target_basename) ) if isinstance(f, ObjDirPath): @@ -151,11 +170,29 @@ class FasterMakeBackend(CommonBackend, PartialBackend): mk.add_statement('INSTALL_MANIFESTS = %s' % ' '.join(self._install_manifests.keys())) - # Add dependencies we infered: + # Add dependencies we inferred: for target, deps in self._dependencies.iteritems(): mk.create_rule([target]).add_dependencies( '$(TOPOBJDIR)/%s' % d for d in deps) + + # This is not great, but it's better to have some dependencies on these Python files. + python_deps = [ + '$(TOPSRCDIR)/python/mozbuild/mozbuild/action/l10n_merge.py', + '$(TOPSRCDIR)/third_party/python/compare-locales/compare_locales/compare.py', + '$(TOPSRCDIR)/third_party/python/compare-locales/compare_locales/paths.py', + ] + # Add l10n dependencies we inferred: + for target, deps in self._l10n_dependencies.iteritems(): + mk.create_rule([target]).add_dependencies( + '%s' % d[0] for d in deps) + for (merge, ref_file, l10n_file) in deps: + rule = mk.create_rule([merge]).add_dependencies( + [ref_file, l10n_file] + python_deps) + rule.add_commands(['$(PYTHON) -m mozbuild.action.l10n_merge --output {} --ref-file {} --l10n-file {}'.format(merge, ref_file, l10n_file)]) + # Add a dummy rule for the l10n file since it might not exist. + mk.create_rule([l10n_file]) + mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk') for base, install_manifest in self._install_manifests.iteritems(): diff --git a/python/mozbuild/mozbuild/test/backend/common.py b/python/mozbuild/mozbuild/test/backend/common.py index b50acc467931..bd6428973ca4 100644 --- a/python/mozbuild/mozbuild/test/backend/common.py +++ b/python/mozbuild/mozbuild/test/backend/common.py @@ -236,6 +236,7 @@ class BackendTester(unittest.TestCase): environment is cleaned up automatically when the test finishes. """ config = CONFIGS[name] + config['substs']['MOZ_UI_LOCALE'] = 'en-US' objdir = mkdtemp() self.addCleanup(rmtree, objdir) diff --git a/python/mozbuild/mozbuild/test/backend/test_build.py b/python/mozbuild/mozbuild/test/backend/test_build.py index 06c7332e2444..b8c06ff7f45b 100644 --- a/python/mozbuild/mozbuild/test/backend/test_build.py +++ b/python/mozbuild/mozbuild/test/backend/test_build.py @@ -26,6 +26,7 @@ from tempfile import mkdtemp BASE_SUBSTS = [ ('PYTHON', mozpath.normsep(sys.executable)), + ('MOZ_UI_LOCALE', 'en-US'), ]