Bug 1492070, update l10n python libraries, r=stas

update fluent.syntax to 0.9.0
update compare-locales to 5.0.2
update fluent.migrate to https://hg.mozilla.org/l10n/fluent-migration/rev/3877312dc1f4
This includes migration-specific fixes for bugs 1491591, 1491833, 1491859, 1496127

All of these changes have been reviewed before in their respective upstream
repositories.

Differential Revision: https://phabricator.services.mozilla.com/D11861

--HG--
extra : moz-landing-system : lando

third_party/python/compare-locales/compare_locales/__init__.py

@@ -1 +1 @@
-version = "3.3.0"
+version = "5.0.2"

third_party/python/compare-locales/compare_locales/checks/android.py

@@ -6,13 +6,14 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import re
+from xml.dom import minidom
 
 from .base import Checker
 from ..parser.android import textContent
 
 
 class AndroidChecker(Checker):
-    pattern = re.compile('.*/strings.*\\.xml$')
+    pattern = re.compile('(.*/)?strings.*\\.xml$')
 
     def check(self, refEnt, l10nEnt):
         '''Given the reference and localized Entities, performs checks.
@@ -33,14 +34,15 @@ class AndroidChecker(Checker):
         if refNode.nodeName != "string":
             yield ("warning", 0, "Unsupported resource type", "android")
             return
-        for report_tuple in self.check_string([refNode], l10nNode):
+        for report_tuple in self.check_string([refNode], l10nEnt):
             yield report_tuple
 
-    def check_string(self, refs, l10n):
+    def check_string(self, refs, l10nEnt):
         '''Check a single string literal against a list of references.
 
         There should be multiple nodes given for <plurals> or <string-array>.
         '''
+        l10n = l10nEnt.node
         if self.not_translatable(l10n, *refs):
             yield (
                 "error",
@@ -49,10 +51,45 @@ class AndroidChecker(Checker):
                 "android"
             )
             return
-        l10nstring = textContent(l10n)
-        for report_tuple in check_apostrophes(l10nstring):
+        if self.no_at_string(l10n):
+            yield (
+                "error",
+                0,
+                "strings must be translatable",
+                "android"
+            )
+            return
+        if self.no_at_string(*refs):
+            yield (
+                "warning",
+                0,
+                "strings must be translatable",
+                "android"
+            )
+        if self.non_simple_data(l10n):
+            yield (
+                "error",
+                0,
+                "Only plain text allowed, "
+                "or one CDATA surrounded by whitespace",
+                "android"
+            )
+            return
+        for report_tuple in check_apostrophes(l10nEnt.val):
             yield report_tuple
 
+        params, errors = get_params(refs)
+        for error, pos in errors:
+            yield (
+                "warning",
+                pos,
+                error,
+                "android"
+            )
+        if params:
+            for report_tuple in check_params(params, l10nEnt.val):
+                yield report_tuple
+
     def not_translatable(self, *nodes):
         return any(
             node.hasAttribute("translatable")
@@ -60,6 +97,44 @@ class AndroidChecker(Checker):
             for node in nodes
         )
 
+    def no_at_string(self, *ref_nodes):
+        '''Android allows to reference other strings by using
+        @string/identifier
+        instead of the actual value. Those references don't belong into
+        a localizable file, warn on that.
+        '''
+        return any(
+            textContent(node).startswith('@string/')
+            for node in ref_nodes
+        )
+
+    def non_simple_data(self, node):
+        '''Only allow single text nodes, or, a single CDATA node
+        surrounded by whitespace.
+        '''
+        cdata = [
+            child
+            for child in node.childNodes
+            if child.nodeType == minidom.Node.CDATA_SECTION_NODE
+        ]
+        if len(cdata) == 0:
+            if node.childNodes.length == 0:
+                # empty translation is OK
+                return False
+            if node.childNodes.length != 1:
+                return True
+            return node.childNodes[0].nodeType != minidom.Node.TEXT_NODE
+        if len(cdata) > 1:
+            return True
+        for child in node.childNodes:
+            if child == cdata[0]:
+                continue
+            if child.nodeType != minidom.Node.TEXT_NODE:
+                return True
+            if child.data.strip() != "":
+                return True
+        return False
+
 
 silencer = re.compile(r'\\.|""')
 
@@ -98,3 +173,77 @@ def check_apostrophes(string):
                 "Apostrophe must be escaped",
                 "android"
             )
+
+
+def get_params(refs):
+    '''Get printf parameters and internal errors.
+
+    Returns a sparse map of positions to formatter, and a list
+    of errors. Errors covered so far are mismatching formatters.
+    '''
+    params = {}
+    errors = []
+    next_implicit = 1
+    for ref in refs:
+        if isinstance(ref, minidom.Node):
+            ref = textContent(ref)
+        for m in re.finditer(r'%(?P<order>[1-9]\$)?(?P<format>[sSd])', ref):
+            order = m.group('order')
+            if order:
+                order = int(order[0])
+            else:
+                order = next_implicit
+                next_implicit += 1
+            fmt = m.group('format')
+            if order not in params:
+                params[order] = fmt
+            else:
+                # check for consistency errors
+                if params[order] == fmt:
+                    continue
+                msg = "Conflicting formatting, %{order}${f1} vs %{order}${f2}"
+                errors.append((
+                    msg.format(order=order, f1=fmt, f2=params[order]),
+                    m.start()
+                ))
+    return params, errors
+
+
+def check_params(params, string):
+    '''Compare the printf parameters in the given string to the reference
+    parameters.
+
+    Also yields errors that are internal to the parameters inside string,
+    as found by `get_params`.
+    '''
+    lparams, errors = get_params([string])
+    for error, pos in errors:
+        yield (
+            "error",
+            pos,
+            error,
+            "android"
+        )
+    # Compare reference for each localized parameter.
+    # If there's no reference found, error, as an out-of-bounds
+    # parameter crashes.
+    # This assumes that all parameters are actually used in the reference,
+    # which should be OK.
+    # If there's a mismatch in the formatter, error.
+    for order in sorted(lparams):
+        if order not in params:
+            yield (
+                "error",
+                0,
+                "Formatter %{}${} not found in reference".format(
+                    order, lparams[order]
+                ),
+                "android"
+            )
+        elif params[order] != lparams[order]:
+            yield (
+                "error",
+                0,
+                "Mismatching formatter",
+                "android"
+            )

third_party/python/compare-locales/compare_locales/checks/dtd.py

@@ -42,7 +42,7 @@ class DTDChecker(Checker):
     def known_entities(self, refValue):
         if self.__known_entities is None and self.reference is not None:
             self.__known_entities = set()
-            for ent in self.reference:
+            for ent in self.reference.values():
                 self.__known_entities.update(
                     self.entities_for_value(ent.raw_val))
         return self.__known_entities if self.__known_entities is not None \

third_party/python/compare-locales/compare_locales/commands.py

@@ -8,23 +8,26 @@ from __future__ import absolute_import
 from __future__ import print_function
 import logging
 from argparse import ArgumentParser
+from json import dump as json_dump
 import os
+import sys
 
+from compare_locales import mozpath
 from compare_locales import version
 from compare_locales.paths import EnumerateApp, TOMLParser, ConfigNotFound
-from compare_locales.compare import compareProjects, Observer
+from compare_locales.compare import compareProjects
 
 
 class CompareLocales(object):
     """Check the localization status of gecko applications.
-The first arguments are paths to the l10n.ini or toml files for the
+The first arguments are paths to the l10n.toml or ini files for the
 applications, followed by the base directory of the localization repositories.
 Then you pass in the list of locale codes you want to compare. If there are
-not locales given, the list of locales will be taken from the l10n.toml file
+no locales given, the list of locales will be taken from the l10n.toml file
 or the all-locales file referenced by the application\'s l10n.ini."""
 
     def __init__(self):
-        self.parser = None
+        self.parser = self.get_parser()
 
     def get_parser(self):
         """Get an ArgumentParser, with class docstring as description.
@@ -36,8 +39,8 @@ or the all-locales file referenced by the application\'s l10n.ini."""
                             default=0, help='Make more noise')
         parser.add_argument('-q', '--quiet', action='count',
                             default=0, help='''Show less data.
-Specified once, doesn't record entities. Specified twice, also drops
-missing and obsolete files. Specify thrice to hide errors and warnings and
+Specified once, don't show obsolete entities. Specified twice, also hide
+missing entities. Specify thrice to exclude warnings and four times to
 just show stats''')
         parser.add_argument('--validate', action='store_true',
                             help='Run compare-locales against reference')
@@ -51,13 +54,15 @@ use {ab_CD} to specify a different directory for each locale''')
         parser.add_argument('locales', nargs='*', metavar='locale-code',
                             help='Locale code and top-level directory of '
                                  'each localization')
+        parser.add_argument('--json',
+                            help='''Serialize to JSON. Value is the name of
+the output file, pass "-" to serialize to stdout and hide the default output.
+''')
         parser.add_argument('-D', action='append', metavar='var=value',
                             default=[], dest='defines',
                             help='Overwrite variables in TOML files')
-        parser.add_argument('--unified', action="store_true",
-                            help="Show output for all projects unified")
         parser.add_argument('--full', action="store_true",
-                            help="Compare projects that are disabled")
+                            help="Compare sub-projects that are disabled")
         parser.add_argument('--return-zero', action="store_true",
                             help="Return 0 regardless of l10n status")
         parser.add_argument('--clobber-merge', action="store_true",
@@ -67,14 +72,6 @@ Use this option with care. If specified, the merge directory will
 be clobbered for each module. That means, the subdirectory will
 be completely removed, any files that were there are lost.
 Be careful to specify the right merge directory when using this option.""")
-        parser.add_argument('--data', choices=['text', 'exhibit', 'json'],
-                            default='text',
-                            help='''Choose data and format (one of text,
-exhibit, json); text: (default) Show which files miss which strings, together
-with warnings and errors. Also prints a summary; json: Serialize the internal
-tree, useful for tools. Also always succeeds; exhibit: Serialize the summary
-data in a json useful for Exhibit
-''')
         return parser
 
     @classmethod
@@ -84,38 +81,107 @@ data in a json useful for Exhibit
         subclasses.
         """
         cmd = cls()
-        return cmd.handle_()
+        args = cmd.parser.parse_args()
+        return cmd.handle(**vars(args))
 
-    def handle_(self):
-        """The instance part of the classmethod call."""
-        self.parser = self.get_parser()
-        args = self.parser.parse_args()
+    def handle(
+        self,
+        quiet=0, verbose=0,
+        validate=False,
+        merge=None,
+        config_paths=[], l10n_base_dir=None, locales=[],
+        defines=[],
+        full=False,
+        return_zero=False,
+        clobber=False,
+        json=None,
+    ):
+        """The instance part of the classmethod call.
+
+        Using keyword arguments as that is what we need for mach
+        commands in mozilla-central.
+        """
         # log as verbose or quiet as we want, warn by default
-        logging_level = logging.WARNING - (args.verbose - args.quiet) * 10
+        logging_level = logging.WARNING - (verbose - quiet) * 10
         logging.basicConfig()
         logging.getLogger().setLevel(logging_level)
-        kwargs = vars(args)
-        # strip handled arguments
-        kwargs.pop('verbose')
-        return_zero = kwargs.pop('return_zero')
-        rv = self.handle(**kwargs)
-        if return_zero:
-            rv = 0
+
+        config_paths, l10n_base_dir, locales = self.extract_positionals(
+            validate=validate,
+            config_paths=config_paths,
+            l10n_base_dir=l10n_base_dir,
+            locales=locales,
+        )
+
+        # when we compare disabled projects, we set our locales
+        # on all subconfigs, so deep is True.
+        locales_deep = full
+        configs = []
+        config_env = {
+            'l10n_base': l10n_base_dir
+        }
+        for define in defines:
+            var, _, value = define.partition('=')
+            config_env[var] = value
+        for config_path in config_paths:
+            if config_path.endswith('.toml'):
+                try:
+                    config = TOMLParser().parse(config_path, env=config_env)
+                except ConfigNotFound as e:
+                    self.parser.exit('config file %s not found' % e.filename)
+                if locales:
+                    config.set_locales(locales, deep=locales_deep)
+                configs.append(config)
+            else:
+                app = EnumerateApp(
+                    config_path, l10n_base_dir, locales)
+                configs.append(app.asConfig())
+        try:
+            observers = compareProjects(
+                configs,
+                l10n_base_dir,
+                quiet=quiet,
+                merge_stage=merge, clobber_merge=clobber)
+        except (OSError, IOError) as exc:
+            print("FAIL: " + str(exc))
+            self.parser.exit(2)
+
+        if json is None or json != '-':
+            details = observers.serializeDetails()
+            if details:
+                print(details)
+            if len(configs) > 1:
+                if details:
+                    print('')
+                print("Summaries for")
+                for config_path in config_paths:
+                    print("  " + config_path)
+                print("    and the union of these, counting each string once")
+            print(observers.serializeSummaries())
+        if json is not None:
+            data = [observer.toJSON() for observer in observers]
+            stdout = json == '-'
+            indent = 1 if stdout else None
+            fh = sys.stdout if stdout else open(json, 'w')
+            json_dump(data, fh, sort_keys=True, indent=indent)
+            if stdout:
+                fh.write('\n')
+            fh.close()
+        rv = 1 if not return_zero and observers.error else 0
         return rv
 
-    def handle(self, config_paths, l10n_base_dir, locales,
-               merge=None, defines=None, unified=False, full=False, quiet=0,
-               validate=False,
-               clobber=False, data='text'):
+    def extract_positionals(
+        self,
+        validate=False,
+        config_paths=[], l10n_base_dir=None, locales=[],
+    ):
         # using nargs multiple times in argparser totally screws things
         # up, repair that.
         # First files are configs, then the base dir, everything else is
         # locales
         all_args = config_paths + [l10n_base_dir] + locales
         config_paths = []
-        locales = []
-        if defines is None:
-            defines = []
+        # The first directory is our l10n base, split there.
         while all_args and not os.path.isdir(all_args[0]):
             config_paths.append(all_args.pop(0))
         if not config_paths:
@@ -125,60 +191,11 @@ data in a json useful for Exhibit
                 self.parser.error('config file %s not found' % cf)
         if not all_args:
             self.parser.error('l10n-base-dir not found')
-        l10n_base_dir = all_args.pop(0)
+        l10n_base_dir = mozpath.abspath(all_args.pop(0))
         if validate:
             # signal validation mode by setting locale list to [None]
             locales = [None]
         else:
-            locales.extend(all_args)
-        # when we compare disabled projects, we set our locales
-        # on all subconfigs, so deep is True.
-        locales_deep = full
-        configs = []
-        config_env = {}
-        for define in defines:
-            var, _, value = define.partition('=')
-            config_env[var] = value
-        for config_path in config_paths:
-            if config_path.endswith('.toml'):
-                try:
-                    config = TOMLParser.parse(config_path, env=config_env)
-                except ConfigNotFound as e:
-                    self.parser.exit('config file %s not found' % e.filename)
-                config.add_global_environment(l10n_base=l10n_base_dir)
-                if locales:
-                    config.set_locales(locales, deep=locales_deep)
-                configs.append(config)
-            else:
-                app = EnumerateApp(
-                    config_path, l10n_base_dir, locales)
-                configs.append(app.asConfig())
-        try:
-            unified_observer = None
-            if unified:
-                unified_observer = Observer(quiet=quiet)
-            observers = compareProjects(
-                configs,
-                quiet=quiet,
-                stat_observer=unified_observer,
-                merge_stage=merge, clobber_merge=clobber)
-        except (OSError, IOError) as exc:
-            print("FAIL: " + str(exc))
-            self.parser.exit(2)
-        if unified:
-            observers = [unified_observer]
+            locales = all_args
 
-        rv = 0
-        for observer in observers:
-            print(observer.serialize(type=data))
-            # summary is a dict of lang-summary dicts
-            # find out if any of our results has errors, return 1 if so
-            if rv > 0:
-                continue  # we already have errors
-            for loc, summary in observer.summary.items():
-                if summary.get('errors', 0) > 0:
-                    rv = 1
-                    # no need to check further summaries, but
-                    # continue to run through observers
-                    break
-        return rv
+        return config_paths, l10n_base_dir, locales

third_party/python/compare-locales/compare_locales/compare.py

@@ -1,724 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-'Mozilla l10n compare locales tool'
-
-from __future__ import absolute_import
-from __future__ import print_function
-import codecs
-import os
-import shutil
-import re
-from collections import defaultdict
-import six
-from six.moves import map
-from six.moves import zip
-
-from json import dumps
-
-from compare_locales import parser
-from compare_locales import paths, mozpath
-from compare_locales.checks import getChecker
-
-
-class Tree(object):
-    def __init__(self, valuetype):
-        self.branches = dict()
-        self.valuetype = valuetype
-        self.value = None
-
-    def __getitem__(self, leaf):
-        parts = []
-        if isinstance(leaf, paths.File):
-            parts = [] if not leaf.locale else [leaf.locale]
-            if leaf.module:
-                parts += leaf.module.split('/')
-            parts += leaf.file.split('/')
-        else:
-            parts = leaf.split('/')
-        return self.__get(parts)
-
-    def __get(self, parts):
-        common = None
-        old = None
-        new = tuple(parts)
-        t = self
-        for k, v in six.iteritems(self.branches):
-            for i, part in enumerate(zip(k, parts)):
-                if part[0] != part[1]:
-                    i -= 1
-                    break
-            if i < 0:
-                continue
-            i += 1
-            common = tuple(k[:i])
-            old = tuple(k[i:])
-            new = tuple(parts[i:])
-            break
-        if old:
-            self.branches.pop(k)
-            t = Tree(self.valuetype)
-            t.branches[old] = v
-            self.branches[common] = t
-        elif common:
-            t = self.branches[common]
-        if new:
-            if common:
-                return t.__get(new)
-            t2 = t
-            t = Tree(self.valuetype)
-            t2.branches[new] = t
-        if t.value is None:
-            t.value = t.valuetype()
-        return t.value
-
-    indent = '  '
-
-    def getContent(self, depth=0):
-        '''
-        Returns iterator of (depth, flag, key_or_value) tuples.
-        If flag is 'value', key_or_value is a value object, otherwise
-        (flag is 'key') it's a key string.
-        '''
-        keys = sorted(self.branches.keys())
-        if self.value is not None:
-            yield (depth, 'value', self.value)
-        for key in keys:
-            yield (depth, 'key', key)
-            for child in self.branches[key].getContent(depth + 1):
-                yield child
-
-    def toJSON(self):
-        '''
-        Returns this Tree as a JSON-able tree of hashes.
-        Only the values need to take care that they're JSON-able.
-        '''
-        if self.value is not None:
-            return self.value
-        return dict(('/'.join(key), self.branches[key].toJSON())
-                    for key in self.branches.keys())
-
-    def getStrRows(self):
-        def tostr(t):
-            if t[1] == 'key':
-                return self.indent * t[0] + '/'.join(t[2])
-            return self.indent * (t[0] + 1) + str(t[2])
-
-        return [tostr(c) for c in self.getContent()]
-
-    def __str__(self):
-        return '\n'.join(self.getStrRows())
-
-
-class AddRemove(object):
-    def __init__(self):
-        self.left = self.right = None
-
-    def set_left(self, left):
-        if not isinstance(left, list):
-            left = list(l for l in left)
-        self.left = left
-
-    def set_right(self, right):
-        if not isinstance(right, list):
-            right = list(l for l in right)
-        self.right = right
-
-    def __iter__(self):
-        # order_map stores index in left and then index in right
-        order_map = dict((item, (i, -1)) for i, item in enumerate(self.left))
-        left_items = set(order_map)
-        # as we go through the right side, keep track of which left
-        # item we had in right last, and for items not in left,
-        # set the sortmap to (left_offset, right_index)
-        left_offset = -1
-        right_items = set()
-        for i, item in enumerate(self.right):
-            right_items.add(item)
-            if item in order_map:
-                left_offset = order_map[item][0]
-            else:
-                order_map[item] = (left_offset, i)
-        for item in sorted(order_map, key=lambda item: order_map[item]):
-            if item in left_items and item in right_items:
-                yield ('equal', item)
-            elif item in left_items:
-                yield ('delete', item)
-            else:
-                yield ('add', item)
-
-
-class Observer(object):
-
-    def __init__(self, quiet=0, filter=None, file_stats=False):
-        '''Create Observer
-        For quiet=1, skip per-entity missing and obsolete strings,
-        for quiet=2, skip missing and obsolete files. For quiet=3,
-        skip warnings and errors.
-        '''
-        self.summary = defaultdict(lambda: defaultdict(int))
-        self.details = Tree(list)
-        self.quiet = quiet
-        self.filter = filter
-        self.file_stats = None
-        if file_stats:
-            self.file_stats = defaultdict(lambda: defaultdict(dict))
-
-    # support pickling
-    def __getstate__(self):
-        state = dict(summary=self._dictify(self.summary), details=self.details)
-        if self.file_stats is not None:
-            state['file_stats'] = self._dictify(self.file_stats)
-        return state
-
-    def __setstate__(self, state):
-        self.summary = defaultdict(lambda: defaultdict(int))
-        if 'summary' in state:
-            for loc, stats in six.iteritems(state['summary']):
-                self.summary[loc].update(stats)
-        self.file_stats = None
-        if 'file_stats' in state:
-            self.file_stats = defaultdict(lambda: defaultdict(dict))
-            for k, d in six.iteritems(state['file_stats']):
-                self.file_stats[k].update(d)
-        self.details = state['details']
-        self.filter = None
-
-    def _dictify(self, d):
-        plaindict = {}
-        for k, v in six.iteritems(d):
-            plaindict[k] = dict(v)
-        return plaindict
-
-    def toJSON(self):
-        # Don't export file stats, even if we collected them.
-        # Those are not part of the data we use toJSON for.
-        return {
-            'summary': self._dictify(self.summary),
-            'details': self.details.toJSON()
-        }
-
-    def updateStats(self, file, stats):
-        # in multi-project scenarios, this file might not be ours,
-        # check that.
-        # Pass in a dummy entity key '' to avoid getting in to
-        # generic file filters. If we have stats for those,
-        # we want to aggregate the counts
-        if (self.filter is not None and
-                self.filter(file, entity='') == 'ignore'):
-            return
-        for category, value in six.iteritems(stats):
-            self.summary[file.locale][category] += value
-        if self.file_stats is None:
-            return
-        if 'missingInFiles' in stats:
-            # keep track of how many strings are in a missing file
-            # we got the {'missingFile': 'error'} from the notify pass
-            self.details[file].append({'count': stats['missingInFiles']})
-            # missingInFiles should just be "missing" in file stats
-            self.file_stats[file.locale][file.localpath]['missing'] = \
-                stats['missingInFiles']
-            return  # there are no other stats for missing files
-        self.file_stats[file.locale][file.localpath].update(stats)
-
-    def notify(self, category, file, data):
-        rv = 'error'
-        if category in ['missingFile', 'obsoleteFile']:
-            if self.filter is not None:
-                rv = self.filter(file)
-            if rv != "ignore" and self.quiet < 2:
-                self.details[file].append({category: rv})
-            return rv
-        if category in ['missingEntity', 'obsoleteEntity']:
-            if self.filter is not None:
-                rv = self.filter(file, data)
-            if rv == "ignore":
-                return rv
-            if self.quiet < 1:
-                self.details[file].append({category: data})
-            return rv
-        if category in ('error', 'warning') and self.quiet < 3:
-            self.details[file].append({category: data})
-            self.summary[file.locale][category + 's'] += 1
-        return rv
-
-    def toExhibit(self):
-        items = []
-        for locale in sorted(six.iterkeys(self.summary)):
-            summary = self.summary[locale]
-            if locale is not None:
-                item = {'id': 'xxx/' + locale,
-                        'label': locale,
-                        'locale': locale}
-            else:
-                item = {'id': 'xxx',
-                        'label': 'xxx',
-                        'locale': 'xxx'}
-            item['type'] = 'Build'
-            total = sum([summary[k]
-                         for k in ('changed', 'unchanged', 'report', 'missing',
-                                   'missingInFiles')
-                         if k in summary])
-            total_w = sum([summary[k]
-                           for k in ('changed_w', 'unchanged_w', 'missing_w')
-                           if k in summary])
-            rate = (('changed' in summary and summary['changed'] * 100) or
-                    0) / total
-            item.update((k, summary.get(k, 0))
-                        for k in ('changed', 'unchanged'))
-            item.update((k, summary[k])
-                        for k in ('report', 'errors', 'warnings')
-                        if k in summary)
-            item['missing'] = summary.get('missing', 0) + \
-                summary.get('missingInFiles', 0)
-            item['completion'] = rate
-            item['total'] = total
-            item.update((k, summary.get(k, 0))
-                        for k in ('changed_w', 'unchanged_w', 'missing_w'))
-            item['total_w'] = total_w
-            result = 'success'
-            if item.get('warnings', 0):
-                result = 'warning'
-            if item.get('errors', 0) or item.get('missing', 0):
-                result = 'failure'
-            item['result'] = result
-            items.append(item)
-        data = {
-            "properties": dict.fromkeys(
-                ("completion", "errors", "warnings", "missing", "report",
-                 "missing_w", "changed_w", "unchanged_w",
-                 "unchanged", "changed", "obsolete"),
-                {"valueType": "number"}),
-            "types": {
-                "Build": {"pluralLabel": "Builds"}
-            }}
-        data['items'] = items
-        return dumps(data, indent=2)
-
-    def serialize(self, type="text"):
-        if type == "exhibit":
-            return self.toExhibit()
-        if type == "json":
-            return dumps(self.toJSON())
-
-        def tostr(t):
-            if t[1] == 'key':
-                return '  ' * t[0] + '/'.join(t[2])
-            o = []
-            indent = '  ' * (t[0] + 1)
-            for item in t[2]:
-                if 'error' in item:
-                    o += [indent + 'ERROR: ' + item['error']]
-                elif 'warning' in item:
-                    o += [indent + 'WARNING: ' + item['warning']]
-                elif 'missingEntity' in item:
-                    o += [indent + '+' + item['missingEntity']]
-                elif 'obsoleteEntity' in item:
-                    o += [indent + '-' + item['obsoleteEntity']]
-                elif 'missingFile' in item:
-                    o.append(indent + '// add and localize this file')
-                elif 'obsoleteFile' in item:
-                    o.append(indent + '// remove this file')
-            return '\n'.join(o)
-
-        out = []
-        for locale, summary in sorted(six.iteritems(self.summary)):
-            if locale is not None:
-                out.append(locale + ':')
-            out += [
-                k + ': ' + str(v) for k, v in sorted(six.iteritems(summary))]
-            total = sum([summary[k]
-                         for k in ['changed', 'unchanged', 'report', 'missing',
-                                   'missingInFiles']
-                         if k in summary])
-            rate = 0
-            if total:
-                rate = (('changed' in summary and summary['changed'] * 100) or
-                        0) / total
-            out.append('%d%% of entries changed' % rate)
-        return '\n'.join([tostr(c) for c in self.details.getContent()] + out)
-
-    def __str__(self):
-        return 'observer'
-
-
-class ContentComparer:
-    keyRE = re.compile('[kK]ey')
-    nl = re.compile('\n', re.M)
-
-    def __init__(self, observers, stat_observers=None):
-        '''Create a ContentComparer.
-        observer is usually a instance of Observer. The return values
-        of the notify method are used to control the handling of missing
-        entities.
-        '''
-        self.observers = observers
-        if stat_observers is None:
-            stat_observers = []
-        self.stat_observers = stat_observers
-
-    def create_merge_dir(self, merge_file):
-        outdir = mozpath.dirname(merge_file)
-        if not os.path.isdir(outdir):
-            os.makedirs(outdir)
-
-    def merge(self, ref_entities, ref_map, ref_file, l10n_file, merge_file,
-              missing, skips, ctx, capabilities, encoding):
-        '''Create localized file in merge dir
-
-        `ref_entities` and `ref_map` are the parser result of the
-        reference file
-        `ref_file` and `l10n_file` are the File objects for the reference and
-        the l10n file, resp.
-        `merge_file` is the output path for the generated content. This is None
-        if we're just comparing or validating.
-        `missing` are the missing messages in l10n - potentially copied from
-        reference
-        `skips` are entries to be dropped from the localized file
-        `ctx` is the parsing context
-        `capabilities` are the capabilities for the merge algorithm
-        `encoding` is the encoding to be used when serializing, usually utf-8
-        '''
-
-        if not merge_file:
-            return
-
-        if capabilities == parser.CAN_NONE:
-            return
-
-        self.create_merge_dir(merge_file)
-
-        if capabilities & parser.CAN_COPY:
-            # copy the l10n file if it's good, or the reference file if not
-            if skips or missing:
-                src = ref_file.fullpath
-            else:
-                src = l10n_file.fullpath
-            shutil.copyfile(src, merge_file)
-            print("copied reference to " + merge_file)
-            return
-
-        if not (capabilities & parser.CAN_SKIP):
-            return
-
-        # Start with None in case the merge file doesn't need to be created.
-        f = None
-
-        if skips:
-            # skips come in ordered by key name, we need them in file order
-            skips.sort(key=lambda s: s.span[0])
-
-            # we need to skip a few erroneous blocks in the input, copy by hand
-            f = codecs.open(merge_file, 'wb', encoding)
-            offset = 0
-            for skip in skips:
-                chunk = skip.span
-                f.write(ctx.contents[offset:chunk[0]])
-                offset = chunk[1]
-            f.write(ctx.contents[offset:])
-
-        if f is None:
-            # l10n file is a good starting point
-            shutil.copyfile(l10n_file.fullpath, merge_file)
-
-        if not (capabilities & parser.CAN_MERGE):
-            if f:
-                f.close()
-            return
-
-        if skips or missing:
-            if f is None:
-                f = codecs.open(merge_file, 'ab', encoding)
-            trailing = (['\n'] +
-                        [ref_entities[ref_map[key]].all for key in missing] +
-                        [ref_entities[ref_map[skip.key]].all for skip in skips
-                         if not isinstance(skip, parser.Junk)])
-
-            def ensureNewline(s):
-                if not s.endswith('\n'):
-                    return s + '\n'
-                return s
-
-            print("adding to " + merge_file)
-            f.write(''.join(map(ensureNewline, trailing)))
-
-        if f is not None:
-            f.close()
-
-    def notify(self, category, file, data):
-        """Check observer for the found data, and if it's
-        not to ignore, notify stat_observers.
-        """
-        rvs = set(
-            observer.notify(category, file, data)
-            for observer in self.observers
-            )
-        if all(rv == 'ignore' for rv in rvs):
-            return 'ignore'
-        rvs.discard('ignore')
-        for obs in self.stat_observers:
-            # non-filtering stat_observers, ignore results
-            obs.notify(category, file, data)
-        if 'error' in rvs:
-            return 'error'
-        assert len(rvs) == 1
-        return rvs.pop()
-
-    def updateStats(self, file, stats):
-        """Check observer for the found data, and if it's
-        not to ignore, notify stat_observers.
-        """
-        for observer in self.observers + self.stat_observers:
-            observer.updateStats(file, stats)
-
-    def remove(self, ref_file, l10n, merge_file):
-        '''Obsolete l10n file.
-
-        Copy to merge stage if we can.
-        '''
-        self.notify('obsoleteFile', l10n, None)
-        self.merge(
-            [], {}, ref_file, l10n, merge_file,
-            [], [], None, parser.CAN_COPY, None
-        )
-
-    def compare(self, ref_file, l10n, merge_file, extra_tests=None):
-        try:
-            p = parser.getParser(ref_file.file)
-        except UserWarning:
-            # no comparison, XXX report?
-            # At least, merge
-            self.merge(
-                [], {},
-                ref_file, l10n, merge_file, [], [], None,
-                parser.CAN_COPY, None)
-            return
-        try:
-            p.readContents(ref_file.getContents())
-        except Exception as e:
-            self.notify('error', ref_file, str(e))
-            return
-        ref_entities, ref_map = p.parse()
-        try:
-            p.readContents(l10n.getContents())
-            l10n_entities, l10n_map = p.parse()
-            l10n_ctx = p.ctx
-        except Exception as e:
-            self.notify('error', l10n, str(e))
-            return
-
-        ar = AddRemove()
-        ar.set_left(e.key for e in ref_entities)
-        ar.set_right(e.key for e in l10n_entities)
-        report = missing = obsolete = changed = unchanged = keys = 0
-        missing_w = changed_w = unchanged_w = 0  # word stats
-        missings = []
-        skips = []
-        checker = getChecker(l10n, extra_tests=extra_tests)
-        if checker and checker.needs_reference:
-            checker.set_reference(ref_entities)
-        for msg in p.findDuplicates(ref_entities):
-            self.notify('warning', l10n, msg)
-        for msg in p.findDuplicates(l10n_entities):
-            self.notify('error', l10n, msg)
-        for action, entity_id in ar:
-            if action == 'delete':
-                # missing entity
-                if isinstance(ref_entities[ref_map[entity_id]], parser.Junk):
-                    self.notify('warning', l10n, 'Parser error in en-US')
-                    continue
-                _rv = self.notify('missingEntity', l10n, entity_id)
-                if _rv == "ignore":
-                    continue
-                if _rv == "error":
-                    # only add to missing entities for l10n-merge on error,
-                    # not report
-                    missings.append(entity_id)
-                    missing += 1
-                    refent = ref_entities[ref_map[entity_id]]
-                    missing_w += refent.count_words()
-                else:
-                    # just report
-                    report += 1
-            elif action == 'add':
-                # obsolete entity or junk
-                if isinstance(l10n_entities[l10n_map[entity_id]],
-                              parser.Junk):
-                    junk = l10n_entities[l10n_map[entity_id]]
-                    params = (junk.val,) + junk.position() + junk.position(-1)
-                    self.notify('error', l10n,
-                                'Unparsed content "%s" from line %d column %d'
-                                ' to line %d column %d' % params)
-                    if merge_file is not None:
-                        skips.append(junk)
-                elif self.notify('obsoleteEntity', l10n,
-                                 entity_id) != 'ignore':
-                    obsolete += 1
-            else:
-                # entity found in both ref and l10n, check for changed
-                refent = ref_entities[ref_map[entity_id]]
-                l10nent = l10n_entities[l10n_map[entity_id]]
-                if self.keyRE.search(entity_id):
-                    keys += 1
-                else:
-                    if refent.equals(l10nent):
-                        self.doUnchanged(l10nent)
-                        unchanged += 1
-                        unchanged_w += refent.count_words()
-                    else:
-                        self.doChanged(ref_file, refent, l10nent)
-                        changed += 1
-                        changed_w += refent.count_words()
-                        # run checks:
-                if checker:
-                    for tp, pos, msg, cat in checker.check(refent, l10nent):
-                        line, col = l10nent.value_position(pos)
-                        # skip error entities when merging
-                        if tp == 'error' and merge_file is not None:
-                            skips.append(l10nent)
-                        self.notify(tp, l10n,
-                                    u"%s at line %d, column %d for %s" %
-                                    (msg, line, col, refent.key))
-                pass
-
-        if merge_file is not None:
-            self.merge(
-                ref_entities, ref_map, ref_file,
-                l10n, merge_file, missings, skips, l10n_ctx,
-                p.capabilities, p.encoding)
-
-        stats = {}
-        for cat, value in (
-                ('missing', missing),
-                ('missing_w', missing_w),
-                ('report', report),
-                ('obsolete', obsolete),
-                ('changed', changed),
-                ('changed_w', changed_w),
-                ('unchanged', unchanged),
-                ('unchanged_w', unchanged_w),
-                ('keys', keys)):
-            if value:
-                stats[cat] = value
-        self.updateStats(l10n, stats)
-        pass
-
-    def add(self, orig, missing, merge_file):
-        ''' Add missing localized file.'''
-        f = orig
-        try:
-            p = parser.getParser(f.file)
-        except UserWarning:
-            p = None
-
-        # if we don't support this file, assume CAN_COPY to mimick
-        # l10n dir as closely as possible
-        caps = p.capabilities if p else parser.CAN_COPY
-        if (caps & (parser.CAN_COPY | parser.CAN_MERGE)):
-            # even if we can merge, pretend we can only copy
-            self.merge(
-                [], {}, orig, missing, merge_file,
-                ['trigger copy'], [], None, parser.CAN_COPY, None
-            )
-
-        if self.notify('missingFile', missing, None) == "ignore":
-            # filter said that we don't need this file, don't count it
-            return
-
-        if p is None:
-            # We don't have a parser, cannot count missing strings
-            return
-
-        try:
-            p.readContents(f.getContents())
-            entities, map = p.parse()
-        except Exception as ex:
-            self.notify('error', f, str(ex))
-            return
-        # strip parse errors
-        entities = [e for e in entities if not isinstance(e, parser.Junk)]
-        self.updateStats(missing, {'missingInFiles': len(entities)})
-        missing_w = 0
-        for e in entities:
-            missing_w += e.count_words()
-        self.updateStats(missing, {'missing_w': missing_w})
-
-    def doUnchanged(self, entity):
-        # overload this if needed
-        pass
-
-    def doChanged(self, file, ref_entity, l10n_entity):
-        # overload this if needed
-        pass
-
-
-def compareProjects(
-            project_configs,
-            stat_observer=None,
-            file_stats=False,
-            merge_stage=None,
-            clobber_merge=False,
-            quiet=0,
-        ):
-    locales = set()
-    observers = []
-    for project in project_configs:
-        # disable filter if we're in validation mode
-        if None in project.locales:
-            filter = None
-        else:
-            filter = project.filter
-        observers.append(
-            Observer(
-                quiet=quiet,
-                filter=filter,
-                file_stats=file_stats,
-            ))
-        locales.update(project.locales)
-    if stat_observer is not None:
-        stat_observers = [stat_observer]
-    else:
-        stat_observers = None
-    comparer = ContentComparer(observers, stat_observers=stat_observers)
-    for locale in sorted(locales):
-        files = paths.ProjectFiles(locale, project_configs,
-                                   mergebase=merge_stage)
-        root = mozpath.commonprefix([m['l10n'].prefix for m in files.matchers])
-        if merge_stage is not None:
-            if clobber_merge:
-                mergematchers = set(_m.get('merge') for _m in files.matchers)
-                mergematchers.discard(None)
-                for matcher in mergematchers:
-                    clobberdir = matcher.prefix
-                    if os.path.exists(clobberdir):
-                        shutil.rmtree(clobberdir)
-                        print("clobbered " + clobberdir)
-        for l10npath, refpath, mergepath, extra_tests in files:
-            # module and file path are needed for legacy filter.py support
-            module = None
-            fpath = mozpath.relpath(l10npath, root)
-            for _m in files.matchers:
-                if _m['l10n'].match(l10npath):
-                    if _m['module']:
-                        # legacy ini support, set module, and resolve
-                        # local path against the matcher prefix,
-                        # which includes the module
-                        module = _m['module']
-                        fpath = mozpath.relpath(l10npath, _m['l10n'].prefix)
-                    break
-            reffile = paths.File(refpath, fpath or refpath, module=module)
-            if locale is None:
-                # When validating the reference files, set locale
-                # to a private subtag. This only shows in the output.
-                locale = paths.REFERENCE_LOCALE
-            l10n = paths.File(l10npath, fpath or l10npath,
-                              module=module, locale=locale)
-            if not os.path.exists(l10npath):
-                comparer.add(reffile, l10n, mergepath)
-                continue
-            if not os.path.exists(refpath):
-                comparer.remove(reffile, l10n, mergepath)
-                continue
-            comparer.compare(reffile, l10n, mergepath, extra_tests)
-    return observers

third_party/python/compare-locales/compare_locales/compare/__init__.py

@@ -0,0 +1,89 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'Mozilla l10n compare locales tool'
+
+from __future__ import absolute_import
+from __future__ import print_function
+import os
+import shutil
+
+from compare_locales import paths, mozpath
+
+from .content import ContentComparer
+from .observer import Observer, ObserverList
+from .utils import Tree, AddRemove
+
+
+__all__ = [
+    'ContentComparer',
+    'Observer', 'ObserverList',
+    'AddRemove', 'Tree',
+    'compareProjects',
+]
+
+
+def compareProjects(
+            project_configs,
+            l10n_base_dir,
+            stat_observer=None,
+            merge_stage=None,
+            clobber_merge=False,
+            quiet=0,
+        ):
+    locales = set()
+    comparer = ContentComparer(quiet)
+    observers = comparer.observers
+    for project in project_configs:
+        # disable filter if we're in validation mode
+        if None in project.locales:
+            filter = None
+        else:
+            filter = project.filter
+        observers.append(
+            Observer(
+                quiet=quiet,
+                filter=filter,
+            ))
+        locales.update(project.locales)
+    for locale in sorted(locales):
+        files = paths.ProjectFiles(locale, project_configs,
+                                   mergebase=merge_stage)
+        if merge_stage is not None:
+            if clobber_merge:
+                mergematchers = set(_m.get('merge') for _m in files.matchers)
+                mergematchers.discard(None)
+                for matcher in mergematchers:
+                    clobberdir = matcher.prefix
+                    if os.path.exists(clobberdir):
+                        shutil.rmtree(clobberdir)
+                        print("clobbered " + clobberdir)
+        for l10npath, refpath, mergepath, extra_tests in files:
+            # module and file path are needed for legacy filter.py support
+            module = None
+            fpath = mozpath.relpath(l10npath, l10n_base_dir)
+            for _m in files.matchers:
+                if _m['l10n'].match(l10npath):
+                    if _m['module']:
+                        # legacy ini support, set module, and resolve
+                        # local path against the matcher prefix,
+                        # which includes the module
+                        module = _m['module']
+                        fpath = mozpath.relpath(l10npath, _m['l10n'].prefix)
+                    break
+            reffile = paths.File(refpath, fpath or refpath, module=module)
+            if locale is None:
+                # When validating the reference files, set locale
+                # to a private subtag. This only shows in the output.
+                locale = paths.REFERENCE_LOCALE
+            l10n = paths.File(l10npath, fpath or l10npath,
+                              module=module, locale=locale)
+            if not os.path.exists(l10npath):
+                comparer.add(reffile, l10n, mergepath)
+                continue
+            if not os.path.exists(refpath):
+                comparer.remove(reffile, l10n, mergepath)
+                continue
+            comparer.compare(reffile, l10n, mergepath, extra_tests)
+    return observers

third_party/python/compare-locales/compare_locales/compare/content.py

@@ -0,0 +1,308 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'Mozilla l10n compare locales tool'
+
+from __future__ import absolute_import
+from __future__ import print_function
+import codecs
+import os
+import shutil
+import re
+
+from compare_locales import parser
+from compare_locales import mozpath
+from compare_locales.checks import getChecker
+from compare_locales.keyedtuple import KeyedTuple
+
+from .observer import ObserverList
+from .utils import AddRemove
+
+
+class ContentComparer:
+    keyRE = re.compile('[kK]ey')
+    nl = re.compile('\n', re.M)
+
+    def __init__(self, quiet=0):
+        '''Create a ContentComparer.
+        observer is usually a instance of Observer. The return values
+        of the notify method are used to control the handling of missing
+        entities.
+        '''
+        self.observers = ObserverList(quiet=quiet)
+
+    def create_merge_dir(self, merge_file):
+        outdir = mozpath.dirname(merge_file)
+        if not os.path.isdir(outdir):
+            os.makedirs(outdir)
+
+    def merge(self, ref_entities, ref_file, l10n_file, merge_file,
+              missing, skips, ctx, capabilities, encoding):
+        '''Create localized file in merge dir
+
+        `ref_entities` and `ref_map` are the parser result of the
+        reference file
+        `ref_file` and `l10n_file` are the File objects for the reference and
+        the l10n file, resp.
+        `merge_file` is the output path for the generated content. This is None
+        if we're just comparing or validating.
+        `missing` are the missing messages in l10n - potentially copied from
+        reference
+        `skips` are entries to be dropped from the localized file
+        `ctx` is the parsing context
+        `capabilities` are the capabilities for the merge algorithm
+        `encoding` is the encoding to be used when serializing, usually utf-8
+        '''
+
+        if not merge_file:
+            return
+
+        if capabilities == parser.CAN_NONE:
+            return
+
+        self.create_merge_dir(merge_file)
+
+        if capabilities & parser.CAN_COPY:
+            # copy the l10n file if it's good, or the reference file if not
+            if skips or missing:
+                src = ref_file.fullpath
+            else:
+                src = l10n_file.fullpath
+            shutil.copyfile(src, merge_file)
+            print("copied reference to " + merge_file)
+            return
+
+        if not (capabilities & parser.CAN_SKIP):
+            return
+
+        # Start with None in case the merge file doesn't need to be created.
+        f = None
+
+        if skips:
+            # skips come in ordered by key name, we need them in file order
+            skips.sort(key=lambda s: s.span[0])
+
+            # we need to skip a few erroneous blocks in the input, copy by hand
+            f = codecs.open(merge_file, 'wb', encoding)
+            offset = 0
+            for skip in skips:
+                chunk = skip.span
+                f.write(ctx.contents[offset:chunk[0]])
+                offset = chunk[1]
+            f.write(ctx.contents[offset:])
+
+        if f is None:
+            # l10n file is a good starting point
+            shutil.copyfile(l10n_file.fullpath, merge_file)
+
+        if not (capabilities & parser.CAN_MERGE):
+            if f:
+                f.close()
+            return
+
+        if skips or missing:
+            if f is None:
+                f = codecs.open(merge_file, 'ab', encoding)
+            trailing = (['\n'] +
+                        [ref_entities[key].all for key in missing] +
+                        [ref_entities[skip.key].all for skip in skips
+                         if not isinstance(skip, parser.Junk)])
+
+            def ensureNewline(s):
+                if not s.endswith('\n'):
+                    return s + '\n'
+                return s
+
+            print("adding to " + merge_file)
+            f.write(''.join(map(ensureNewline, trailing)))
+
+        if f is not None:
+            f.close()
+
+    def remove(self, ref_file, l10n, merge_file):
+        '''Obsolete l10n file.
+
+        Copy to merge stage if we can.
+        '''
+        self.observers.notify('obsoleteFile', l10n, None)
+        self.merge(
+            KeyedTuple([]), ref_file, l10n, merge_file,
+            [], [], None, parser.CAN_COPY, None
+        )
+
+    def compare(self, ref_file, l10n, merge_file, extra_tests=None):
+        try:
+            p = parser.getParser(ref_file.file)
+        except UserWarning:
+            # no comparison, XXX report?
+            # At least, merge
+            self.merge(
+                KeyedTuple([]), ref_file, l10n, merge_file, [], [], None,
+                parser.CAN_COPY, None)
+            return
+        try:
+            p.readFile(ref_file)
+        except Exception as e:
+            self.observers.notify('error', ref_file, str(e))
+            return
+        ref_entities = p.parse()
+        try:
+            p.readFile(l10n)
+            l10n_entities = p.parse()
+            l10n_ctx = p.ctx
+        except Exception as e:
+            self.observers.notify('error', l10n, str(e))
+            return
+
+        ar = AddRemove()
+        ar.set_left(ref_entities.keys())
+        ar.set_right(l10n_entities.keys())
+        report = missing = obsolete = changed = unchanged = keys = 0
+        missing_w = changed_w = unchanged_w = 0  # word stats
+        missings = []
+        skips = []
+        checker = getChecker(l10n, extra_tests=extra_tests)
+        if checker and checker.needs_reference:
+            checker.set_reference(ref_entities)
+        for msg in p.findDuplicates(ref_entities):
+            self.observers.notify('warning', l10n, msg)
+        for msg in p.findDuplicates(l10n_entities):
+            self.observers.notify('error', l10n, msg)
+        for action, entity_id in ar:
+            if action == 'delete':
+                # missing entity
+                if isinstance(ref_entities[entity_id], parser.Junk):
+                    self.observers.notify(
+                        'warning', l10n, 'Parser error in en-US'
+                    )
+                    continue
+                _rv = self.observers.notify('missingEntity', l10n, entity_id)
+                if _rv == "ignore":
+                    continue
+                if _rv == "error":
+                    # only add to missing entities for l10n-merge on error,
+                    # not report
+                    missings.append(entity_id)
+                    missing += 1
+                    refent = ref_entities[entity_id]
+                    missing_w += refent.count_words()
+                else:
+                    # just report
+                    report += 1
+            elif action == 'add':
+                # obsolete entity or junk
+                if isinstance(l10n_entities[entity_id],
+                              parser.Junk):
+                    junk = l10n_entities[entity_id]
+                    params = (junk.val,) + junk.position() + junk.position(-1)
+                    self.observers.notify(
+                        'error', l10n,
+                        'Unparsed content "%s" from line %d column %d'
+                        ' to line %d column %d' % params
+                    )
+                    if merge_file is not None:
+                        skips.append(junk)
+                elif (
+                    self.observers.notify('obsoleteEntity', l10n, entity_id)
+                    != 'ignore'
+                ):
+                    obsolete += 1
+            else:
+                # entity found in both ref and l10n, check for changed
+                refent = ref_entities[entity_id]
+                l10nent = l10n_entities[entity_id]
+                if self.keyRE.search(entity_id):
+                    keys += 1
+                else:
+                    if refent.equals(l10nent):
+                        self.doUnchanged(l10nent)
+                        unchanged += 1
+                        unchanged_w += refent.count_words()
+                    else:
+                        self.doChanged(ref_file, refent, l10nent)
+                        changed += 1
+                        changed_w += refent.count_words()
+                        # run checks:
+                if checker:
+                    for tp, pos, msg, cat in checker.check(refent, l10nent):
+                        line, col = l10nent.value_position(pos)
+                        # skip error entities when merging
+                        if tp == 'error' and merge_file is not None:
+                            skips.append(l10nent)
+                        self.observers.notify(
+                            tp, l10n,
+                            u"%s at line %d, column %d for %s" %
+                            (msg, line, col, refent.key)
+                        )
+                pass
+
+        if merge_file is not None:
+            self.merge(
+                ref_entities, ref_file,
+                l10n, merge_file, missings, skips, l10n_ctx,
+                p.capabilities, p.encoding)
+
+        stats = {}
+        for cat, value in (
+                ('missing', missing),
+                ('missing_w', missing_w),
+                ('report', report),
+                ('obsolete', obsolete),
+                ('changed', changed),
+                ('changed_w', changed_w),
+                ('unchanged', unchanged),
+                ('unchanged_w', unchanged_w),
+                ('keys', keys)):
+            if value:
+                stats[cat] = value
+        self.observers.updateStats(l10n, stats)
+        pass
+
+    def add(self, orig, missing, merge_file):
+        ''' Add missing localized file.'''
+        f = orig
+        try:
+            p = parser.getParser(f.file)
+        except UserWarning:
+            p = None
+
+        # if we don't support this file, assume CAN_COPY to mimick
+        # l10n dir as closely as possible
+        caps = p.capabilities if p else parser.CAN_COPY
+        if (caps & (parser.CAN_COPY | parser.CAN_MERGE)):
+            # even if we can merge, pretend we can only copy
+            self.merge(
+                KeyedTuple([]), orig, missing, merge_file,
+                ['trigger copy'], [], None, parser.CAN_COPY, None
+            )
+
+        if self.observers.notify('missingFile', missing, None) == "ignore":
+            # filter said that we don't need this file, don't count it
+            return
+
+        if p is None:
+            # We don't have a parser, cannot count missing strings
+            return
+
+        try:
+            p.readFile(f)
+            entities = p.parse()
+        except Exception as ex:
+            self.observers.notify('error', f, str(ex))
+            return
+        # strip parse errors
+        entities = [e for e in entities if not isinstance(e, parser.Junk)]
+        self.observers.updateStats(missing, {'missingInFiles': len(entities)})
+        missing_w = 0
+        for e in entities:
+            missing_w += e.count_words()
+        self.observers.updateStats(missing, {'missing_w': missing_w})
+
+    def doUnchanged(self, entity):
+        # overload this if needed
+        pass
+
+    def doChanged(self, file, ref_entity, l10n_entity):
+        # overload this if needed
+        pass

third_party/python/compare-locales/compare_locales/compare/observer.py

@@ -0,0 +1,213 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'Mozilla l10n compare locales tool'
+
+from __future__ import absolute_import
+from __future__ import print_function
+from collections import defaultdict
+import six
+
+from .utils import Tree
+
+
+class Observer(object):
+
+    def __init__(self, quiet=0, filter=None):
+        '''Create Observer
+        For quiet=1, skip per-entity missing and obsolete strings,
+        for quiet=2, skip missing and obsolete files. For quiet=3,
+        skip warnings and errors.
+        '''
+        self.summary = defaultdict(lambda: defaultdict(int))
+        self.details = Tree(list)
+        self.quiet = quiet
+        self.filter = filter
+        self.error = False
+
+    def _dictify(self, d):
+        plaindict = {}
+        for k, v in six.iteritems(d):
+            plaindict[k] = dict(v)
+        return plaindict
+
+    def toJSON(self):
+        # Don't export file stats, even if we collected them.
+        # Those are not part of the data we use toJSON for.
+        return {
+            'summary': self._dictify(self.summary),
+            'details': self.details.toJSON()
+        }
+
+    def updateStats(self, file, stats):
+        # in multi-project scenarios, this file might not be ours,
+        # check that.
+        # Pass in a dummy entity key '' to avoid getting in to
+        # generic file filters. If we have stats for those,
+        # we want to aggregate the counts
+        if (self.filter is not None and
+                self.filter(file, entity='') == 'ignore'):
+            return
+        for category, value in six.iteritems(stats):
+            if category == 'errors':
+                # updateStats isn't called with `errors`, but make sure
+                # we handle this if that changes
+                self.error = True
+            self.summary[file.locale][category] += value
+
+    def notify(self, category, file, data):
+        rv = 'error'
+        if category in ['missingFile', 'obsoleteFile']:
+            if self.filter is not None:
+                rv = self.filter(file)
+            if rv == "ignore" or self.quiet >= 2:
+                return rv
+            if self.quiet == 0 or category == 'missingFile':
+                self.details[file].append({category: rv})
+            return rv
+        if self.filter is not None:
+            rv = self.filter(file, data)
+            if rv == "ignore":
+                return rv
+        if category in ['missingEntity', 'obsoleteEntity']:
+            if (
+                (category == 'missingEntity' and self.quiet < 2)
+                or (category == 'obsoleteEntity' and self.quiet < 1)
+            ):
+                self.details[file].append({category: data})
+            return rv
+        if category == 'error':
+            # Set error independently of quiet
+            self.error = True
+        if category in ('error', 'warning'):
+            if (
+                (category == 'error' and self.quiet < 4)
+                or (category == 'warning' and self.quiet < 3)
+            ):
+                self.details[file].append({category: data})
+            self.summary[file.locale][category + 's'] += 1
+        return rv
+
+
+class ObserverList(Observer):
+    def __init__(self, quiet=0):
+        super(ObserverList, self).__init__(quiet=quiet)
+        self.observers = []
+
+    def __iter__(self):
+        return iter(self.observers)
+
+    def append(self, observer):
+        self.observers.append(observer)
+
+    def notify(self, category, file, data):
+        """Check observer for the found data, and if it's
+        not to ignore, notify stat_observers.
+        """
+        rvs = set(
+            observer.notify(category, file, data)
+            for observer in self.observers
+            )
+        if all(rv == 'ignore' for rv in rvs):
+            return 'ignore'
+        # our return value doesn't count
+        super(ObserverList, self).notify(category, file, data)
+        rvs.discard('ignore')
+        if 'error' in rvs:
+            return 'error'
+        assert len(rvs) == 1
+        return rvs.pop()
+
+    def updateStats(self, file, stats):
+        """Check observer for the found data, and if it's
+        not to ignore, notify stat_observers.
+        """
+        for observer in self.observers:
+            observer.updateStats(file, stats)
+        super(ObserverList, self).updateStats(file, stats)
+
+    def serializeDetails(self):
+
+        def tostr(t):
+            if t[1] == 'key':
+                return '  ' * t[0] + '/'.join(t[2])
+            o = []
+            indent = '  ' * (t[0] + 1)
+            for item in t[2]:
+                if 'error' in item:
+                    o += [indent + 'ERROR: ' + item['error']]
+                elif 'warning' in item:
+                    o += [indent + 'WARNING: ' + item['warning']]
+                elif 'missingEntity' in item:
+                    o += [indent + '+' + item['missingEntity']]
+                elif 'obsoleteEntity' in item:
+                    o += [indent + '-' + item['obsoleteEntity']]
+                elif 'missingFile' in item:
+                    o.append(indent + '// add and localize this file')
+                elif 'obsoleteFile' in item:
+                    o.append(indent + '// remove this file')
+            return '\n'.join(o)
+
+        return '\n'.join(tostr(c) for c in self.details.getContent())
+
+    def serializeSummaries(self):
+        summaries = {
+            loc: []
+            for loc in self.summary.keys()
+        }
+        for observer in self.observers:
+            for loc, lst in summaries.items():
+                lst.append(observer.summary.get(loc))
+        if len(self.observers) > 1:
+            # add ourselves if there's more than one project
+            for loc, lst in summaries.items():
+                lst.append(self.summary.get(loc))
+        # normalize missing and missingInFiles -> missing
+        for summarylist in summaries.values():
+            for summary in summarylist:
+                if 'missingInFiles' in summary:
+                    summary['missing'] = (
+                        summary.get('missing', 0)
+                        + summary.pop('missingInFiles')
+                    )
+        keys = (
+            'errors',
+            'warnings',
+            'missing', 'missing_w',
+            'obsolete', 'obsolete_w',
+            'changed', 'changed_w',
+            'unchanged', 'unchanged_w',
+            'keys',
+        )
+        leads = [
+            '{:12}'.format(k) for k in keys
+        ]
+        out = []
+        for locale, summaries in sorted(six.iteritems(summaries)):
+            if locale:
+                out.append(locale + ':')
+            segment = [''] * len(keys)
+            for summary in summaries:
+                for row, key in enumerate(keys):
+                    segment[row] += ' {:6}'.format(summary.get(key, ''))
+
+            out += [
+                lead + row
+                for lead, row in zip(leads, segment)
+                if row.strip()
+            ]
+
+            total = sum([summaries[-1].get(k, 0)
+                         for k in ['changed', 'unchanged', 'report', 'missing',
+                                   'missingInFiles']
+                         ])
+            rate = 0
+            if total:
+                rate = (('changed' in summary and summary['changed'] * 100) or
+                        0) / total
+            out.append('%d%% of entries changed' % rate)
+        return '\n'.join(out)
+
+    def __str__(self):
+        return 'observer'

third_party/python/compare-locales/compare_locales/compare/utils.py

@@ -0,0 +1,140 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'Mozilla l10n compare locales tool'
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import six
+from six.moves import zip
+
+from compare_locales import paths
+
+
+class Tree(object):
+    def __init__(self, valuetype):
+        self.branches = dict()
+        self.valuetype = valuetype
+        self.value = None
+
+    def __getitem__(self, leaf):
+        parts = []
+        if isinstance(leaf, paths.File):
+            parts = []
+            if leaf.module:
+                parts += [leaf.locale] + leaf.module.split('/')
+            parts += leaf.file.split('/')
+        else:
+            parts = leaf.split('/')
+        return self.__get(parts)
+
+    def __get(self, parts):
+        common = None
+        old = None
+        new = tuple(parts)
+        t = self
+        for k, v in six.iteritems(self.branches):
+            for i, part in enumerate(zip(k, parts)):
+                if part[0] != part[1]:
+                    i -= 1
+                    break
+            if i < 0:
+                continue
+            i += 1
+            common = tuple(k[:i])
+            old = tuple(k[i:])
+            new = tuple(parts[i:])
+            break
+        if old:
+            self.branches.pop(k)
+            t = Tree(self.valuetype)
+            t.branches[old] = v
+            self.branches[common] = t
+        elif common:
+            t = self.branches[common]
+        if new:
+            if common:
+                return t.__get(new)
+            t2 = t
+            t = Tree(self.valuetype)
+            t2.branches[new] = t
+        if t.value is None:
+            t.value = t.valuetype()
+        return t.value
+
+    indent = '  '
+
+    def getContent(self, depth=0):
+        '''
+        Returns iterator of (depth, flag, key_or_value) tuples.
+        If flag is 'value', key_or_value is a value object, otherwise
+        (flag is 'key') it's a key string.
+        '''
+        keys = sorted(self.branches.keys())
+        if self.value is not None:
+            yield (depth, 'value', self.value)
+        for key in keys:
+            yield (depth, 'key', key)
+            for child in self.branches[key].getContent(depth + 1):
+                yield child
+
+    def toJSON(self):
+        '''
+        Returns this Tree as a JSON-able tree of hashes.
+        Only the values need to take care that they're JSON-able.
+        '''
+        if self.value is not None:
+            return self.value
+        return dict(('/'.join(key), self.branches[key].toJSON())
+                    for key in self.branches.keys())
+
+    def getStrRows(self):
+        def tostr(t):
+            if t[1] == 'key':
+                return self.indent * t[0] + '/'.join(t[2])
+            return self.indent * (t[0] + 1) + str(t[2])
+
+        return [tostr(c) for c in self.getContent()]
+
+    def __str__(self):
+        return '\n'.join(self.getStrRows())
+
+
+class AddRemove(object):
+    def __init__(self):
+        self.left = self.right = None
+
+    def set_left(self, left):
+        if not isinstance(left, list):
+            left = list(l for l in left)
+        self.left = left
+
+    def set_right(self, right):
+        if not isinstance(right, list):
+            right = list(l for l in right)
+        self.right = right
+
+    def __iter__(self):
+        # order_map stores index in left and then index in right
+        order_map = dict((item, (i, -1)) for i, item in enumerate(self.left))
+        left_items = set(order_map)
+        # as we go through the right side, keep track of which left
+        # item we had in right last, and for items not in left,
+        # set the sortmap to (left_offset, right_index)
+        left_offset = -1
+        right_items = set()
+        for i, item in enumerate(self.right):
+            right_items.add(item)
+            if item in order_map:
+                left_offset = order_map[item][0]
+            else:
+                order_map[item] = (left_offset, i)
+        for item in sorted(order_map, key=lambda item: order_map[item]):
+            if item in left_items and item in right_items:
+                yield ('equal', item)
+            elif item in left_items:
+                yield ('delete', item)
+            else:
+                yield ('add', item)

third_party/python/compare-locales/compare_locales/keyedtuple.py

@@ -0,0 +1,58 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''A tuple with keys.
+
+A Sequence type that allows to refer to its elements by key.
+Making this immutable, 'cause keeping track of mutations is hard.
+
+compare-locales uses strings for Entity keys, and tuples in the
+case of PO. Support both.
+
+In the interfaces that check for membership, dicts check keys and
+sequences check values. Always try our dict cache `__map` first,
+and fall back to the superclass implementation.
+'''
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+
+class KeyedTuple(tuple):
+
+    def __new__(cls, iterable):
+        return super(KeyedTuple, cls).__new__(cls, iterable)
+
+    def __init__(self, iterable):
+        self.__map = {}
+        if iterable:
+            for index, item in enumerate(self):
+                self.__map[item.key] = index
+
+    def __contains__(self, key):
+        try:
+            contains = key in self.__map
+            if contains:
+                return True
+        except TypeError:
+            pass
+        return super(KeyedTuple, self).__contains__(key)
+
+    def __getitem__(self, key):
+        try:
+            key = self.__map[key]
+        except (KeyError, TypeError):
+            pass
+        return super(KeyedTuple, self).__getitem__(key)
+
+    def keys(self):
+        for value in self:
+            yield value.key
+
+    def items(self):
+        for value in self:
+            yield value.key, value
+
+    def values(self):
+        return self

third_party/python/compare-locales/compare_locales/merge.py

@@ -2,7 +2,18 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-'Merge resources across channels.'
+'''Merge resources across channels.
+
+Merging resources is done over a series of parsed resources, or source
+strings.
+The nomenclature is that the resources are ordered from newest to oldest.
+The generated file structure is taken from the newest file, and then the
+next-newest, etc. The values of the returned entities are taken from the
+newest to the oldest resource, too.
+
+In merge_resources, there's an option to choose the values from oldest
+to newest instead.
+'''
 
 from collections import OrderedDict, defaultdict
 from codecs import encode
@@ -10,27 +21,33 @@ import six
 
 
 from compare_locales import parser as cl
-from compare_locales.compare import AddRemove
+from compare_locales.compare.utils import AddRemove
 
 
 class MergeNotSupportedError(ValueError):
     pass
 
 
-def merge_channels(name, *resources):
+def merge_channels(name, resources):
     try:
         parser = cl.getParser(name)
     except UserWarning:
         raise MergeNotSupportedError(
             'Unsupported file format ({}).'.format(name))
 
-    entities = merge_resources(parser, *resources)
+    entities = merge_resources(parser, resources)
     return encode(serialize_legacy_resource(entities), parser.encoding)
 
 
-def merge_resources(parser, *resources):
-    # A map of comments to the keys of entities they belong to.
-    comments = {}
+def merge_resources(parser, resources, keep_newest=True):
+    '''Merge parsed or unparsed resources, returning a enumerable of Entities.
+
+    Resources are ordered from newest to oldest in the input. The structure
+    of the generated content is taken from the newest resource first, and
+    then filled by the next etc.
+    Values are also taken from the newest, unless keep_newest is False,
+    then values are taken from the oldest first.
+    '''
 
     def parse_resource(resource):
         # The counter dict keeps track of number of identical comments.
@@ -54,44 +71,39 @@ def merge_resources(parser, *resources):
             # prune.
             return (entity, entity)
 
-        # When comments change, AddRemove gives us one 'add' and one 'delete'
-        # (because a comment's key is its content).  In merge_two we'll try to
-        # de-duplicate comments by looking at the entity they belong to.  Set
-        # up the back-reference from the comment to its entity here.
-        if isinstance(entity, cl.Entity) and entity.pre_comment:
-            comments[entity.pre_comment] = entity.key
-
         return (entity.key, entity)
 
     entities = six.moves.reduce(
-        lambda x, y: merge_two(comments, x, y),
+        lambda x, y: merge_two(x, y, keep_newer=keep_newest),
         map(parse_resource, resources))
-    return entities
+    return entities.values()
 
 
-def merge_two(comments, newer, older):
+def merge_two(newer, older, keep_newer=True):
+    '''Merge two OrderedDicts.
+
+    The order of the result dict is determined by `newer`.
+    The values in the dict are the newer ones by default, too.
+    If `keep_newer` is False, the values will be taken from the older
+    dict.
+    '''
     diff = AddRemove()
     diff.set_left(newer.keys())
     diff.set_right(older.keys())
 
     def get_entity(key):
-        entity = newer.get(key, None)
+        if keep_newer:
+            default, backup = newer, older
+        else:
+            default, backup = older, newer
+
+        entity = default.get(key, None)
 
         # Always prefer the newer version.
         if entity is not None:
             return entity
 
-        entity = older.get(key)
-
-        # If it's an old comment attached to an entity, try to find that
-        # entity in newer and return None to use its comment instead in prune.
-        if isinstance(entity, cl.Comment) and entity in comments:
-            next_entity = newer.get(comments[entity], None)
-            if next_entity is not None and next_entity.pre_comment:
-                # We'll prune this before returning the merged result.
-                return None
-
-        return entity
+        return backup.get(key)
 
     # Create a flat sequence of all entities in order reported by AddRemove.
     contents = [(key, get_entity(key)) for _, key in diff]
@@ -119,4 +131,4 @@ def merge_two(comments, newer, older):
 
 
 def serialize_legacy_resource(entities):
-    return "".join((entity.all for entity in entities.values()))
+    return "".join((entity.all for entity in entities))

third_party/python/compare-locales/compare_locales/parser/__init__.py

@@ -9,7 +9,7 @@ import re
 from .base import (
     CAN_NONE, CAN_COPY, CAN_SKIP, CAN_MERGE,
     EntityBase, Entity, Comment, OffsetComment, Junk, Whitespace,
-    Parser
+    BadEntity, Parser,
 )
 from .android import (
     AndroidParser
@@ -26,6 +26,9 @@ from .fluent import (
 from .ini import (
     IniParser, IniSection,
 )
+from .po import (
+    PoParser
+)
 from .properties import (
     PropertiesParser, PropertiesEntity
 )
@@ -33,13 +36,14 @@ from .properties import (
 __all__ = [
     "CAN_NONE", "CAN_COPY", "CAN_SKIP", "CAN_MERGE",
     "Junk", "EntityBase", "Entity", "Whitespace", "Comment", "OffsetComment",
-    "Parser",
+    "BadEntity", "Parser",
     "AndroidParser",
     "DefinesParser", "DefinesInstruction",
     "DTDParser", "DTDEntity",
     "FluentParser", "FluentComment", "FluentEntity",
     "FluentMessage", "FluentTerm",
     "IniParser", "IniSection",
+    "PoParser",
     "PropertiesParser", "PropertiesEntity",
 ]
 
@@ -60,4 +64,5 @@ __constructors = [
     ('\\.ini$', IniParser()),
     ('\\.inc$', DefinesParser()),
     ('\\.ftl$', FluentParser()),
+    ('\\.pot?$', PoParser()),
 ]

third_party/python/compare-locales/compare_locales/parser/android.py

@@ -13,22 +13,26 @@ break the full parsing, and result in a single Junk entry.
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import re
 from xml.dom import minidom
 from xml.dom.minidom import Node
 
 from .base import (
     CAN_SKIP,
     EntityBase, Entity, Comment, Junk, Whitespace,
+    LiteralEntity,
     Parser
 )
 
 
 class AndroidEntity(Entity):
-    def __init__(self, ctx, pre_comment, node, all, key, raw_val, val):
+    def __init__(
+        self, ctx, pre_comment, white_space, node, all, key, raw_val, val
+    ):
         # fill out superclass as good as we can right now
         # most span can get modified at endElement
         super(AndroidEntity, self).__init__(
-            ctx, pre_comment,
+            ctx, pre_comment, white_space,
             (None, None),
             (None, None),
             (None, None)
@@ -41,7 +45,13 @@ class AndroidEntity(Entity):
 
     @property
     def all(self):
-        return self._all_literal
+        chunks = []
+        if self.pre_comment is not None:
+            chunks.append(self.pre_comment.all)
+        if self.inner_white is not None:
+            chunks.append(self.inner_white.all)
+        chunks.append(self._all_literal)
+        return ''.join(chunks)
 
     @property
     def key(self):
@@ -54,6 +64,23 @@ class AndroidEntity(Entity):
     def value_position(self, offset=0):
         return (0, offset)
 
+    def wrap(self, raw_val):
+        clone = self.node.cloneNode(True)
+        if clone.childNodes.length == 1:
+            child = clone.childNodes[0]
+        else:
+            for child in clone.childNodes:
+                if child.nodeType == Node.CDATA_SECTION_NODE:
+                    break
+        child.data = raw_val
+        all = []
+        if self.pre_comment is not None:
+            all.append(self.pre_comment.all)
+        if self.inner_white is not None:
+            all.append(self.inner_white.all)
+        all.append(clone.toxml())
+        return LiteralEntity(self.key, raw_val, ''.join(all))
+
 
 class NodeMixin(object):
     def __init__(self, all, value):
@@ -82,6 +109,10 @@ class XMLComment(NodeMixin, Comment):
     def val(self):
         return self._val_literal
 
+    @property
+    def key(self):
+        return None
+
 
 class DocumentWrapper(NodeMixin, EntityBase):
     def __init__(self, all):
@@ -100,12 +131,25 @@ class XMLJunk(Junk):
 
 
 def textContent(node):
+    if node.childNodes.length == 0:
+        return ''
+    for child in node.childNodes:
+        if child.nodeType == minidom.Node.CDATA_SECTION_NODE:
+            return child.data
     if (
-            node.nodeType == minidom.Node.TEXT_NODE or
-            node.nodeType == minidom.Node.CDATA_SECTION_NODE
+            node.childNodes.length != 1 or
+            node.childNodes[0].nodeType != minidom.Node.TEXT_NODE
     ):
-        return node.nodeValue
-    return ''.join(textContent(child) for child in node.childNodes)
+        # Return something, we'll fail in checks on this
+        return node.toxml()
+    return node.childNodes[0].data
+
+
+NEWLINE = re.compile(r'[ \t]*\n[ \t]*')
+
+
+def normalize(val):
+    return NEWLINE.sub('\n', val.strip(' \t'))
 
 
 class AndroidParser(Parser):
@@ -135,25 +179,62 @@ class AndroidParser(Parser):
             yield DocumentWrapper(
                 '<?xml version="1.0" encoding="utf-8"?>\n<resources>'
             )
-        for node in root_children:
-            if node.nodeType == Node.ELEMENT_NODE:
-                yield self.handleElement(node)
-                self.last_comment = None
-            if node.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):
-                if not only_localizable:
-                    yield XMLWhitespace(node.toxml(), node.nodeValue)
+        child_num = 0
+        while child_num < len(root_children):
+            node = root_children[child_num]
             if node.nodeType == Node.COMMENT_NODE:
-                self.last_comment = XMLComment(node.toxml(), node.nodeValue)
+                current_comment, child_num = self.handleComment(
+                    node, root_children, child_num
+                )
+                if child_num < len(root_children):
+                    node = root_children[child_num]
+                else:
+                    if not only_localizable:
+                        yield current_comment
+                    break
+            else:
+                current_comment = None
+            if node.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):
+                white_space = XMLWhitespace(node.toxml(), node.nodeValue)
+                child_num += 1
+                if current_comment is None:
+                    if not only_localizable:
+                        yield white_space
+                    continue
+                if node.nodeValue.count('\n') > 1:
+                    if not only_localizable:
+                        if current_comment is not None:
+                            yield current_comment
+                        yield white_space
+                    continue
+                if child_num < len(root_children):
+                    node = root_children[child_num]
+                else:
+                    if not only_localizable:
+                        if current_comment is not None:
+                            yield current_comment
+                        yield white_space
+                    break
+            else:
+                white_space = None
+            if node.nodeType == Node.ELEMENT_NODE:
+                yield self.handleElement(node, current_comment, white_space)
+            else:
                 if not only_localizable:
-                    yield self.last_comment
+                    if current_comment:
+                        yield current_comment
+                    if white_space:
+                        yield white_space
+            child_num += 1
         if not only_localizable:
             yield DocumentWrapper('</resources>\n')
 
-    def handleElement(self, element):
+    def handleElement(self, element, current_comment, white_space):
         if element.nodeName == 'string' and element.hasAttribute('name'):
             return AndroidEntity(
                 self.ctx,
-                self.last_comment,
+                current_comment,
+                white_space,
                 element,
                 element.toxml(),
                 element.getAttribute('name'),
@@ -162,3 +243,33 @@ class AndroidParser(Parser):
             )
         else:
             return XMLJunk(element.toxml())
+
+    def handleComment(self, node, root_children, child_num):
+        all = node.toxml()
+        val = normalize(node.nodeValue)
+        while True:
+            child_num += 1
+            if child_num >= len(root_children):
+                break
+            node = root_children[child_num]
+            if node.nodeType == Node.TEXT_NODE:
+                if node.nodeValue.count('\n') > 1:
+                    break
+                white = node
+                child_num += 1
+                if child_num >= len(root_children):
+                    break
+                node = root_children[child_num]
+            else:
+                white = None
+            if node.nodeType != Node.COMMENT_NODE:
+                if white is not None:
+                    # do not consume this node
+                    child_num -= 1
+                break
+            if white:
+                all += white.toxml()
+                val += normalize(white.nodeValue)
+            all += node.toxml()
+            val += normalize(node.nodeValue)
+        return XMLComment(all, val), child_num

third_party/python/compare-locales/compare_locales/parser/base.py

@@ -9,6 +9,8 @@ import bisect
 import codecs
 from collections import Counter
 import logging
+from compare_locales.keyedtuple import KeyedTuple
+from compare_locales.paths import File
 
 import six
 
@@ -43,12 +45,15 @@ class EntityBase(object):
 
     <--- definition ---->
     '''
-    def __init__(self, ctx, pre_comment, span, key_span, val_span):
+    def __init__(
+        self, ctx, pre_comment, inner_white, span, key_span, val_span
+    ):
         self.ctx = ctx
         self.span = span
         self.key_span = key_span
         self.val_span = val_span
         self.pre_comment = pre_comment
+        self.inner_white = inner_white
 
     def position(self, offset=0):
         """Get the 1-based line and column of the character
@@ -75,9 +80,17 @@ class EntityBase(object):
             pos = self.val_span[0] + offset
         return self.ctx.linecol(pos)
 
+    def _span_start(self):
+        start = self.span[0]
+        if hasattr(self, 'pre_comment') and self.pre_comment is not None:
+            start = self.pre_comment.span[0]
+        return start
+
     @property
     def all(self):
-        return self.ctx.contents[self.span[0]:self.span[1]]
+        start = self._span_start()
+        end = self.span[1]
+        return self.ctx.contents[start:end]
 
     @property
     def key(self):
@@ -112,7 +125,63 @@ class EntityBase(object):
 
 
 class Entity(EntityBase):
-    pass
+    @property
+    def localized(self):
+        '''Is this entity localized.
+
+        Always true for monolingual files.
+        In bilingual files, this is a dynamic property.
+        '''
+        return True
+
+    def unwrap(self):
+        """Return the literal value to be used by tools.
+        """
+        return self.raw_val
+
+    def wrap(self, raw_val):
+        """Create literal entity based on reference and raw value.
+
+        This is used by the serialization logic.
+        """
+        start = self._span_start()
+        all = (
+            self.ctx.contents[start:self.val_span[0]] +
+            raw_val +
+            self.ctx.contents[self.val_span[1]:self.span[1]]
+        )
+        return LiteralEntity(self.key, raw_val, all)
+
+
+class LiteralEntity(Entity):
+    """Subclass of Entity to represent entities without context slices.
+
+    It's storing string literals for key, raw_val and all instead of spans.
+    """
+    def __init__(self, key, val, all):
+        super(LiteralEntity, self).__init__(None, None, None, None, None, None)
+        self._key = key
+        self._raw_val = val
+        self._all = all
+
+    @property
+    def key(self):
+        return self._key
+
+    @property
+    def raw_val(self):
+        return self._raw_val
+
+    @property
+    def all(self):
+        return self._all
+
+
+class PlaceholderEntity(LiteralEntity):
+    """Subclass of Entity to be removed in merges.
+    """
+    def __init__(self, key):
+        super(PlaceholderEntity, self).__init__(key, "", "\nplaceholder\n")
 
 
 class Comment(EntityBase):
@@ -206,6 +275,12 @@ class Whitespace(EntityBase):
         return self.raw_val
 
 
+class BadEntity(ValueError):
+    '''Raised when the parser can't create an Entity for a found match.
+    '''
+    pass
+
+
 class Parser(object):
     capabilities = CAN_SKIP | CAN_MERGE
     reWhitespace = re.compile('[ \t\r\n]+', re.M)
@@ -217,8 +292,7 @@ class Parser(object):
         "Fixture for content and line numbers"
         def __init__(self, contents):
             self.contents = contents
-            # Subclasses may use bitmasks to keep state.
-            self.state = 0
+            # cache split lines
             self._lines = None
 
         def linecol(self, position):
@@ -238,10 +312,11 @@ class Parser(object):
         if not hasattr(self, 'encoding'):
             self.encoding = 'utf-8'
         self.ctx = None
-        self.last_comment = None
 
     def readFile(self, file):
         '''Read contents from disk, with universal_newlines'''
+        if isinstance(file, File):
+            file = file.fullpath
         # python 2 has binary input with universal newlines,
         # python 3 doesn't. Let's split code paths
         if six.PY2:
@@ -268,12 +343,10 @@ class Parser(object):
         self.readUnicode(contents)
 
     def readUnicode(self, contents):
-        self.ctx = Parser.Context(contents)
+        self.ctx = self.Context(contents)
 
     def parse(self):
-        list_ = list(self)
-        map_ = dict((e.key, i) for i, e in enumerate(list_))
-        return (list_, map_)
+        return KeyedTuple(self)
 
     def __iter__(self):
         return self.walk(only_localizable=True)
@@ -297,17 +370,55 @@ class Parser(object):
             next_offset = entity.span[1]
 
     def getNext(self, ctx, offset):
-        m = self.reWhitespace.match(ctx.contents, offset)
-        if m:
-            return Whitespace(ctx, m.span())
-        m = self.reKey.match(ctx.contents, offset)
-        if m:
-            return self.createEntity(ctx, m)
+        '''Parse the next fragment.
+
+        Parse comments first, then white-space.
+        If an entity follows, create that entity with such pre_comment and
+        inner white-space. If not, emit comment or white-space as standlone.
+        It's OK that this might parse whitespace more than once.
+        Comments are associated with entities if they're not separated by
+        blank lines. Multiple consecutive comments are joined.
+        '''
+        junk_offset = offset
         m = self.reComment.match(ctx.contents, offset)
         if m:
-            self.last_comment = self.Comment(ctx, m.span())
-            return self.last_comment
-        return self.getJunk(ctx, offset, self.reKey, self.reComment)
+            current_comment = self.Comment(ctx, m.span())
+            if offset < 2 and 'License' in current_comment.val:
+                # Heuristic. A early comment with "License" is probably
+                # a license header, and should be standalone.
+                # Not glueing ourselves to offset == 0 as we might have
+                # skipped a BOM.
+                return current_comment
+            offset = m.end()
+        else:
+            current_comment = None
+        m = self.reWhitespace.match(ctx.contents, offset)
+        if m:
+            white_space = Whitespace(ctx, m.span())
+            offset = m.end()
+            if (
+                current_comment is not None
+                and white_space.raw_val.count('\n') > 1
+            ):
+                # standalone comment
+                # return the comment, and reparse the whitespace next time
+                return current_comment
+            if current_comment is None:
+                return white_space
+        else:
+            white_space = None
+        m = self.reKey.match(ctx.contents, offset)
+        if m:
+            try:
+                return self.createEntity(ctx, m, current_comment, white_space)
+            except BadEntity:
+                # fall through to Junk, probably
+                pass
+        if current_comment is not None:
+            return current_comment
+        if white_space is not None:
+            return white_space
+        return self.getJunk(ctx, junk_offset, self.reKey, self.reComment)
 
     def getJunk(self, ctx, offset, *expressions):
         junkend = None
@@ -317,10 +428,11 @@ class Parser(object):
                 junkend = min(junkend, m.start()) if junkend else m.start()
         return Junk(ctx, (offset, junkend or len(ctx.contents)))
 
-    def createEntity(self, ctx, m):
-        pre_comment = self.last_comment
-        self.last_comment = None
-        return Entity(ctx, pre_comment, m.span(), m.span('key'), m.span('val'))
+    def createEntity(self, ctx, m, current_comment, white_space):
+        return Entity(
+            ctx, current_comment, white_space,
+            m.span(), m.span('key'), m.span('val')
+        )
 
     @classmethod
     def findDuplicates(cls, entities):

third_party/python/compare-locales/compare_locales/parser/defines.py

@@ -31,11 +31,15 @@ class DefinesParser(Parser):
     reWhitespace = re.compile('\n+', re.M)
 
     EMPTY_LINES = 1 << 0
-    PAST_FIRST_LINE = 1 << 1
 
     class Comment(OffsetComment):
         comment_offset = 2
 
+    class Context(Parser.Context):
+        def __init__(self, contents):
+            super(DefinesParser.Context, self).__init__(contents)
+            self.filter_empty_lines = False
+
     def __init__(self):
         self.reComment = re.compile('(?:^# .*?\n)*(?:^# [^\n]*)', re.M)
         # corresponds to
@@ -46,34 +50,57 @@ class DefinesParser(Parser):
         Parser.__init__(self)
 
     def getNext(self, ctx, offset):
+        junk_offset = offset
         contents = ctx.contents
 
+        m = self.reComment.match(ctx.contents, offset)
+        if m:
+            current_comment = self.Comment(ctx, m.span())
+            offset = m.end()
+        else:
+            current_comment = None
+
         m = self.reWhitespace.match(contents, offset)
         if m:
-            if ctx.state & self.EMPTY_LINES:
-                return Whitespace(ctx, m.span())
-            if ctx.state & self.PAST_FIRST_LINE and len(m.group()) == 1:
-                return Whitespace(ctx, m.span())
-            else:
+            # blank lines outside of filter_empty_lines or
+            # leading whitespace are bad
+            if (
+                offset == 0 or
+                not (len(m.group()) == 1 or ctx.filter_empty_lines)
+            ):
+                if current_comment:
+                    return current_comment
                 return Junk(ctx, m.span())
+            white_space = Whitespace(ctx, m.span())
+            offset = m.end()
+            if (
+                current_comment is not None
+                and white_space.raw_val.count('\n') > 1
+            ):
+                # standalone comment
+                # return the comment, and reparse the whitespace next time
+                return current_comment
+            if current_comment is None:
+                return white_space
+        else:
+            white_space = None
 
-        # We're not in the first line anymore.
-        ctx.state |= self.PAST_FIRST_LINE
-
-        m = self.reComment.match(contents, offset)
-        if m:
-            self.last_comment = self.Comment(ctx, m.span())
-            return self.last_comment
         m = self.reKey.match(contents, offset)
         if m:
-            return self.createEntity(ctx, m)
+            return self.createEntity(ctx, m, current_comment, white_space)
+        # defines instructions don't have comments
+        # Any pending commment is standalone
+        if current_comment:
+            return current_comment
+        if white_space:
+            return white_space
         m = self.rePI.match(contents, offset)
         if m:
             instr = DefinesInstruction(ctx, m.span(), m.span('val'))
             if instr.val == 'filter emptyLines':
-                ctx.state |= self.EMPTY_LINES
+                ctx.filter_empty_lines = True
             if instr.val == 'unfilter emptyLines':
-                ctx.state &= ~ self.EMPTY_LINES
+                ctx.filter_empty_lines = False
             return instr
         return self.getJunk(
-            ctx, offset, self.reComment, self.reKey, self.rePI)
+            ctx, junk_offset, self.reComment, self.reKey, self.rePI)

third_party/python/compare-locales/compare_locales/parser/dtd.py

@@ -107,15 +107,12 @@ class DTDParser(Parser):
         if (entity and isinstance(entity, Junk)) or entity is None:
             m = self.rePE.match(ctx.contents, offset)
             if m:
-                self.last_comment = None
                 entity = DTDEntity(
-                    ctx, '', m.span(), m.span('key'), m.span('val'))
+                    ctx, None, None, m.span(), m.span('key'), m.span('val'))
         return entity
 
-    def createEntity(self, ctx, m):
+    def createEntity(self, ctx, m, current_comment, white_space):
         valspan = m.span('val')
         valspan = (valspan[0]+1, valspan[1]-1)
-        pre_comment = self.last_comment
-        self.last_comment = None
-        return DTDEntity(ctx, pre_comment,
+        return DTDEntity(ctx, current_comment, white_space,
                          m.span(), m.span('key'), valspan)

third_party/python/compare-locales/compare_locales/parser/fluent.py

@@ -8,9 +8,11 @@ import re
 
 from fluent.syntax import FluentParser as FTLParser
 from fluent.syntax import ast as ftl
+from fluent.syntax.serializer import serialize_comment
 from .base import (
     CAN_SKIP,
     EntityBase, Entity, Comment, Junk, Whitespace,
+    LiteralEntity,
     Parser
 )
 
@@ -94,6 +96,21 @@ class FluentEntity(Entity):
         for attr_node in self.entry.attributes:
             yield FluentAttribute(self, attr_node)
 
+    def unwrap(self):
+        return self.all
+
+    def wrap(self, raw_val):
+        """Create literal entity the given raw value.
+
+        For Fluent, we're exposing the message source to tools like
+        Pontoon.
+        We also recreate the comment from this entity to the created entity.
+        """
+        all = raw_val
+        if self.entry.comment is not None:
+            all = serialize_comment(self.entry.comment) + all
+        return LiteralEntity(self.key, raw_val, all)
+
 
 class FluentMessage(FluentEntity):
     pass
@@ -150,10 +167,18 @@ class FluentParser(Parser):
                 end = entry.span.end
                 # strip leading whitespace
                 start += re.match('[ \t\r\n]*', entry.content).end()
+                if not only_localizable and entry.span.start < start:
+                    yield Whitespace(
+                        self.ctx, (entry.span.start, start)
+                    )
                 # strip trailing whitespace
                 ws, we = re.search('[ \t\r\n]*$', entry.content).span()
                 end -= we - ws
                 yield Junk(self.ctx, (start, end))
+                if not only_localizable and end < entry.span.end:
+                    yield Whitespace(
+                        self.ctx, (end, entry.span.end)
+                    )
             elif isinstance(entry, ftl.BaseComment) and not only_localizable:
                 span = (entry.span.start, entry.span.end)
                 yield FluentComment(self.ctx, span, entry)

third_party/python/compare-locales/compare_locales/parser/ini.py

@@ -7,7 +7,7 @@ from __future__ import unicode_literals
 import re
 
 from .base import (
-    EntityBase, OffsetComment, Whitespace,
+    EntityBase, OffsetComment,
     Parser
 )
 
@@ -45,18 +45,14 @@ class IniParser(Parser):
 
     def getNext(self, ctx, offset):
         contents = ctx.contents
-        m = self.reWhitespace.match(contents, offset)
-        if m:
-            return Whitespace(ctx, m.span())
-        m = self.reComment.match(contents, offset)
-        if m:
-            self.last_comment = self.Comment(ctx, m.span())
-            return self.last_comment
         m = self.reSection.match(contents, offset)
         if m:
             return IniSection(ctx, m.span(), m.span('val'))
-        m = self.reKey.match(contents, offset)
-        if m:
-            return self.createEntity(ctx, m)
-        return self.getJunk(
-            ctx, offset, self.reComment, self.reSection, self.reKey)
+
+        return super(IniParser, self).getNext(ctx, offset)
+
+    def getJunk(self, ctx, offset, *expressions):
+        # base.Parser.getNext calls us with self.reKey, self.reComment.
+        # Add self.reSection to the end-of-junk expressions
+        expressions = expressions + (self.reSection,)
+        return super(IniParser, self).getJunk(ctx, offset, *expressions)

third_party/python/compare-locales/compare_locales/parser/po.py

@@ -0,0 +1,123 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""Gettext PO(T) parser
+
+Parses gettext po and pot files.
+"""
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import re
+
+from .base import (
+    CAN_SKIP,
+    Entity,
+    BadEntity,
+    Parser
+)
+
+
+class PoEntityMixin(object):
+
+    @property
+    def val(self):
+        return self.stringlist_val
+
+    @property
+    def key(self):
+        return self.stringlist_key
+
+    @property
+    def localized(self):
+        # gettext denotes a non-localized string by an empty value
+        return bool(self.val)
+
+    def __repr__(self):
+        return self.key[0]
+
+
+class PoEntity(PoEntityMixin, Entity):
+    pass
+
+
+# Unescape and concat a string list
+def eval_stringlist(lines):
+    return ''.join(
+        (
+            l
+            .replace(r'\\', '\\')
+            .replace(r'\t', '\t')
+            .replace(r'\r', '\r')
+            .replace(r'\n', '\n')
+            .replace(r'\"', '"')
+        )
+        for l in lines
+    )
+
+
+class PoParser(Parser):
+    # gettext l10n fallback at runtime, don't merge en-US strings
+    capabilities = CAN_SKIP
+
+    reKey = re.compile('msgctxt|msgid')
+    reValue = re.compile('(?P<white>[ \t\r\n]*)(?P<cmd>msgstr)')
+    reComment = re.compile(r'(?:#.*?\n)+')
+    # string list item:
+    # leading whitespace
+    # `"`
+    # escaped quotes etc, not quote, newline, backslash
+    # `"`
+    reListItem = re.compile(r'[ \t\r\n]*"((?:\\[\\trn"]|[^"\n\\])*)"')
+
+    def __init__(self):
+        super(PoParser, self).__init__()
+
+    def createEntity(self, ctx, m, current_comment, white_space):
+        start = cursor = m.start()
+        id_start = cursor
+        try:
+            msgctxt, cursor = self._parse_string_list(ctx, cursor, 'msgctxt')
+            m = self.reWhitespace.match(ctx.contents, cursor)
+            if m:
+                cursor = m.end()
+        except BadEntity:
+            # no msgctxt is OK
+            msgctxt = None
+        if id_start is None:
+            id_start = cursor
+        msgid, cursor = self._parse_string_list(ctx, cursor, 'msgid')
+        id_end = cursor
+        m = self.reWhitespace.match(ctx.contents, cursor)
+        if m:
+            cursor = m.end()
+        val_start = cursor
+        msgstr, cursor = self._parse_string_list(ctx, cursor, 'msgstr')
+        e = PoEntity(
+            ctx,
+            current_comment,
+            white_space,
+            (start, cursor),
+            (id_start, id_end),
+            (val_start, cursor)
+        )
+        e.stringlist_key = (msgid, msgctxt)
+        e.stringlist_val = msgstr
+        return e
+
+    def _parse_string_list(self, ctx, cursor, key):
+        if not ctx.contents.startswith(key, cursor):
+            raise BadEntity
+        cursor += len(key)
+        frags = []
+        while True:
+            m = self.reListItem.match(ctx.contents, cursor)
+            if not m:
+                break
+            frags.append(m.group(1))
+            cursor = m.end()
+        if not frags:
+            raise BadEntity
+        return eval_stringlist(frags), cursor

third_party/python/compare-locales/compare_locales/parser/properties.py

@@ -48,17 +48,35 @@ class PropertiesParser(Parser):
         Parser.__init__(self)
 
     def getNext(self, ctx, offset):
+        junk_offset = offset
         # overwritten to parse values line by line
         contents = ctx.contents
 
-        m = self.reWhitespace.match(contents, offset)
-        if m:
-            return Whitespace(ctx, m.span())
-
         m = self.reComment.match(contents, offset)
         if m:
-            self.last_comment = self.Comment(ctx, m.span())
-            return self.last_comment
+            current_comment = self.Comment(ctx, m.span())
+            if offset == 0 and 'License' in current_comment.val:
+                # Heuristic. A early comment with "License" is probably
+                # a license header, and should be standalone.
+                return current_comment
+            offset = m.end()
+        else:
+            current_comment = None
+
+        m = self.reWhitespace.match(contents, offset)
+        if m:
+            white_space = Whitespace(ctx, m.span())
+            offset = m.end()
+            if (
+                current_comment is not None
+                and white_space.raw_val.count('\n') > 1
+            ):
+                # standalone comment
+                return current_comment
+            if current_comment is None:
+                return white_space
+        else:
+            white_space = None
 
         m = self.reKey.match(contents, offset)
         if m:
@@ -83,13 +101,16 @@ class PropertiesParser(Parser):
             if ws:
                 endval = ws.start()
 
-            pre_comment = self.last_comment
-            self.last_comment = None
             entity = PropertiesEntity(
-                ctx, pre_comment,
+                ctx, current_comment, white_space,
                 (m.start(), endval),   # full span
                 m.span('key'),
                 (m.end(), endval))   # value span
             return entity
 
-        return self.getJunk(ctx, offset, self.reKey, self.reComment)
+        if current_comment is not None:
+            return current_comment
+        if white_space is not None:
+            return white_space
+
+        return self.getJunk(ctx, junk_offset, self.reKey, self.reComment)

third_party/python/compare-locales/compare_locales/paths.py

@@ -1,821 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import
-import os
-import re
-from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
-from collections import defaultdict
-import errno
-import itertools
-import logging
-import warnings
-from compare_locales import util, mozpath
-import pytoml as toml
-import six
-
-
-REFERENCE_LOCALE = 'en-x-moz-reference'
-
-
-class Matcher(object):
-    '''Path pattern matcher
-    Supports path matching similar to mozpath.match(), but does
-    not match trailing file paths without trailing wildcards.
-    Also gets a prefix, which is the path before the first wildcard,
-    which is good for filesystem iterations, and allows to replace
-    the own matches in a path on a different Matcher. compare-locales
-    uses that to transform l10n and en-US paths back and forth.
-    '''
-
-    def __init__(self, pattern):
-        '''Create regular expression similar to mozpath.match().
-        '''
-        prefix = ''
-        last_end = 0
-        p = ''
-        r = ''
-        backref = itertools.count(1)
-        for m in re.finditer(r'(?:(^|/)\*\*(/|$))|(?P<star>\*)', pattern):
-            if m.start() > last_end:
-                p += re.escape(pattern[last_end:m.start()])
-                r += pattern[last_end:m.start()]
-                if last_end == 0:
-                    prefix = pattern[last_end:m.start()]
-            if m.group('star'):
-                p += '([^/]*)'
-                r += r'\%s' % next(backref)
-            else:
-                p += re.escape(m.group(1)) + r'(.+%s)?' % m.group(2)
-                r += m.group(1) + r'\%s' % next(backref) + m.group(2)
-            last_end = m.end()
-        p += re.escape(pattern[last_end:]) + '$'
-        r += pattern[last_end:]
-        if last_end == 0:
-            prefix = pattern
-        self.prefix = prefix
-        self.regex = re.compile(p)
-        self.placable = r
-
-    def match(self, path):
-        '''
-        True if the given path matches the file pattern.
-        '''
-        return self.regex.match(path) is not None
-
-    def sub(self, other, path):
-        '''
-        Replace the wildcard matches in this pattern into the
-        pattern of the other Match object.
-        '''
-        if not self.match(path):
-            return None
-        return self.regex.sub(other.placable, path)
-
-
-class ProjectConfig(object):
-    '''Abstraction of l10n project configuration data.
-    '''
-
-    def __init__(self):
-        self.filter_py = None  # legacy filter code
-        # {
-        #  'l10n': pattern,
-        #  'reference': pattern,  # optional
-        #  'locales': [],  # optional
-        #  'test': [],  # optional
-        # }
-        self.paths = []
-        self.rules = []
-        self.locales = []
-        self.environ = {}
-        self.children = []
-        self._cache = None
-
-    variable = re.compile('{ *([\w]+) *}')
-
-    def expand(self, path, env=None):
-        if env is None:
-            env = {}
-
-        def _expand(m):
-            _var = m.group(1)
-            for _env in (env, self.environ):
-                if _var in _env:
-                    return self.expand(_env[_var], env)
-            return '{{{}}}'.format(_var)
-        return self.variable.sub(_expand, path)
-
-    def lazy_expand(self, pattern):
-        def lazy_l10n_expanded_pattern(env):
-            return Matcher(self.expand(pattern, env))
-        return lazy_l10n_expanded_pattern
-
-    def add_global_environment(self, **kwargs):
-        self.add_environment(**kwargs)
-        for child in self.children:
-            child.add_global_environment(**kwargs)
-
-    def add_environment(self, **kwargs):
-        self.environ.update(kwargs)
-
-    def add_paths(self, *paths):
-        '''Add path dictionaries to this config.
-        The dictionaries must have a `l10n` key. For monolingual files,
-        `reference` is also required.
-        An optional key `test` is allowed to enable additional tests for this
-        path pattern.
-        '''
-
-        for d in paths:
-            rv = {
-                'l10n': self.lazy_expand(d['l10n']),
-                'module': d.get('module')
-            }
-            if 'reference' in d:
-                rv['reference'] = Matcher(d['reference'])
-            if 'test' in d:
-                rv['test'] = d['test']
-            if 'locales' in d:
-                rv['locales'] = d['locales'][:]
-            self.paths.append(rv)
-
-    def set_filter_py(self, filter_function):
-        '''Set legacy filter.py code.
-        Assert that no rules are set.
-        Also, normalize output already here.
-        '''
-        assert not self.rules
-
-        def filter_(module, path, entity=None):
-            try:
-                rv = filter_function(module, path, entity=entity)
-            except BaseException:  # we really want to handle EVERYTHING here
-                return 'error'
-            rv = {
-                True: 'error',
-                False: 'ignore',
-                'report': 'warning'
-            }.get(rv, rv)
-            assert rv in ('error', 'ignore', 'warning', None)
-            return rv
-        self.filter_py = filter_
-
-    def add_rules(self, *rules):
-        '''Add rules to filter on.
-        Assert that there's no legacy filter.py code hooked up.
-        '''
-        assert self.filter_py is None
-        for rule in rules:
-            self.rules.extend(self._compile_rule(rule))
-
-    def add_child(self, child):
-        self.children.append(child)
-
-    def set_locales(self, locales, deep=False):
-        self.locales = locales
-        for child in self.children:
-            if not child.locales or deep:
-                child.set_locales(locales, deep=True)
-            else:
-                locs = [loc for loc in locales if loc in child.locales]
-                child.set_locales(locs)
-
-    @property
-    def configs(self):
-        'Recursively get all configs in this project and its children'
-        yield self
-        for child in self.children:
-            for config in child.configs:
-                yield config
-
-    def filter(self, l10n_file, entity=None):
-        '''Filter a localization file or entities within, according to
-        this configuration file.'''
-        if self.filter_py is not None:
-            return self.filter_py(l10n_file.module, l10n_file.file,
-                                  entity=entity)
-        rv = self._filter(l10n_file, entity=entity)
-        if rv is None:
-            return 'ignore'
-        return rv
-
-    class FilterCache(object):
-        def __init__(self, locale):
-            self.locale = locale
-            self.rules = []
-            self.l10n_paths = []
-
-    def cache(self, locale):
-        if self._cache and self._cache.locale == locale:
-            return self._cache
-        self._cache = self.FilterCache(locale)
-        for paths in self.paths:
-            self._cache.l10n_paths.append(paths['l10n']({
-                "locale": locale
-            }))
-        for rule in self.rules:
-            cached_rule = rule.copy()
-            cached_rule['path'] = rule['path']({
-                "locale": locale
-            })
-            self._cache.rules.append(cached_rule)
-        return self._cache
-
-    def _filter(self, l10n_file, entity=None):
-        actions = set(
-            child._filter(l10n_file, entity=entity)
-            for child in self.children)
-        if 'error' in actions:
-            # return early if we know we'll error
-            return 'error'
-
-        cached = self.cache(l10n_file.locale)
-        if any(p.match(l10n_file.fullpath) for p in cached.l10n_paths):
-            action = 'error'
-            for rule in reversed(cached.rules):
-                if not rule['path'].match(l10n_file.fullpath):
-                    continue
-                if ('key' in rule) ^ (entity is not None):
-                    # key/file mismatch, not a matching rule
-                    continue
-                if 'key' in rule and not rule['key'].match(entity):
-                    continue
-                action = rule['action']
-                break
-            actions.add(action)
-        if 'error' in actions:
-            return 'error'
-        if 'warning' in actions:
-            return 'warning'
-        if 'ignore' in actions:
-            return 'ignore'
-
-    def _compile_rule(self, rule):
-        assert 'path' in rule
-        if isinstance(rule['path'], list):
-            for path in rule['path']:
-                _rule = rule.copy()
-                _rule['path'] = self.lazy_expand(path)
-                for __rule in self._compile_rule(_rule):
-                    yield __rule
-            return
-        if isinstance(rule['path'], six.string_types):
-            rule['path'] = self.lazy_expand(rule['path'])
-        if 'key' not in rule:
-            yield rule
-            return
-        if not isinstance(rule['key'], six.string_types):
-            for key in rule['key']:
-                _rule = rule.copy()
-                _rule['key'] = key
-                for __rule in self._compile_rule(_rule):
-                    yield __rule
-            return
-        rule = rule.copy()
-        key = rule['key']
-        if key.startswith('re:'):
-            key = key[3:]
-        else:
-            key = re.escape(key) + '$'
-        rule['key'] = re.compile(key)
-        yield rule
-
-
-class ProjectFiles(object):
-    '''Iterable object to get all files and tests for a locale and a
-    list of ProjectConfigs.
-
-    If the given locale is None, iterate over reference files as
-    both reference and locale for a reference self-test.
-    '''
-    def __init__(self, locale, projects, mergebase=None):
-        self.locale = locale
-        self.matchers = []
-        self.mergebase = mergebase
-        configs = []
-        for project in projects:
-            configs.extend(project.configs)
-        for pc in configs:
-            if locale and locale not in pc.locales:
-                continue
-            for paths in pc.paths:
-                if (
-                    locale and
-                    'locales' in paths and
-                    locale not in paths['locales']
-                ):
-                    continue
-                m = {
-                    'l10n': paths['l10n']({
-                        "locale": locale or REFERENCE_LOCALE
-                    }),
-                    'module': paths.get('module'),
-                }
-                if 'reference' in paths:
-                    m['reference'] = paths['reference']
-                if self.mergebase is not None:
-                    m['merge'] = paths['l10n']({
-                        "locale": locale,
-                        "l10n_base": self.mergebase
-                    })
-                m['test'] = set(paths.get('test', []))
-                if 'locales' in paths:
-                    m['locales'] = paths['locales'][:]
-                self.matchers.append(m)
-        self.matchers.reverse()  # we always iterate last first
-        # Remove duplicate patterns, comparing each matcher
-        # against all other matchers.
-        # Avoid n^2 comparisons by only scanning the upper triangle
-        # of a n x n matrix of all possible combinations.
-        # Using enumerate and keeping track of indexes, as we can't
-        # modify the list while iterating over it.
-        drops = set()  # duplicate matchers to remove
-        for i, m in enumerate(self.matchers[:-1]):
-            if i in drops:
-                continue  # we're dropping this anyway, don't search again
-            for i_, m_ in enumerate(self.matchers[(i+1):]):
-                if (mozpath.realpath(m['l10n'].prefix) !=
-                        mozpath.realpath(m_['l10n'].prefix)):
-                    # ok, not the same thing, continue
-                    continue
-                # check that we're comparing the same thing
-                if 'reference' in m:
-                    if (mozpath.realpath(m['reference'].prefix) !=
-                            mozpath.realpath(m_.get('reference').prefix)):
-                        raise RuntimeError('Mismatch in reference for ' +
-                                           mozpath.realpath(m['l10n'].prefix))
-                drops.add(i_ + i + 1)
-                m['test'] |= m_['test']
-        drops = sorted(drops, reverse=True)
-        for i in drops:
-            del self.matchers[i]
-
-    def __iter__(self):
-        # The iteration is pretty different when we iterate over
-        # a localization vs over the reference. We do that latter
-        # when running in validation mode.
-        inner = self.iter_locale() if self.locale else self.iter_reference()
-        for t in inner:
-            yield t
-
-    def iter_locale(self):
-        '''Iterate over locale files.'''
-        known = {}
-        for matchers in self.matchers:
-            matcher = matchers['l10n']
-            for path in self._files(matcher):
-                if path not in known:
-                    known[path] = {'test': matchers.get('test')}
-                    if 'reference' in matchers:
-                        known[path]['reference'] = matcher.sub(
-                            matchers['reference'], path)
-                    if 'merge' in matchers:
-                        known[path]['merge'] = matcher.sub(
-                            matchers['merge'], path)
-            if 'reference' not in matchers:
-                continue
-            matcher = matchers['reference']
-            for path in self._files(matcher):
-                l10npath = matcher.sub(matchers['l10n'], path)
-                if l10npath not in known:
-                    known[l10npath] = {
-                        'reference': path,
-                        'test': matchers.get('test')
-                    }
-                    if 'merge' in matchers:
-                        known[l10npath]['merge'] = \
-                            matcher.sub(matchers['merge'], path)
-        for path, d in sorted(known.items()):
-            yield (path, d.get('reference'), d.get('merge'), d['test'])
-
-    def iter_reference(self):
-        '''Iterate over reference files.'''
-        known = {}
-        for matchers in self.matchers:
-            if 'reference' not in matchers:
-                continue
-            matcher = matchers['reference']
-            for path in self._files(matcher):
-                refpath = matcher.sub(matchers['reference'], path)
-                if refpath not in known:
-                    known[refpath] = {
-                        'reference': path,
-                        'test': matchers.get('test')
-                    }
-        for path, d in sorted(known.items()):
-            yield (path, d.get('reference'), None, d['test'])
-
-    def _files(self, matcher):
-        '''Base implementation of getting all files in a hierarchy
-        using the file system.
-        Subclasses might replace this method to support different IO
-        patterns.
-        '''
-        base = matcher.prefix
-        if os.path.isfile(base):
-            if matcher.match(base):
-                yield base
-            return
-        for d, dirs, files in os.walk(base):
-            for f in files:
-                p = mozpath.join(d, f)
-                if matcher.match(p):
-                    yield p
-
-    def match(self, path):
-        '''Return the tuple of l10n_path, reference, mergepath, tests
-        if the given path matches any config, otherwise None.
-
-        This routine doesn't check that the files actually exist.
-        '''
-        for matchers in self.matchers:
-            matcher = matchers['l10n']
-            if matcher.match(path):
-                ref = merge = None
-                if 'reference' in matchers:
-                    ref = matcher.sub(matchers['reference'], path)
-                if 'merge' in matchers:
-                    merge = matcher.sub(matchers['merge'], path)
-                return path, ref, merge, matchers.get('test')
-            if 'reference' not in matchers:
-                continue
-            matcher = matchers['reference']
-            if matcher.match(path):
-                merge = None
-                l10n = matcher.sub(matchers['l10n'], path)
-                if 'merge' in matchers:
-                    merge = matcher.sub(matchers['merge'], path)
-                return l10n, path, merge, matchers.get('test')
-
-
-class ConfigNotFound(EnvironmentError):
-    def __init__(self, path):
-        super(ConfigNotFound, self).__init__(
-            errno.ENOENT,
-            'Configuration file not found',
-            path)
-
-
-class TOMLParser(object):
-    @classmethod
-    def parse(cls, path, env=None, ignore_missing_includes=False):
-        parser = cls(path, env=env,
-                     ignore_missing_includes=ignore_missing_includes)
-        parser.load()
-        parser.processEnv()
-        parser.processPaths()
-        parser.processFilters()
-        parser.processIncludes()
-        parser.processLocales()
-        return parser.asConfig()
-
-    def __init__(self, path, env=None, ignore_missing_includes=False):
-        self.path = path
-        self.env = env if env is not None else {}
-        self.ignore_missing_includes = ignore_missing_includes
-        self.data = None
-        self.pc = ProjectConfig()
-        self.pc.PATH = path
-
-    def load(self):
-        try:
-            with open(self.path, 'rb') as fin:
-                self.data = toml.load(fin)
-        except (toml.TomlError, IOError):
-            raise ConfigNotFound(self.path)
-
-    def processEnv(self):
-        assert self.data is not None
-        self.pc.add_environment(**self.data.get('env', {}))
-
-    def processLocales(self):
-        assert self.data is not None
-        if 'locales' in self.data:
-            self.pc.set_locales(self.data['locales'])
-
-    def processPaths(self):
-        assert self.data is not None
-        for data in self.data.get('paths', []):
-            l10n = data['l10n']
-            if not l10n.startswith('{'):
-                # l10n isn't relative to a variable, expand
-                l10n = self.resolvepath(l10n)
-            paths = {
-                "l10n": l10n,
-            }
-            if 'locales' in data:
-                paths['locales'] = data['locales']
-            if 'reference' in data:
-                paths['reference'] = self.resolvepath(data['reference'])
-            self.pc.add_paths(paths)
-
-    def processFilters(self):
-        assert self.data is not None
-        for data in self.data.get('filters', []):
-            paths = data['path']
-            if isinstance(paths, six.string_types):
-                paths = [paths]
-            # expand if path isn't relative to a variable
-            paths = [
-                self.resolvepath(path) if not path.startswith('{')
-                else path
-                for path in paths
-            ]
-            rule = {
-                "path": paths,
-                "action": data['action']
-            }
-            if 'key' in data:
-                rule['key'] = data['key']
-            self.pc.add_rules(rule)
-
-    def processIncludes(self):
-        assert self.data is not None
-        if 'includes' not in self.data:
-            return
-        for include in self.data['includes']:
-            p = include['path']
-            p = self.resolvepath(p)
-            try:
-                child = self.parse(
-                    p, env=self.env,
-                    ignore_missing_includes=self.ignore_missing_includes
-                )
-            except ConfigNotFound as e:
-                if not self.ignore_missing_includes:
-                    raise
-                (logging
-                    .getLogger('compare-locales.io')
-                    .error('%s: %s', e.strerror, e.filename))
-                continue
-            self.pc.add_child(child)
-
-    def resolvepath(self, path):
-        path = self.pc.expand(path, env=self.env)
-        path = mozpath.join(
-            mozpath.dirname(self.path),
-            self.data.get('basepath', '.'),
-            path)
-        return mozpath.normpath(path)
-
-    def asConfig(self):
-        return self.pc
-
-
-class L10nConfigParser(object):
-    '''Helper class to gather application information from ini files.
-
-    This class is working on synchronous open to read files or web data.
-    Subclass this and overwrite loadConfigs and addChild if you need async.
-    '''
-    def __init__(self, inipath, **kwargs):
-        """Constructor for L10nConfigParsers
-
-        inipath -- l10n.ini path
-        Optional keyword arguments are fowarded to the inner ConfigParser as
-        defaults.
-        """
-        self.inipath = mozpath.normpath(inipath)
-        # l10n.ini files can import other l10n.ini files, store the
-        # corresponding L10nConfigParsers
-        self.children = []
-        # we really only care about the l10n directories described in l10n.ini
-        self.dirs = []
-        # optional defaults to be passed to the inner ConfigParser (unused?)
-        self.defaults = kwargs
-
-    def getDepth(self, cp):
-        '''Get the depth for the comparison from the parsed l10n.ini.
-        '''
-        try:
-            depth = cp.get('general', 'depth')
-        except (NoSectionError, NoOptionError):
-            depth = '.'
-        return depth
-
-    def getFilters(self):
-        '''Get the test functions from this ConfigParser and all children.
-
-        Only works with synchronous loads, used by compare-locales, which
-        is local anyway.
-        '''
-        filter_path = mozpath.join(mozpath.dirname(self.inipath), 'filter.py')
-        try:
-            local = {}
-            with open(filter_path) as f:
-                exec(compile(f.read(), filter_path, 'exec'), {}, local)
-            if 'test' in local and callable(local['test']):
-                filters = [local['test']]
-            else:
-                filters = []
-        except BaseException:  # we really want to handle EVERYTHING here
-            filters = []
-
-        for c in self.children:
-            filters += c.getFilters()
-
-        return filters
-
-    def loadConfigs(self):
-        """Entry point to load the l10n.ini file this Parser refers to.
-
-        This implementation uses synchronous loads, subclasses might overload
-        this behaviour. If you do, make sure to pass a file-like object
-        to onLoadConfig.
-        """
-        cp = ConfigParser(self.defaults)
-        cp.read(self.inipath)
-        depth = self.getDepth(cp)
-        self.base = mozpath.join(mozpath.dirname(self.inipath), depth)
-        # create child loaders for any other l10n.ini files to be included
-        try:
-            for title, path in cp.items('includes'):
-                # skip default items
-                if title in self.defaults:
-                    continue
-                # add child config parser
-                self.addChild(title, path, cp)
-        except NoSectionError:
-            pass
-        # try to load the "dirs" defined in the "compare" section
-        try:
-            self.dirs.extend(cp.get('compare', 'dirs').split())
-        except (NoOptionError, NoSectionError):
-            pass
-        # try to set "all_path" and "all_url"
-        try:
-            self.all_path = mozpath.join(self.base, cp.get('general', 'all'))
-        except (NoOptionError, NoSectionError):
-            self.all_path = None
-        return cp
-
-    def addChild(self, title, path, orig_cp):
-        """Create a child L10nConfigParser and load it.
-
-        title -- indicates the module's name
-        path -- indicates the path to the module's l10n.ini file
-        orig_cp -- the configuration parser of this l10n.ini
-        """
-        cp = L10nConfigParser(mozpath.join(self.base, path), **self.defaults)
-        cp.loadConfigs()
-        self.children.append(cp)
-
-    def dirsIter(self):
-        """Iterate over all dirs and our base path for this l10n.ini"""
-        for dir in self.dirs:
-            yield dir, (self.base, dir)
-
-    def directories(self):
-        """Iterate over all dirs and base paths for this l10n.ini as well
-        as the included ones.
-        """
-        for t in self.dirsIter():
-            yield t
-        for child in self.children:
-            for t in child.directories():
-                yield t
-
-    def allLocales(self):
-        """Return a list of all the locales of this project"""
-        with open(self.all_path) as f:
-            return util.parseLocales(f.read())
-
-
-class SourceTreeConfigParser(L10nConfigParser):
-    '''Subclassing L10nConfigParser to work with just the repos
-    checked out next to each other instead of intermingled like
-    we do for real builds.
-    '''
-
-    def __init__(self, inipath, base, redirects):
-        '''Add additional arguments basepath.
-
-        basepath is used to resolve local paths via branchnames.
-        redirects is used in unified repository, mapping upstream
-        repos to local clones.
-        '''
-        L10nConfigParser.__init__(self, inipath)
-        self.base = base
-        self.redirects = redirects
-
-    def addChild(self, title, path, orig_cp):
-        # check if there's a section with details for this include
-        # we might have to check a different repo, or even VCS
-        # for example, projects like "mail" indicate in
-        # an "include_" section where to find the l10n.ini for "toolkit"
-        details = 'include_' + title
-        if orig_cp.has_section(details):
-            branch = orig_cp.get(details, 'mozilla')
-            branch = self.redirects.get(branch, branch)
-            inipath = orig_cp.get(details, 'l10n.ini')
-            path = mozpath.join(self.base, branch, inipath)
-        else:
-            path = mozpath.join(self.base, path)
-        cp = SourceTreeConfigParser(path, self.base, self.redirects,
-                                    **self.defaults)
-        cp.loadConfigs()
-        self.children.append(cp)
-
-
-class File(object):
-
-    def __init__(self, fullpath, file, module=None, locale=None):
-        self.fullpath = fullpath
-        self.file = file
-        self.module = module
-        self.locale = locale
-        pass
-
-    def getContents(self):
-        # open with universal line ending support and read
-        # ignore universal newlines deprecation
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            with open(self.fullpath, 'rbU') as f:
-                return f.read()
-
-    @property
-    def localpath(self):
-        f = self.file
-        if self.module:
-            f = mozpath.join(self.module, f)
-        return f
-
-    def __hash__(self):
-        return hash(self.localpath)
-
-    def __str__(self):
-        return self.fullpath
-
-    def __eq__(self, other):
-        if not isinstance(other, File):
-            return False
-        return vars(self) == vars(other)
-
-    def __ne__(self, other):
-        return not (self == other)
-
-
-class EnumerateApp(object):
-    reference = 'en-US'
-
-    def __init__(self, inipath, l10nbase, locales=None):
-        self.setupConfigParser(inipath)
-        self.modules = defaultdict(dict)
-        self.l10nbase = mozpath.abspath(l10nbase)
-        self.filters = []
-        self.addFilters(*self.config.getFilters())
-        self.locales = locales or self.config.allLocales()
-        self.locales.sort()
-
-    def setupConfigParser(self, inipath):
-        self.config = L10nConfigParser(inipath)
-        self.config.loadConfigs()
-
-    def addFilters(self, *args):
-        self.filters += args
-
-    def asConfig(self):
-        config = ProjectConfig()
-        self._config_for_ini(config, self.config)
-        filters = self.config.getFilters()
-        if filters:
-            config.set_filter_py(filters[0])
-        config.locales += self.locales
-        return config
-
-    def _config_for_ini(self, projectconfig, aConfig):
-        for k, (basepath, module) in aConfig.dirsIter():
-            paths = {
-                'module': module,
-                'reference': mozpath.normpath('%s/%s/locales/en-US/**' %
-                                              (basepath, module)),
-                'l10n': mozpath.normpath('{l10n_base}/{locale}/%s/**' %
-                                         module)
-            }
-            if module == 'mobile/android/base':
-                paths['test'] = ['android-dtd']
-            projectconfig.add_paths(paths)
-            projectconfig.add_global_environment(l10n_base=self.l10nbase)
-        for child in aConfig.children:
-            self._config_for_ini(projectconfig, child)
-
-
-class EnumerateSourceTreeApp(EnumerateApp):
-    '''Subclass EnumerateApp to work on side-by-side checked out
-    repos, and to no pay attention to how the source would actually
-    be checked out for building.
-    '''
-
-    def __init__(self, inipath, basepath, l10nbase, redirects,
-                 locales=None):
-        self.basepath = basepath
-        self.redirects = redirects
-        EnumerateApp.__init__(self, inipath, l10nbase, locales)
-
-    def setupConfigParser(self, inipath):
-        self.config = SourceTreeConfigParser(inipath, self.basepath,
-                                             self.redirects)
-        self.config.loadConfigs()

third_party/python/compare-locales/compare_locales/paths/__init__.py

@@ -0,0 +1,54 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+from compare_locales import mozpath
+from .files import ProjectFiles, REFERENCE_LOCALE
+from .ini import (
+    L10nConfigParser, SourceTreeConfigParser,
+    EnumerateApp, EnumerateSourceTreeApp,
+)
+from .matcher import Matcher
+from .project import ProjectConfig
+from .configparser import TOMLParser, ConfigNotFound
+
+
+__all__ = [
+    'Matcher',
+    'ProjectConfig',
+    'L10nConfigParser', 'SourceTreeConfigParser',
+    'EnumerateApp', 'EnumerateSourceTreeApp',
+    'ProjectFiles', 'REFERENCE_LOCALE',
+    'TOMLParser', 'ConfigNotFound',
+]
+
+
+class File(object):
+
+    def __init__(self, fullpath, file, module=None, locale=None):
+        self.fullpath = fullpath
+        self.file = file
+        self.module = module
+        self.locale = locale
+        pass
+
+    @property
+    def localpath(self):
+        if self.module:
+            return mozpath.join(self.locale, self.module, self.file)
+        return self.file
+
+    def __hash__(self):
+        return hash(self.localpath)
+
+    def __str__(self):
+        return self.fullpath
+
+    def __eq__(self, other):
+        if not isinstance(other, File):
+            return False
+        return vars(self) == vars(other)
+
+    def __ne__(self, other):
+        return not (self == other)

third_party/python/compare-locales/compare_locales/paths/configparser.py

@@ -0,0 +1,131 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import errno
+import logging
+from compare_locales import mozpath
+from .project import ProjectConfig
+from .matcher import expand
+import pytoml as toml
+import six
+
+
+class ConfigNotFound(EnvironmentError):
+    def __init__(self, path):
+        super(ConfigNotFound, self).__init__(
+            errno.ENOENT,
+            'Configuration file not found',
+            path)
+
+
+class ParseContext(object):
+    def __init__(self, path, env, ignore_missing_includes):
+        self.path = path
+        self.env = env
+        self.ignore_missing_includes = ignore_missing_includes
+        self.data = None
+        self.pc = ProjectConfig(path)
+
+
+class TOMLParser(object):
+    def parse(self, path, env=None, ignore_missing_includes=False):
+        ctx = self.context(
+            path, env=env, ignore_missing_includes=ignore_missing_includes
+        )
+        self.load(ctx)
+        self.processBasePath(ctx)
+        self.processEnv(ctx)
+        self.processPaths(ctx)
+        self.processFilters(ctx)
+        self.processIncludes(ctx)
+        self.processLocales(ctx)
+        return self.asConfig(ctx)
+
+    def context(self, path, env=None, ignore_missing_includes=False):
+        return ParseContext(
+            path,
+            env if env is not None else {},
+            ignore_missing_includes,
+        )
+
+    def load(self, ctx):
+        try:
+            with open(ctx.path, 'rb') as fin:
+                ctx.data = toml.load(fin)
+        except (toml.TomlError, IOError):
+            raise ConfigNotFound(ctx.path)
+
+    def processBasePath(self, ctx):
+        assert ctx.data is not None
+        ctx.pc.set_root(ctx.data.get('basepath', '.'))
+
+    def processEnv(self, ctx):
+        assert ctx.data is not None
+        ctx.pc.add_environment(**ctx.data.get('env', {}))
+        # add parser environment, possibly overwriting file variables
+        ctx.pc.add_environment(**ctx.env)
+
+    def processLocales(self, ctx):
+        assert ctx.data is not None
+        if 'locales' in ctx.data:
+            ctx.pc.set_locales(ctx.data['locales'])
+
+    def processPaths(self, ctx):
+        assert ctx.data is not None
+        for data in ctx.data.get('paths', []):
+            paths = {
+                "l10n": data['l10n']
+            }
+            if 'locales' in data:
+                paths['locales'] = data['locales']
+            if 'reference' in data:
+                paths['reference'] = data['reference']
+            if 'test' in data:
+                paths['test'] = data['test']
+            ctx.pc.add_paths(paths)
+
+    def processFilters(self, ctx):
+        assert ctx.data is not None
+        for data in ctx.data.get('filters', []):
+            paths = data['path']
+            if isinstance(paths, six.string_types):
+                paths = [paths]
+            rule = {
+                "path": paths,
+                "action": data['action']
+            }
+            if 'key' in data:
+                rule['key'] = data['key']
+            ctx.pc.add_rules(rule)
+
+    def processIncludes(self, ctx):
+        assert ctx.data is not None
+        if 'includes' not in ctx.data:
+            return
+        for include in ctx.data['includes']:
+            # resolve include['path'] against our root and env
+            p = mozpath.normpath(
+                expand(
+                    ctx.pc.root,
+                    include['path'],
+                    ctx.pc.environ
+                )
+            )
+            try:
+                child = self.parse(
+                    p, env=ctx.env,
+                    ignore_missing_includes=ctx.ignore_missing_includes
+                )
+            except ConfigNotFound as e:
+                if not ctx.ignore_missing_includes:
+                    raise
+                (logging
+                    .getLogger('compare-locales.io')
+                    .error('%s: %s', e.strerror, e.filename))
+                continue
+            ctx.pc.add_child(child)
+
+    def asConfig(self, ctx):
+        return ctx.pc

third_party/python/compare-locales/compare_locales/paths/files.py

@@ -0,0 +1,184 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import os
+from compare_locales import mozpath
+
+
+REFERENCE_LOCALE = 'en-x-moz-reference'
+
+
+class ProjectFiles(object):
+    '''Iterable object to get all files and tests for a locale and a
+    list of ProjectConfigs.
+
+    If the given locale is None, iterate over reference files as
+    both reference and locale for a reference self-test.
+    '''
+    def __init__(self, locale, projects, mergebase=None):
+        self.locale = locale
+        self.matchers = []
+        self.mergebase = mergebase
+        configs = []
+        for project in projects:
+            configs.extend(project.configs)
+        for pc in configs:
+            if locale and locale not in pc.locales:
+                continue
+            for paths in pc.paths:
+                if (
+                    locale and
+                    'locales' in paths and
+                    locale not in paths['locales']
+                ):
+                    continue
+                m = {
+                    'l10n': paths['l10n'].with_env({
+                        "locale": locale or REFERENCE_LOCALE
+                    }),
+                    'module': paths.get('module'),
+                }
+                if 'reference' in paths:
+                    m['reference'] = paths['reference']
+                if self.mergebase is not None:
+                    m['merge'] = paths['l10n'].with_env({
+                        "locale": locale,
+                        "l10n_base": self.mergebase
+                    })
+                m['test'] = set(paths.get('test', []))
+                if 'locales' in paths:
+                    m['locales'] = paths['locales'][:]
+                self.matchers.append(m)
+        self.matchers.reverse()  # we always iterate last first
+        # Remove duplicate patterns, comparing each matcher
+        # against all other matchers.
+        # Avoid n^2 comparisons by only scanning the upper triangle
+        # of a n x n matrix of all possible combinations.
+        # Using enumerate and keeping track of indexes, as we can't
+        # modify the list while iterating over it.
+        drops = set()  # duplicate matchers to remove
+        for i, m in enumerate(self.matchers[:-1]):
+            if i in drops:
+                continue  # we're dropping this anyway, don't search again
+            for i_, m_ in enumerate(self.matchers[(i+1):]):
+                if (mozpath.realpath(m['l10n'].prefix) !=
+                        mozpath.realpath(m_['l10n'].prefix)):
+                    # ok, not the same thing, continue
+                    continue
+                # check that we're comparing the same thing
+                if 'reference' in m:
+                    if (mozpath.realpath(m['reference'].prefix) !=
+                            mozpath.realpath(m_.get('reference').prefix)):
+                        raise RuntimeError('Mismatch in reference for ' +
+                                           mozpath.realpath(m['l10n'].prefix))
+                drops.add(i_ + i + 1)
+                m['test'] |= m_['test']
+        drops = sorted(drops, reverse=True)
+        for i in drops:
+            del self.matchers[i]
+
+    def __iter__(self):
+        # The iteration is pretty different when we iterate over
+        # a localization vs over the reference. We do that latter
+        # when running in validation mode.
+        inner = self.iter_locale() if self.locale else self.iter_reference()
+        for t in inner:
+            yield t
+
+    def iter_locale(self):
+        '''Iterate over locale files.'''
+        known = {}
+        for matchers in self.matchers:
+            matcher = matchers['l10n']
+            for path in self._files(matcher):
+                if path not in known:
+                    known[path] = {'test': matchers.get('test')}
+                    if 'reference' in matchers:
+                        known[path]['reference'] = matcher.sub(
+                            matchers['reference'], path)
+                    if 'merge' in matchers:
+                        known[path]['merge'] = matcher.sub(
+                            matchers['merge'], path)
+            if 'reference' not in matchers:
+                continue
+            matcher = matchers['reference']
+            for path in self._files(matcher):
+                l10npath = matcher.sub(matchers['l10n'], path)
+                if l10npath not in known:
+                    known[l10npath] = {
+                        'reference': path,
+                        'test': matchers.get('test')
+                    }
+                    if 'merge' in matchers:
+                        known[l10npath]['merge'] = \
+                            matcher.sub(matchers['merge'], path)
+        for path, d in sorted(known.items()):
+            yield (path, d.get('reference'), d.get('merge'), d['test'])
+
+    def iter_reference(self):
+        '''Iterate over reference files.'''
+        known = {}
+        for matchers in self.matchers:
+            if 'reference' not in matchers:
+                continue
+            matcher = matchers['reference']
+            for path in self._files(matcher):
+                refpath = matcher.sub(matchers['reference'], path)
+                if refpath not in known:
+                    known[refpath] = {
+                        'reference': path,
+                        'test': matchers.get('test')
+                    }
+        for path, d in sorted(known.items()):
+            yield (path, d.get('reference'), None, d['test'])
+
+    def _files(self, matcher):
+        '''Base implementation of getting all files in a hierarchy
+        using the file system.
+        Subclasses might replace this method to support different IO
+        patterns.
+        '''
+        base = matcher.prefix
+        if self._isfile(base):
+            if matcher.match(base) is not None:
+                yield base
+            return
+        for d, dirs, files in self._walk(base):
+            for f in files:
+                p = mozpath.join(d, f)
+                if matcher.match(p) is not None:
+                    yield p
+
+    def _isfile(self, path):
+        return os.path.isfile(path)
+
+    def _walk(self, base):
+        for d, dirs, files in os.walk(base):
+            yield d, dirs, files
+
+    def match(self, path):
+        '''Return the tuple of l10n_path, reference, mergepath, tests
+        if the given path matches any config, otherwise None.
+
+        This routine doesn't check that the files actually exist.
+        '''
+        for matchers in self.matchers:
+            matcher = matchers['l10n']
+            if matcher.match(path) is not None:
+                ref = merge = None
+                if 'reference' in matchers:
+                    ref = matcher.sub(matchers['reference'], path)
+                if 'merge' in matchers:
+                    merge = matcher.sub(matchers['merge'], path)
+                return path, ref, merge, matchers.get('test')
+            if 'reference' not in matchers:
+                continue
+            matcher = matchers['reference']
+            if matcher.match(path) is not None:
+                merge = None
+                l10n = matcher.sub(matchers['l10n'], path)
+                if 'merge' in matchers:
+                    merge = matcher.sub(matchers['merge'], path)
+                return l10n, path, merge, matchers.get('test')

third_party/python/compare-locales/compare_locales/paths/ini.py

@@ -0,0 +1,230 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
+from collections import defaultdict
+from compare_locales import util, mozpath
+from .project import ProjectConfig
+
+
+class L10nConfigParser(object):
+    '''Helper class to gather application information from ini files.
+
+    This class is working on synchronous open to read files or web data.
+    Subclass this and overwrite loadConfigs and addChild if you need async.
+    '''
+    def __init__(self, inipath, **kwargs):
+        """Constructor for L10nConfigParsers
+
+        inipath -- l10n.ini path
+        Optional keyword arguments are fowarded to the inner ConfigParser as
+        defaults.
+        """
+        self.inipath = mozpath.normpath(inipath)
+        # l10n.ini files can import other l10n.ini files, store the
+        # corresponding L10nConfigParsers
+        self.children = []
+        # we really only care about the l10n directories described in l10n.ini
+        self.dirs = []
+        # optional defaults to be passed to the inner ConfigParser (unused?)
+        self.defaults = kwargs
+
+    def getDepth(self, cp):
+        '''Get the depth for the comparison from the parsed l10n.ini.
+        '''
+        try:
+            depth = cp.get('general', 'depth')
+        except (NoSectionError, NoOptionError):
+            depth = '.'
+        return depth
+
+    def getFilters(self):
+        '''Get the test functions from this ConfigParser and all children.
+
+        Only works with synchronous loads, used by compare-locales, which
+        is local anyway.
+        '''
+        filter_path = mozpath.join(mozpath.dirname(self.inipath), 'filter.py')
+        try:
+            local = {}
+            with open(filter_path) as f:
+                exec(compile(f.read(), filter_path, 'exec'), {}, local)
+            if 'test' in local and callable(local['test']):
+                filters = [local['test']]
+            else:
+                filters = []
+        except BaseException:  # we really want to handle EVERYTHING here
+            filters = []
+
+        for c in self.children:
+            filters += c.getFilters()
+
+        return filters
+
+    def loadConfigs(self):
+        """Entry point to load the l10n.ini file this Parser refers to.
+
+        This implementation uses synchronous loads, subclasses might overload
+        this behaviour. If you do, make sure to pass a file-like object
+        to onLoadConfig.
+        """
+        cp = ConfigParser(self.defaults)
+        cp.read(self.inipath)
+        depth = self.getDepth(cp)
+        self.base = mozpath.join(mozpath.dirname(self.inipath), depth)
+        # create child loaders for any other l10n.ini files to be included
+        try:
+            for title, path in cp.items('includes'):
+                # skip default items
+                if title in self.defaults:
+                    continue
+                # add child config parser
+                self.addChild(title, path, cp)
+        except NoSectionError:
+            pass
+        # try to load the "dirs" defined in the "compare" section
+        try:
+            self.dirs.extend(cp.get('compare', 'dirs').split())
+        except (NoOptionError, NoSectionError):
+            pass
+        # try to set "all_path" and "all_url"
+        try:
+            self.all_path = mozpath.join(self.base, cp.get('general', 'all'))
+        except (NoOptionError, NoSectionError):
+            self.all_path = None
+        return cp
+
+    def addChild(self, title, path, orig_cp):
+        """Create a child L10nConfigParser and load it.
+
+        title -- indicates the module's name
+        path -- indicates the path to the module's l10n.ini file
+        orig_cp -- the configuration parser of this l10n.ini
+        """
+        cp = L10nConfigParser(mozpath.join(self.base, path), **self.defaults)
+        cp.loadConfigs()
+        self.children.append(cp)
+
+    def dirsIter(self):
+        """Iterate over all dirs and our base path for this l10n.ini"""
+        for dir in self.dirs:
+            yield dir, (self.base, dir)
+
+    def directories(self):
+        """Iterate over all dirs and base paths for this l10n.ini as well
+        as the included ones.
+        """
+        for t in self.dirsIter():
+            yield t
+        for child in self.children:
+            for t in child.directories():
+                yield t
+
+    def allLocales(self):
+        """Return a list of all the locales of this project"""
+        with open(self.all_path) as f:
+            return util.parseLocales(f.read())
+
+
+class SourceTreeConfigParser(L10nConfigParser):
+    '''Subclassing L10nConfigParser to work with just the repos
+    checked out next to each other instead of intermingled like
+    we do for real builds.
+    '''
+
+    def __init__(self, inipath, base, redirects):
+        '''Add additional arguments basepath.
+
+        basepath is used to resolve local paths via branchnames.
+        redirects is used in unified repository, mapping upstream
+        repos to local clones.
+        '''
+        L10nConfigParser.__init__(self, inipath)
+        self.base = base
+        self.redirects = redirects
+
+    def addChild(self, title, path, orig_cp):
+        # check if there's a section with details for this include
+        # we might have to check a different repo, or even VCS
+        # for example, projects like "mail" indicate in
+        # an "include_" section where to find the l10n.ini for "toolkit"
+        details = 'include_' + title
+        if orig_cp.has_section(details):
+            branch = orig_cp.get(details, 'mozilla')
+            branch = self.redirects.get(branch, branch)
+            inipath = orig_cp.get(details, 'l10n.ini')
+            path = mozpath.join(self.base, branch, inipath)
+        else:
+            path = mozpath.join(self.base, path)
+        cp = SourceTreeConfigParser(path, self.base, self.redirects,
+                                    **self.defaults)
+        cp.loadConfigs()
+        self.children.append(cp)
+
+
+class EnumerateApp(object):
+    reference = 'en-US'
+
+    def __init__(self, inipath, l10nbase, locales=None):
+        self.setupConfigParser(inipath)
+        self.modules = defaultdict(dict)
+        self.l10nbase = mozpath.abspath(l10nbase)
+        self.filters = []
+        self.addFilters(*self.config.getFilters())
+        self.locales = locales or self.config.allLocales()
+        self.locales.sort()
+
+    def setupConfigParser(self, inipath):
+        self.config = L10nConfigParser(inipath)
+        self.config.loadConfigs()
+
+    def addFilters(self, *args):
+        self.filters += args
+
+    def asConfig(self):
+        # We've already normalized paths in the ini parsing.
+        # Set the path and root to None to just keep our paths as is.
+        config = ProjectConfig(None)
+        config.set_root('.')  # sets to None because path is None
+        config.add_environment(l10n_base=self.l10nbase)
+        self._config_for_ini(config, self.config)
+        filters = self.config.getFilters()
+        if filters:
+            config.set_filter_py(filters[0])
+        config.locales += self.locales
+        return config
+
+    def _config_for_ini(self, projectconfig, aConfig):
+        for k, (basepath, module) in aConfig.dirsIter():
+            paths = {
+                'module': module,
+                'reference': mozpath.normpath('%s/%s/locales/en-US/**' %
+                                              (basepath, module)),
+                'l10n': mozpath.normpath('{l10n_base}/{locale}/%s/**' %
+                                         module)
+            }
+            if module == 'mobile/android/base':
+                paths['test'] = ['android-dtd']
+            projectconfig.add_paths(paths)
+        for child in aConfig.children:
+            self._config_for_ini(projectconfig, child)
+
+
+class EnumerateSourceTreeApp(EnumerateApp):
+    '''Subclass EnumerateApp to work on side-by-side checked out
+    repos, and to no pay attention to how the source would actually
+    be checked out for building.
+    '''
+
+    def __init__(self, inipath, basepath, l10nbase, redirects,
+                 locales=None):
+        self.basepath = basepath
+        self.redirects = redirects
+        EnumerateApp.__init__(self, inipath, l10nbase, locales)
+
+    def setupConfigParser(self, inipath):
+        self.config = SourceTreeConfigParser(inipath, self.basepath,
+                                             self.redirects)
+        self.config.loadConfigs()

third_party/python/compare-locales/compare_locales/paths/matcher.py

@@ -0,0 +1,459 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import os
+import re
+import itertools
+from compare_locales import mozpath
+import six
+
+
+# Android uses non-standard locale codes, these are the mappings
+# back and forth
+ANDROID_LEGACY_MAP = {
+    'he': 'iw',
+    'id': 'in',
+    'yi': 'ji'
+}
+ANDROID_STANDARD_MAP = {
+    legacy: standard
+    for standard, legacy in six.iteritems(ANDROID_LEGACY_MAP)
+}
+
+
+class Matcher(object):
+    '''Path pattern matcher
+    Supports path matching similar to mozpath.match(), but does
+    not match trailing file paths without trailing wildcards.
+    Also gets a prefix, which is the path before the first wildcard,
+    which is good for filesystem iterations, and allows to replace
+    the own matches in a path on a different Matcher. compare-locales
+    uses that to transform l10n and en-US paths back and forth.
+    '''
+
+    def __init__(self, pattern_or_other, env={}, root=None):
+        '''Create regular expression similar to mozpath.match().
+        '''
+        parser = PatternParser()
+        real_env = {k: parser.parse(v) for k, v in env.items()}
+        self._cached_re = None
+        if root is not None:
+            # make sure that our root is fully expanded and ends with /
+            root = mozpath.abspath(root) + '/'
+        # allow constructing Matchers from Matchers
+        if isinstance(pattern_or_other, Matcher):
+            other = pattern_or_other
+            self.pattern = Pattern(other.pattern)
+            self.env = other.env.copy()
+            self.env.update(real_env)
+            if root is not None:
+                self.pattern.root = root
+            return
+        self.env = real_env
+        pattern = pattern_or_other
+        self.pattern = parser.parse(pattern)
+        if root is not None:
+            self.pattern.root = root
+
+    def with_env(self, environ):
+        return Matcher(self, environ)
+
+    @property
+    def prefix(self):
+        subpattern = Pattern(self.pattern[:self.pattern.prefix_length])
+        subpattern.root = self.pattern.root
+        return subpattern.expand(self.env)
+
+    def match(self, path):
+        '''Test the given path against this matcher and its environment.
+
+        Return None if there's no match, and the dictionary of matched
+        variables in this matcher if there's a match.
+        '''
+        self._cache_regex()
+        m = self._cached_re.match(path)
+        if m is None:
+            return None
+        d = m.groupdict()
+        if 'android_locale' in d and 'locale' not in d:
+            # map android_locale to locale code
+            locale = d['android_locale']
+            # map legacy locale codes, he <-> iw, id <-> in, yi <-> ji
+            locale = re.sub(
+                r'(iw|in|ji)(?=\Z|-)',
+                lambda legacy: ANDROID_STANDARD_MAP[legacy.group(1)],
+                locale
+            )
+            locale = re.sub(r'-r([A-Z]{2})', r'-\1', locale)
+            locale = locale.replace('b+', '').replace('+', '-')
+            d['locale'] = locale
+        return d
+
+    def _cache_regex(self):
+        if self._cached_re is not None:
+            return
+        self._cached_re = re.compile(
+            self.pattern.regex_pattern(self.env) + '$'
+        )
+
+    def sub(self, other, path):
+        '''
+        Replace the wildcard matches in this pattern into the
+        pattern of the other Match object.
+        '''
+        m = self.match(path)
+        if m is None:
+            return None
+        env = {}
+        env.update(
+            (key, Literal(value))
+            for key, value in m.items()
+        )
+        env.update(other.env)
+        return other.pattern.expand(env)
+
+    def concat(self, other):
+        '''Concat two Matcher objects.
+
+        The intent is to create one Matcher with variable substitutions that
+        behaves as if you joined the resulting paths.
+        This doesn't do path separator logic, though, and it won't resolve
+        parent directories.
+        '''
+        if not isinstance(other, Matcher):
+            other_matcher = Matcher(other)
+        else:
+            other_matcher = other
+        other_pattern = other_matcher.pattern
+        if other_pattern.root is not None:
+            raise ValueError('Other matcher must not be rooted')
+        result = Matcher(self)
+        result.pattern += other_pattern
+        if self.pattern.prefix_length == len(self.pattern):
+            result.pattern.prefix_length += other_pattern.prefix_length
+        result.env.update(other_matcher.env)
+        return result
+
+    def __str__(self):
+        return self.pattern.expand(self.env)
+
+    def __repr__(self):
+        return '{}({!r}, env={!r}, root={!r})'.format(
+            type(self).__name__, self.pattern, self.env, self.pattern.root
+        )
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __eq__(self, other):
+        '''Equality for Matcher.
+
+        The equality for Matchers is defined to have the same pattern,
+        and no conflicting environment. Additional environment settings
+        in self or other are OK.
+        '''
+        if other.__class__ is not self.__class__:
+            return NotImplemented
+        if self.pattern != other.pattern:
+            return False
+        if self.env and other.env:
+            for k in self.env:
+                if k not in other.env:
+                    continue
+                if self.env[k] != other.env[k]:
+                    return False
+        return True
+
+
+def expand(root, path, env):
+    '''Expand a given path relative to the given root,
+    using the given env to resolve variables.
+
+    This will break if the path contains wildcards.
+    '''
+    matcher = Matcher(path, env=env, root=root)
+    return str(matcher)
+
+
+class MissingEnvironment(Exception):
+    pass
+
+
+class Node(object):
+    '''Abstract base class for all nodes in parsed patterns.'''
+    def regex_pattern(self, env):
+        '''Create a regular expression fragment for this Node.'''
+        raise NotImplementedError
+
+    def expand(self, env):
+        '''Convert this node to a string with the given environment.'''
+        raise NotImplementedError
+
+
+class Pattern(list, Node):
+    def __init__(self, iterable=[]):
+        list.__init__(self, iterable)
+        self.root = getattr(iterable, 'root', None)
+        self.prefix_length = getattr(iterable, 'prefix_length', None)
+
+    def regex_pattern(self, env):
+        root = ''
+        if self.root is not None:
+            # make sure we're not hiding a full path
+            first_seg = self[0].expand(env)
+            if not os.path.isabs(first_seg):
+                root = re.escape(self.root)
+        return root + ''.join(
+            child.regex_pattern(env) for child in self
+        )
+
+    def expand(self, env, raise_missing=False):
+        root = ''
+        if self.root is not None:
+            # make sure we're not hiding a full path
+            first_seg = self[0].expand(env)
+            if not os.path.isabs(first_seg):
+                root = self.root
+        return root + ''.join(self._expand_children(env, raise_missing))
+
+    def _expand_children(self, env, raise_missing):
+        # Helper iterator to convert Exception to a stopped iterator
+        for child in self:
+            try:
+                yield child.expand(env, raise_missing=True)
+            except MissingEnvironment:
+                if raise_missing:
+                    raise
+                return
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __eq__(self, other):
+        if not super(Pattern, self).__eq__(other):
+            return False
+        if other.__class__ == list:
+            # good for tests and debugging
+            return True
+        return (
+            self.root == other.root
+            and self.prefix_length == other.prefix_length
+        )
+
+
+class Literal(six.text_type, Node):
+    def regex_pattern(self, env):
+        return re.escape(self)
+
+    def expand(self, env, raise_missing=False):
+        return self
+
+
+class Variable(Node):
+    def __init__(self, name, repeat=False):
+        self.name = name
+        self.repeat = repeat
+
+    def regex_pattern(self, env):
+        if self.repeat:
+            return '(?P={})'.format(self.name)
+        return '(?P<{}>{})'.format(self.name, self._pattern_from_env(env))
+
+    def _pattern_from_env(self, env):
+        if self.name in env:
+            # make sure we match the value in the environment
+            return env[self.name].regex_pattern(self._no_cycle(env))
+        # match anything, including path segments
+        return '.+?'
+
+    def expand(self, env, raise_missing=False):
+        '''Create a string for this Variable.
+
+        This expansion happens recursively. We avoid recusion loops
+        by removing the current variable from the environment that's used
+        to expand child variable references.
+        '''
+        if self.name not in env:
+            raise MissingEnvironment
+        return env[self.name].expand(
+            self._no_cycle(env), raise_missing=raise_missing
+        )
+
+    def _no_cycle(self, env):
+        '''Remove our variable name from the environment.
+        That way, we can't create cyclic references.
+        '''
+        if self.name not in env:
+            return env
+        env = env.copy()
+        env.pop(self.name)
+        return env
+
+    def __repr__(self):
+        return 'Variable(name="{}")'.format(self.name)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __eq__(self, other):
+        if other.__class__ is not self.__class__:
+            return False
+        return (
+            self.name == other.name
+            and self.repeat == other.repeat
+        )
+
+
+class AndroidLocale(Variable):
+    '''Subclass for Android locale code mangling.
+
+    Supports ab-rCD and b+ab+Scrip+DE.
+    Language and Language-Region tags get mapped to ab-rCD, more complex
+    Locale tags to b+.
+    '''
+    def __init__(self, repeat=False):
+        self.name = 'android_locale'
+        self.repeat = repeat
+
+    def _pattern_from_env(self, env):
+        android_locale = self._get_android_locale(env)
+        if android_locale is not None:
+            return re.escape(android_locale)
+        return '.+?'
+
+    def expand(self, env, raise_missing=False):
+        '''Create a string for this Variable.
+
+        This expansion happens recursively. We avoid recusion loops
+        by removing the current variable from the environment that's used
+        to expand child variable references.
+        '''
+        android_locale = self._get_android_locale(env)
+        if android_locale is None:
+            raise MissingEnvironment
+        return android_locale
+
+    def _get_android_locale(self, env):
+        if 'locale' not in env:
+            return None
+        android = bcp47 = env['locale'].expand(self._no_cycle(env))
+        # map legacy locale codes, he <-> iw, id <-> in, yi <-> ji
+        android = bcp47 = re.sub(
+            r'(he|id|yi)(?=\Z|-)',
+            lambda standard: ANDROID_LEGACY_MAP[standard.group(1)],
+            bcp47
+        )
+        if re.match(r'[a-z]{2,3}-[A-Z]{2}', bcp47):
+            android = '{}-r{}'.format(*bcp47.split('-'))
+        elif '-' in bcp47:
+            android = 'b+' + bcp47.replace('-', '+')
+        return android
+
+
+class Star(Node):
+    def __init__(self, number):
+        self.number = number
+
+    def regex_pattern(self, env):
+        return '(?P<s{}>[^/]*)'.format(self.number)
+
+    def expand(self, env, raise_missing=False):
+        return env['s%d' % self.number]
+
+    def __repr__(self):
+        return type(self).__name__
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __eq__(self, other):
+        if other.__class__ is not self.__class__:
+            return False
+        return self.number == other.number
+
+
+class Starstar(Star):
+    def __init__(self, number, suffix):
+        self.number = number
+        self.suffix = suffix
+
+    def regex_pattern(self, env):
+        return '(?P<s{}>.+{})?'.format(self.number, self.suffix)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __eq__(self, other):
+        if not super(Starstar, self).__eq__(other):
+            return False
+        return self.suffix == other.suffix
+
+
+PATH_SPECIAL = re.compile(
+    r'(?P<starstar>(?<![^/}])\*\*(?P<suffix>/|$))'
+    r'|'
+    r'(?P<star>\*)'
+    r'|'
+    r'(?P<variable>{ *(?P<varname>[\w]+) *})'
+)
+
+
+class PatternParser(object):
+    def __init__(self):
+        # Not really initializing anything, just making room for our
+        # result and state members.
+        self.pattern = None
+        self._stargroup = self._cursor = None
+        self._known_vars = None
+
+    def parse(self, pattern):
+        if isinstance(pattern, Pattern):
+            return pattern
+        if isinstance(pattern, Matcher):
+            return pattern.pattern
+        # Initializing result and state
+        self.pattern = Pattern()
+        self._stargroup = itertools.count(1)
+        self._known_vars = set()
+        self._cursor = 0
+        for match in PATH_SPECIAL.finditer(pattern):
+            if match.start() > self._cursor:
+                self.pattern.append(
+                    Literal(pattern[self._cursor:match.start()])
+                )
+            self.handle(match)
+        self.pattern.append(Literal(pattern[self._cursor:]))
+        if self.pattern.prefix_length is None:
+            self.pattern.prefix_length = len(self.pattern)
+        return self.pattern
+
+    def handle(self, match):
+        if match.group('variable'):
+            self.variable(match)
+        else:
+            self.wildcard(match)
+        self._cursor = match.end()
+
+    def variable(self, match):
+        varname = match.group('varname')
+        # Special case Android locale code matching.
+        # It's kinda sad, but true.
+        if varname == 'android_locale':
+            self.pattern.append(AndroidLocale(varname in self._known_vars))
+        else:
+            self.pattern.append(Variable(varname, varname in self._known_vars))
+        self._known_vars.add(varname)
+
+    def wildcard(self, match):
+        # wildcard found, stop prefix
+        if self.pattern.prefix_length is None:
+            self.pattern.prefix_length = len(self.pattern)
+        wildcard = next(self._stargroup)
+        if match.group('star'):
+            # *
+            self.pattern.append(Star(wildcard))
+        else:
+            # **
+            self.pattern.append(Starstar(wildcard, match.group('suffix')))

third_party/python/compare-locales/compare_locales/paths/project.py

@@ -0,0 +1,223 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import re
+from compare_locales import mozpath
+from .matcher import Matcher
+import six
+
+
+class ProjectConfig(object):
+    '''Abstraction of l10n project configuration data.
+    '''
+
+    def __init__(self, path):
+        self.filter_py = None  # legacy filter code
+        # {
+        #  'l10n': pattern,
+        #  'reference': pattern,  # optional
+        #  'locales': [],  # optional
+        #  'test': [],  # optional
+        # }
+        self.path = path
+        self.root = None
+        self.paths = []
+        self.rules = []
+        self.locales = []
+        self.environ = {}
+        self.children = []
+        self._cache = None
+
+    def same(self, other):
+        '''Equality test, ignoring locales.
+        '''
+        if other.__class__ is not self.__class__:
+            return False
+        if len(self.children) != len(other.children):
+            return False
+        for prop in ('path', 'root', 'paths', 'rules', 'environ'):
+            if getattr(self, prop) != getattr(other, prop):
+                return False
+        for this_child, other_child in zip(self.children, other.children):
+            if not this_child.same(other_child):
+                return False
+        return True
+
+    def set_root(self, basepath):
+        if self.path is None:
+            self.root = None
+            return
+        self.root = mozpath.abspath(
+            mozpath.join(mozpath.dirname(self.path), basepath)
+        )
+
+    def add_environment(self, **kwargs):
+        self.environ.update(kwargs)
+
+    def add_paths(self, *paths):
+        '''Add path dictionaries to this config.
+        The dictionaries must have a `l10n` key. For monolingual files,
+        `reference` is also required.
+        An optional key `test` is allowed to enable additional tests for this
+        path pattern.
+        '''
+
+        for d in paths:
+            rv = {
+                'l10n': Matcher(d['l10n'], env=self.environ, root=self.root),
+                'module': d.get('module')
+            }
+            if 'reference' in d:
+                rv['reference'] = Matcher(
+                    d['reference'], env=self.environ, root=self.root
+                )
+            if 'test' in d:
+                rv['test'] = d['test']
+            if 'locales' in d:
+                rv['locales'] = d['locales'][:]
+            self.paths.append(rv)
+
+    def set_filter_py(self, filter_function):
+        '''Set legacy filter.py code.
+        Assert that no rules are set.
+        Also, normalize output already here.
+        '''
+        assert not self.rules
+
+        def filter_(module, path, entity=None):
+            try:
+                rv = filter_function(module, path, entity=entity)
+            except BaseException:  # we really want to handle EVERYTHING here
+                return 'error'
+            rv = {
+                True: 'error',
+                False: 'ignore',
+                'report': 'warning'
+            }.get(rv, rv)
+            assert rv in ('error', 'ignore', 'warning', None)
+            return rv
+        self.filter_py = filter_
+
+    def add_rules(self, *rules):
+        '''Add rules to filter on.
+        Assert that there's no legacy filter.py code hooked up.
+        '''
+        assert self.filter_py is None
+        for rule in rules:
+            self.rules.extend(self._compile_rule(rule))
+
+    def add_child(self, child):
+        self.children.append(child)
+
+    def set_locales(self, locales, deep=False):
+        self.locales = locales
+        for child in self.children:
+            if not child.locales or deep:
+                child.set_locales(locales, deep=deep)
+            else:
+                locs = [loc for loc in locales if loc in child.locales]
+                child.set_locales(locs)
+
+    @property
+    def configs(self):
+        'Recursively get all configs in this project and its children'
+        yield self
+        for child in self.children:
+            for config in child.configs:
+                yield config
+
+    def filter(self, l10n_file, entity=None):
+        '''Filter a localization file or entities within, according to
+        this configuration file.'''
+        if self.filter_py is not None:
+            return self.filter_py(l10n_file.module, l10n_file.file,
+                                  entity=entity)
+        rv = self._filter(l10n_file, entity=entity)
+        if rv is None:
+            return 'ignore'
+        return rv
+
+    class FilterCache(object):
+        def __init__(self, locale):
+            self.locale = locale
+            self.rules = []
+            self.l10n_paths = []
+
+    def cache(self, locale):
+        if self._cache and self._cache.locale == locale:
+            return self._cache
+        self._cache = self.FilterCache(locale)
+        for paths in self.paths:
+            self._cache.l10n_paths.append(paths['l10n'].with_env({
+                "locale": locale
+            }))
+        for rule in self.rules:
+            cached_rule = rule.copy()
+            cached_rule['path'] = rule['path'].with_env({
+                "locale": locale
+            })
+            self._cache.rules.append(cached_rule)
+        return self._cache
+
+    def _filter(self, l10n_file, entity=None):
+        actions = set(
+            child._filter(l10n_file, entity=entity)
+            for child in self.children)
+        if 'error' in actions:
+            # return early if we know we'll error
+            return 'error'
+
+        cached = self.cache(l10n_file.locale)
+        if any(p.match(l10n_file.fullpath) for p in cached.l10n_paths):
+            action = 'error'
+            for rule in reversed(cached.rules):
+                if not rule['path'].match(l10n_file.fullpath):
+                    continue
+                if ('key' in rule) ^ (entity is not None):
+                    # key/file mismatch, not a matching rule
+                    continue
+                if 'key' in rule and not rule['key'].match(entity):
+                    continue
+                action = rule['action']
+                break
+            actions.add(action)
+        if 'error' in actions:
+            return 'error'
+        if 'warning' in actions:
+            return 'warning'
+        if 'ignore' in actions:
+            return 'ignore'
+
+    def _compile_rule(self, rule):
+        assert 'path' in rule
+        if isinstance(rule['path'], list):
+            for path in rule['path']:
+                _rule = rule.copy()
+                _rule['path'] = Matcher(path, env=self.environ, root=self.root)
+                for __rule in self._compile_rule(_rule):
+                    yield __rule
+            return
+        if isinstance(rule['path'], six.string_types):
+            rule['path'] = Matcher(
+                rule['path'], env=self.environ, root=self.root
+            )
+        if 'key' not in rule:
+            yield rule
+            return
+        if not isinstance(rule['key'], six.string_types):
+            for key in rule['key']:
+                _rule = rule.copy()
+                _rule['key'] = key
+                for __rule in self._compile_rule(_rule):
+                    yield __rule
+            return
+        rule = rule.copy()
+        key = rule['key']
+        if key.startswith('re:'):
+            key = key[3:]
+        else:
+            key = re.escape(key) + '$'
+        rule['key'] = re.compile(key)
+        yield rule

third_party/python/compare-locales/compare_locales/serializer.py

@@ -0,0 +1,136 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''Serialize string changes.
+
+The serialization logic is based on the cross-channel merge algorithm.
+It's taking the file structure for the first file, and localizable entries
+from the last.
+Input data is the parsed reference as a list of parser.walk(),
+the existing localized file, also a list of parser.walk(), and a dictionary
+of newly added keys and raw values.
+To remove a string from a localization, pass `None` as value for a key.
+
+The marshalling between raw values and entities is done via Entity.unwrap
+and Entity.wrap.
+
+To avoid adding English reference strings into the generated file, the
+actual entities in the reference are replaced with Placeholders, which
+are removed in a final pass over the result of merge_resources. After that,
+we also prune whitespace once more.`
+'''
+
+from codecs import encode
+import six
+
+from compare_locales.merge import merge_resources, serialize_legacy_resource
+from compare_locales.parser import getParser
+from compare_locales.parser.base import (
+    Entity,
+    PlaceholderEntity,
+    Junk,
+    Whitespace,
+)
+
+
+class SerializationNotSupportedError(ValueError):
+    pass
+
+
+def serialize(filename, reference, old_l10n, new_data):
+    '''Returns a byte string of the serialized content to use.
+
+    Input are a filename to create the right parser, a reference and
+    an existing localization, both as the result of parser.walk().
+    Finally, new_data is a dictionary of key to raw values to serialize.
+
+    Raises a SerializationNotSupportedError if we don't support the file
+    format.
+    '''
+    try:
+        parser = getParser(filename)
+    except UserWarning:
+        raise SerializationNotSupportedError(
+            'Unsupported file format ({}).'.format(filename))
+    # create template, whitespace and all
+    placeholders = [
+        placeholder(entry) for entry in reference
+        if not isinstance(entry, Junk)
+    ]
+    ref_mapping = {
+        entry.key: entry
+        for entry in reference
+        if isinstance(entry, Entity)
+    }
+    # strip obsolete strings
+    old_l10n = sanitize_old(ref_mapping.keys(), old_l10n, new_data)
+    # create new Entities
+    # .val can just be "", merge_channels doesn't need that
+    new_l10n = []
+    for key, new_raw_val in six.iteritems(new_data):
+        if new_raw_val is None or key not in ref_mapping:
+            continue
+        ref_ent = ref_mapping[key]
+        new_l10n.append(ref_ent.wrap(new_raw_val))
+
+    merged = merge_resources(
+        parser,
+        [placeholders, old_l10n, new_l10n],
+        keep_newest=False
+    )
+    pruned = prune_placeholders(merged)
+    return encode(serialize_legacy_resource(pruned), parser.encoding)
+
+
+def sanitize_old(known_keys, old_l10n, new_data):
+    """Strip Junk and replace obsolete messages with placeholders.
+    If new_data has `None` as a value, strip the existing translation.
+    Use placeholders generously, so that we can rely on `prune_placeholders`
+    to find their associated comments and remove them, too.
+    """
+
+    def should_placeholder(entry):
+        # If entry is an Entity, check if it's obsolete
+        # or marked to be removed.
+        if not isinstance(entry, Entity):
+            return False
+        if entry.key not in known_keys:
+            return True
+        return entry.key in new_data and new_data[entry.key] is None
+
+    return [
+        placeholder(entry)
+        if should_placeholder(entry)
+        else entry
+        for entry in old_l10n
+        if not isinstance(entry, Junk)
+    ]
+
+
+def placeholder(entry):
+    if isinstance(entry, Entity):
+        return PlaceholderEntity(entry.key)
+    return entry
+
+
+def prune_placeholders(entries):
+    pruned = [
+        entry for entry in entries
+        if not isinstance(entry, PlaceholderEntity)
+    ]
+
+    def prune_whitespace(acc, entity):
+        if len(acc) and isinstance(entity, Whitespace):
+            prev_entity = acc[-1]
+
+            if isinstance(prev_entity, Whitespace):
+                # Prefer the longer whitespace.
+                if len(entity.all) > len(prev_entity.all):
+                    acc[-1] = entity
+                return acc
+
+        acc.append(entity)
+        return acc
+
+    return six.moves.reduce(prune_whitespace, pruned, [])

third_party/python/compare-locales/compare_locales/tests/__init__.py

@@ -50,6 +50,8 @@ class ParserTestMixin():
             if isinstance(entity, parser.Entity):
                 self.assertEqual(entity.key, ref[0])
                 self.assertEqual(entity.val, ref[1])
+                if len(ref) == 3:
+                    self.assertIn(ref[2], entity.pre_comment.val)
             else:
                 self.assertIsInstance(entity, ref[0])
                 self.assertIn(ref[1], entity.all)

third_party/python/compare-locales/compare_locales/tests/android/test_checks.py

@@ -16,6 +16,61 @@ ANDROID_WRAPPER = b'''<?xml version="1.0" encoding="utf-8"?>
 '''
 
 
+class SimpleStringsTest(BaseHelper):
+    file = File('values/strings.xml', 'values/strings.xml')
+    refContent = ANDROID_WRAPPER % b'plain'
+
+    def test_simple_string(self):
+        self._test(
+            ANDROID_WRAPPER % b'foo',
+            tuple()
+        )
+
+    def test_empty_string(self):
+        self._test(
+            ANDROID_WRAPPER % b'',
+            tuple()
+        )
+
+    def test_single_cdata(self):
+        self._test(
+            ANDROID_WRAPPER % b'<![CDATA[text]]>',
+            tuple()
+        )
+        self._test(
+            ANDROID_WRAPPER % b'<![CDATA[\n  text\n  ]]>',
+            tuple()
+        )
+
+    def test_mix_cdata(self):
+        self._test(
+            ANDROID_WRAPPER % b'<![CDATA[text]]> with <![CDATA[cdatas]]>',
+            (
+                (
+                    "error",
+                    0,
+                    "Only plain text allowed, "
+                    "or one CDATA surrounded by whitespace",
+                    "android"
+                 ),
+            )
+        )
+
+    def test_element_fails(self):
+        self._test(
+            ANDROID_WRAPPER % b'one<br/>two',
+            (
+                (
+                    "error",
+                    0,
+                    "Only plain text allowed, "
+                    "or one CDATA surrounded by whitespace",
+                    "android"
+                 ),
+            )
+        )
+
+
 class QuotesTest(BaseHelper):
     file = File('values/strings.xml', 'values/strings.xml')
     refContent = ANDROID_WRAPPER % b'plain'
@@ -100,3 +155,176 @@ class TranslatableTest(BaseHelper):
                 ),
             )
         )
+
+
+class AtStringTest(BaseHelper):
+    file = File('values/strings.xml', 'values/strings.xml')
+    refContent = (ANDROID_WRAPPER % b'@string/foo')
+
+    def test_translatable(self):
+        self._test(
+            ANDROID_WRAPPER % b'"some"',
+            (
+                (
+                    "warning",
+                    0,
+                    "strings must be translatable",
+                    "android"
+                ),
+            )
+        )
+
+
+class PrintfSTest(BaseHelper):
+    file = File('values/strings.xml', 'values/strings.xml')
+    refContent = ANDROID_WRAPPER % b'%s'
+
+    def test_match(self):
+        self._test(
+            ANDROID_WRAPPER % b'"%s"',
+            tuple()
+        )
+        self._test(
+            ANDROID_WRAPPER % b'"%1$s"',
+            tuple()
+        )
+        self._test(
+            ANDROID_WRAPPER % b'"$s %1$s"',
+            tuple()
+        )
+        self._test(
+            ANDROID_WRAPPER % b'"$1$s %1$s"',
+            tuple()
+        )
+
+    def test_mismatch(self):
+        self._test(
+            ANDROID_WRAPPER % b'"%d"',
+            (
+                (
+                    "error",
+                    0,
+                    "Mismatching formatter",
+                    "android"
+                ),
+            )
+        )
+        self._test(
+            ANDROID_WRAPPER % b'"%S"',
+            (
+                (
+                    "error",
+                    0,
+                    "Mismatching formatter",
+                    "android"
+                ),
+            )
+        )
+
+    def test_off_position(self):
+        self._test(
+            ANDROID_WRAPPER % b'%2$s',
+            (
+                (
+                    "error",
+                    0,
+                    "Formatter %2$s not found in reference",
+                    "android"
+                ),
+            )
+        )
+
+
+class PrintfCapSTest(BaseHelper):
+    file = File('values/strings.xml', 'values/strings.xml')
+    refContent = ANDROID_WRAPPER % b'%S'
+
+    def test_match(self):
+        self._test(
+            ANDROID_WRAPPER % b'"%S"',
+            tuple()
+        )
+
+    def test_mismatch(self):
+        self._test(
+            ANDROID_WRAPPER % b'"%s"',
+            (
+                (
+                    "error",
+                    0,
+                    "Mismatching formatter",
+                    "android"
+                ),
+            )
+        )
+        self._test(
+            ANDROID_WRAPPER % b'"%d"',
+            (
+                (
+                    "error",
+                    0,
+                    "Mismatching formatter",
+                    "android"
+                ),
+            )
+        )
+
+
+class PrintfDTest(BaseHelper):
+    file = File('values/strings.xml', 'values/strings.xml')
+    refContent = ANDROID_WRAPPER % b'%d'
+
+    def test_match(self):
+        self._test(
+            ANDROID_WRAPPER % b'"%d"',
+            tuple()
+        )
+        self._test(
+            ANDROID_WRAPPER % b'"%1$d"',
+            tuple()
+        )
+        self._test(
+            ANDROID_WRAPPER % b'"$d %1$d"',
+            tuple()
+        )
+        self._test(
+            ANDROID_WRAPPER % b'"$1$d %1$d"',
+            tuple()
+        )
+
+    def test_mismatch(self):
+        self._test(
+            ANDROID_WRAPPER % b'"%s"',
+            (
+                (
+                    "error",
+                    0,
+                    "Mismatching formatter",
+                    "android"
+                ),
+            )
+        )
+        self._test(
+            ANDROID_WRAPPER % b'"%S"',
+            (
+                (
+                    "error",
+                    0,
+                    "Mismatching formatter",
+                    "android"
+                ),
+            )
+        )
+
+    def test_off_position(self):
+        self._test(
+            ANDROID_WRAPPER % b'%2$d',
+            (
+                (
+                    "error",
+                    0,
+                    "Formatter %2$d not found in reference",
+                    "android"
+                ),
+            )
+        )

third_party/python/compare-locales/compare_locales/tests/android/test_merge.py

@@ -24,7 +24,7 @@ class TestMerge(unittest.TestCase):
 </resources>
 ''')
         self.assertEqual(
-            merge_channels(self.name, *channels), b'''\
+            merge_channels(self.name, channels), b'''\
 <?xml version="1.0" encoding="utf-8"?>
 <resources>
   <!-- bar -->
@@ -47,7 +47,7 @@ class TestMerge(unittest.TestCase):
 </resources>
 ''')
         self.assertEqual(
-            merge_channels(self.name, *channels), b'''\
+            merge_channels(self.name, channels), b'''\
 <?xml version="1.0" encoding="utf-8"?>
 <resources>
   <!-- Bar -->

third_party/python/compare-locales/compare_locales/tests/android/test_parser.py

@@ -26,6 +26,13 @@ class TestAndroidParser(ParserTestMixin, unittest.TestCase):
 <resources>
   <!-- bar -->
   <string name="foo">value</string>
+  <!-- bar -->
+  <!-- foo -->
+  <string name="bar">multi-line comment</string>
+
+  <!-- standalone -->
+
+  <string name="baz">so lonely</string>
 </resources>
 '''
         self._test(
@@ -33,9 +40,13 @@ class TestAndroidParser(ParserTestMixin, unittest.TestCase):
             (
                 (DocumentWrapper, '<?xml'),
                 (Whitespace, '\n  '),
-                (Comment, ' bar '),
+                ('foo', 'value', 'bar'),
+                (Whitespace, '\n'),
+                ('bar', 'multi-line comment', 'bar\nfoo'),
                 (Whitespace, '\n  '),
-                ('foo', 'value'),
+                (Comment, 'standalone'),
+                (Whitespace, '\n  '),
+                ('baz', 'so lonely'),
                 (Whitespace, '\n'),
                 (DocumentWrapper, '</resources>')
             )
@@ -91,3 +102,24 @@ class TestAndroidParser(ParserTestMixin, unittest.TestCase):
                 (Junk, 'no xml'),
             )
         )
+
+    def test_empty_strings(self):
+        source = '''\
+<?xml version="1.0" ?>
+<resources>
+  <string name="one"></string>
+  <string name="two"/>
+</resources>
+'''
+        self._test(
+            source,
+            (
+                (DocumentWrapper, '<?xml'),
+                (Whitespace, '\n  '),
+                ('one', ''),
+                (Whitespace, '\n  '),
+                ('two', ''),
+                (Whitespace, '\n'),
+                (DocumentWrapper, '</resources>')
+            )
+        )

third_party/python/compare-locales/compare_locales/tests/paths/__init__.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import tempfile
+from compare_locales.paths import (
+    ProjectConfig, File, ProjectFiles, TOMLParser
+)
+from compare_locales import mozpath
+import pytoml as toml
+
+
+class Rooted(object):
+    def setUp(self):
+        # Use tempdir as self.root, that's absolute on all platforms
+        self.root = mozpath.normpath(tempfile.gettempdir())
+
+    def path(self, leaf=''):
+        return self.root + leaf
+
+
+class SetupMixin(object):
+    def setUp(self):
+        self.cfg = ProjectConfig(None)
+        self.file = File(
+            '/tmp/somedir/de/browser/one/two/file.ftl',
+            'file.ftl',
+            module='browser', locale='de')
+        self.other_file = File(
+            '/tmp/somedir/de/toolkit/two/one/file.ftl',
+            'file.ftl',
+            module='toolkit', locale='de')
+
+
+class MockNode(object):
+    def __init__(self, name):
+        self.name = name
+        self.files = []
+        self.dirs = {}
+
+    def walk(self):
+        subdirs = sorted(self.dirs)
+        if self.name is not None:
+            yield self.name, subdirs, self.files
+        for subdir in subdirs:
+            child = self.dirs[subdir]
+            for tpl in child.walk():
+                yield tpl
+
+
+class MockProjectFiles(ProjectFiles):
+    def __init__(self, mocks, locale, projects, mergebase=None):
+        (super(MockProjectFiles, self)
+            .__init__(locale, projects, mergebase=mergebase))
+        self.mocks = mocks
+
+    def _isfile(self, path):
+        return path in self.mocks
+
+    def _walk(self, base):
+        base = mozpath.normpath(base)
+        local_files = [
+            mozpath.split(mozpath.relpath(f, base))
+            for f in self.mocks if f.startswith(base)
+        ]
+        root = MockNode(base)
+        for segs in local_files:
+            node = root
+            for n, seg in enumerate(segs[:-1]):
+                if seg not in node.dirs:
+                    node.dirs[seg] = MockNode('/'.join([base] + segs[:n+1]))
+                node = node.dirs[seg]
+            node.files.append(segs[-1])
+        for tpl in root.walk():
+            yield tpl
+
+
+class MockTOMLParser(TOMLParser):
+    def __init__(self, mock_data):
+        self.mock_data = mock_data
+
+    def load(self, ctx):
+        p = mozpath.basename(ctx.path)
+        ctx.data = toml.loads(self.mock_data[p])

third_party/python/compare-locales/compare_locales/tests/paths/test_configparser.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+import unittest
+
+from . import MockTOMLParser
+from compare_locales.paths.matcher import Matcher
+from compare_locales.paths.project import ProjectConfig
+from compare_locales import mozpath
+
+
+class TestConfigParser(unittest.TestCase):
+    def test_imports(self):
+        parser = MockTOMLParser({
+            "root.toml": """
+basepath = "."
+[env]
+  o = "toolkit"
+[[includes]]
+  path = "{o}/other.toml"
+""",
+            "other.toml": """
+basepath = "."
+"""
+        })
+        config = parser.parse("root.toml")
+        self.assertIsInstance(config, ProjectConfig)
+        configs = list(config.configs)
+        self.assertEqual(configs[0], config)
+        self.assertListEqual(
+            [c.path for c in configs],
+            ["root.toml", mozpath.abspath("toolkit/other.toml")]
+        )
+
+    def test_paths(self):
+        parser = MockTOMLParser({
+            "l10n.toml": """
+[[paths]]
+  l10n = "some/{locale}/*"
+""",
+            "ref.toml": """
+[[paths]]
+  reference = "ref/l10n/*"
+  l10n = "some/{locale}/*"
+""",
+            "tests.toml": """
+[[paths]]
+  l10n = "some/{locale}/*"
+  test = [
+    "run_this",
+  ]
+""",
+        })
+
+        paths = parser.parse("l10n.toml").paths
+        self.assertIn("l10n", paths[0])
+        self.assertIsInstance(paths[0]["l10n"], Matcher)
+        self.assertNotIn("reference", paths[0])
+        self.assertNotIn("test", paths[0])
+        paths = parser.parse("ref.toml").paths
+        self.assertIn("l10n", paths[0])
+        self.assertIsInstance(paths[0]["l10n"], Matcher)
+        self.assertIn("reference", paths[0])
+        self.assertIsInstance(paths[0]["reference"], Matcher)
+        self.assertNotIn("test", paths[0])
+        paths = parser.parse("tests.toml").paths
+        self.assertIn("l10n", paths[0])
+        self.assertIsInstance(paths[0]["l10n"], Matcher)
+        self.assertNotIn("reference", paths[0])
+        self.assertIn("test", paths[0])
+        self.assertListEqual(paths[0]["test"], ["run_this"])

third_party/python/compare-locales/compare_locales/tests/paths/test_files.py

@@ -0,0 +1,291 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import unittest
+
+from compare_locales.paths import (
+    ProjectConfig
+)
+from . import (
+    MockProjectFiles,
+    MockTOMLParser,
+    Rooted,
+)
+
+
+class TestProjectPaths(Rooted, unittest.TestCase):
+    def test_l10n_path(self):
+        cfg = ProjectConfig(None)
+        cfg.add_environment(l10n_base=self.root)
+        cfg.locales.append('de')
+        cfg.add_paths({
+            'l10n': '{l10n_base}/{locale}/*'
+        })
+        mocks = [
+            self.path(leaf)
+            for leaf in (
+                '/de/good.ftl',
+                '/de/not/subdir/bad.ftl',
+                '/fr/good.ftl',
+                '/fr/not/subdir/bad.ftl',
+            )
+        ]
+        files = MockProjectFiles(mocks, 'de', [cfg])
+        self.assertListEqual(
+            list(files),
+            [
+                (self.path('/de/good.ftl'), None, None, set())
+            ]
+        )
+        self.assertTupleEqual(
+            files.match(self.path('/de/good.ftl')),
+            (self.path('/de/good.ftl'), None, None, set())
+        )
+        self.assertIsNone(files.match(self.path('/fr/something.ftl')))
+        files = MockProjectFiles(mocks, 'de', [cfg], mergebase='merging')
+        self.assertListEqual(
+            list(files),
+            [
+                (self.path('/de/good.ftl'), None, 'merging/de/good.ftl', set())
+            ]
+        )
+        self.assertTupleEqual(
+            files.match(self.path('/de/something.ftl')),
+            (self.path('/de/something.ftl'),
+             None,
+             'merging/de/something.ftl',
+             set()))
+        # 'fr' is not in the locale list, should return no files
+        files = MockProjectFiles(mocks, 'fr', [cfg])
+        self.assertListEqual(list(files), [])
+
+    def test_single_reference_path(self):
+        cfg = ProjectConfig(None)
+        cfg.add_environment(l10n_base=self.path('/l10n'))
+        cfg.locales.append('de')
+        cfg.add_paths({
+            'l10n': '{l10n_base}/{locale}/good.ftl',
+            'reference': self.path('/reference/good.ftl')
+        })
+        mocks = [
+            self.path('/reference/good.ftl'),
+            self.path('/reference/not/subdir/bad.ftl'),
+        ]
+        files = MockProjectFiles(mocks, 'de', [cfg])
+        self.assertListEqual(
+            list(files),
+            [
+                (self.path('/l10n/de/good.ftl'),
+                 self.path('/reference/good.ftl'),
+                 None,
+                 set()),
+            ])
+        self.assertTupleEqual(
+            files.match(self.path('/reference/good.ftl')),
+            (self.path('/l10n/de/good.ftl'),
+             self.path('/reference/good.ftl'),
+             None,
+             set()),
+            )
+        self.assertTupleEqual(
+            files.match(self.path('/l10n/de/good.ftl')),
+            (self.path('/l10n/de/good.ftl'),
+             self.path('/reference/good.ftl'),
+             None,
+             set()),
+            )
+
+    def test_reference_path(self):
+        cfg = ProjectConfig(None)
+        cfg.add_environment(l10n_base=self.path('/l10n'))
+        cfg.locales.append('de')
+        cfg.add_paths({
+            'l10n': '{l10n_base}/{locale}/*',
+            'reference': self.path('/reference/*')
+        })
+        mocks = [
+            self.path(leaf)
+            for leaf in [
+                '/l10n/de/good.ftl',
+                '/l10n/de/not/subdir/bad.ftl',
+                '/l10n/fr/good.ftl',
+                '/l10n/fr/not/subdir/bad.ftl',
+                '/reference/ref.ftl',
+                '/reference/not/subdir/bad.ftl',
+            ]
+        ]
+        files = MockProjectFiles(mocks, 'de', [cfg])
+        self.assertListEqual(
+            list(files),
+            [
+                (self.path('/l10n/de/good.ftl'),
+                 self.path('/reference/good.ftl'),
+                 None,
+                 set()),
+                (self.path('/l10n/de/ref.ftl'),
+                 self.path('/reference/ref.ftl'),
+                 None,
+                 set()),
+            ])
+        self.assertTupleEqual(
+            files.match(self.path('/l10n/de/good.ftl')),
+            (self.path('/l10n/de/good.ftl'),
+             self.path('/reference/good.ftl'),
+             None,
+             set()),
+            )
+        self.assertTupleEqual(
+            files.match(self.path('/reference/good.ftl')),
+            (self.path('/l10n/de/good.ftl'),
+             self.path('/reference/good.ftl'),
+             None,
+             set()),
+            )
+        self.assertIsNone(files.match(self.path('/l10n/de/subdir/bad.ftl')))
+        self.assertIsNone(files.match(self.path('/reference/subdir/bad.ftl')))
+        files = MockProjectFiles(mocks, 'de', [cfg], mergebase='merging')
+        self.assertListEqual(
+            list(files),
+            [
+                (self.path('/l10n/de/good.ftl'),
+                 self.path('/reference/good.ftl'),
+                 'merging/de/good.ftl', set()),
+                (self.path('/l10n/de/ref.ftl'),
+                 self.path('/reference/ref.ftl'),
+                 'merging/de/ref.ftl', set()),
+            ])
+        self.assertTupleEqual(
+            files.match(self.path('/l10n/de/good.ftl')),
+            (self.path('/l10n/de/good.ftl'),
+             self.path('/reference/good.ftl'),
+             'merging/de/good.ftl', set()),
+            )
+        self.assertTupleEqual(
+            files.match(self.path('/reference/good.ftl')),
+            (self.path('/l10n/de/good.ftl'),
+             self.path('/reference/good.ftl'),
+             'merging/de/good.ftl', set()),
+            )
+        # 'fr' is not in the locale list, should return no files
+        files = MockProjectFiles(mocks, 'fr', [cfg])
+        self.assertListEqual(list(files), [])
+
+    def test_partial_l10n(self):
+        cfg = ProjectConfig(None)
+        cfg.locales.extend(['de', 'fr'])
+        cfg.add_paths({
+            'l10n': self.path('/{locale}/major/*')
+        }, {
+            'l10n': self.path('/{locale}/minor/*'),
+            'locales': ['de']
+        })
+        mocks = [
+            self.path(leaf)
+            for leaf in [
+                '/de/major/good.ftl',
+                '/de/major/not/subdir/bad.ftl',
+                '/de/minor/good.ftl',
+                '/fr/major/good.ftl',
+                '/fr/major/not/subdir/bad.ftl',
+                '/fr/minor/good.ftl',
+            ]
+        ]
+        files = MockProjectFiles(mocks, 'de', [cfg])
+        self.assertListEqual(
+            list(files),
+            [
+                (self.path('/de/major/good.ftl'), None, None, set()),
+                (self.path('/de/minor/good.ftl'), None, None, set()),
+            ])
+        self.assertTupleEqual(
+            files.match(self.path('/de/major/some.ftl')),
+            (self.path('/de/major/some.ftl'), None, None, set()))
+        self.assertIsNone(files.match(self.path('/de/other/some.ftl')))
+        # 'fr' is not in the locale list of minor, should only return major
+        files = MockProjectFiles(mocks, 'fr', [cfg])
+        self.assertListEqual(
+            list(files),
+            [
+                (self.path('/fr/major/good.ftl'), None, None, set()),
+            ])
+        self.assertIsNone(files.match(self.path('/fr/minor/some.ftl')))
+
+    def test_validation_mode(self):
+        cfg = ProjectConfig(None)
+        cfg.add_environment(l10n_base=self.path('/l10n'))
+        cfg.locales.append('de')
+        cfg.add_paths({
+            'l10n': '{l10n_base}/{locale}/*',
+            'reference': self.path('/reference/*')
+        })
+        mocks = [
+            self.path(leaf)
+            for leaf in [
+                '/l10n/de/good.ftl',
+                '/l10n/de/not/subdir/bad.ftl',
+                '/l10n/fr/good.ftl',
+                '/l10n/fr/not/subdir/bad.ftl',
+                '/reference/ref.ftl',
+                '/reference/not/subdir/bad.ftl',
+            ]
+        ]
+        # `None` switches on validation mode
+        files = MockProjectFiles(mocks, None, [cfg])
+        self.assertListEqual(
+            list(files),
+            [
+                (self.path('/reference/ref.ftl'),
+                 self.path('/reference/ref.ftl'),
+                 None,
+                 set()),
+            ])
+
+
+class TestL10nMerge(Rooted, unittest.TestCase):
+    # need to go through TOMLParser, as that's handling most of the
+    # environment
+    def test_merge_paths(self):
+        parser = MockTOMLParser({
+            "base.toml":
+            '''\
+basepath = "."
+locales = [
+    "de",
+]
+[env]
+    l = "{l10n_base}/{locale}/"
+[[paths]]
+    reference = "reference/*"
+    l10n = "{l}*"
+'''})
+        cfg = parser.parse(
+            self.path('/base.toml'),
+            env={'l10n_base': self.path('/l10n')}
+        )
+        mocks = [
+            self.path(leaf)
+            for leaf in [
+                '/l10n/de/good.ftl',
+                '/l10n/de/not/subdir/bad.ftl',
+                '/l10n/fr/good.ftl',
+                '/l10n/fr/not/subdir/bad.ftl',
+                '/reference/ref.ftl',
+                '/reference/not/subdir/bad.ftl',
+            ]
+        ]
+        files = MockProjectFiles(mocks, 'de', [cfg], self.path('/mergers'))
+        self.assertListEqual(
+            list(files),
+            [
+                (self.path('/l10n/de/good.ftl'),
+                 self.path('/reference/good.ftl'),
+                 self.path('/mergers/de/good.ftl'),
+                 set()),
+                (self.path('/l10n/de/ref.ftl'),
+                 self.path('/reference/ref.ftl'),
+                 self.path('/mergers/de/ref.ftl'),
+                 set()),
+            ])

third_party/python/compare-locales/compare_locales/tests/paths/test_ini.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import unittest
+
+from . import (
+    SetupMixin,
+)
+
+
+class TestConfigLegacy(SetupMixin, unittest.TestCase):
+
+    def test_filter_py_true(self):
+        'Test filter.py just return bool(True)'
+        def filter(mod, path, entity=None):
+            return True
+        self.cfg.set_filter_py(filter)
+        with self.assertRaises(AssertionError):
+            self.cfg.add_rules({})
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'error')
+        rv = self.cfg.filter(self.file, entity='one_entity')
+        self.assertEqual(rv, 'error')
+
+    def test_filter_py_false(self):
+        'Test filter.py just return bool(False)'
+        def filter(mod, path, entity=None):
+            return False
+        self.cfg.set_filter_py(filter)
+        with self.assertRaises(AssertionError):
+            self.cfg.add_rules({})
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.file, entity='one_entity')
+        self.assertEqual(rv, 'ignore')
+
+    def test_filter_py_error(self):
+        'Test filter.py just return str("error")'
+        def filter(mod, path, entity=None):
+            return 'error'
+        self.cfg.set_filter_py(filter)
+        with self.assertRaises(AssertionError):
+            self.cfg.add_rules({})
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'error')
+        rv = self.cfg.filter(self.file, entity='one_entity')
+        self.assertEqual(rv, 'error')
+
+    def test_filter_py_ignore(self):
+        'Test filter.py just return str("ignore")'
+        def filter(mod, path, entity=None):
+            return 'ignore'
+        self.cfg.set_filter_py(filter)
+        with self.assertRaises(AssertionError):
+            self.cfg.add_rules({})
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.file, entity='one_entity')
+        self.assertEqual(rv, 'ignore')
+
+    def test_filter_py_report(self):
+        'Test filter.py just return str("report") and match to "warning"'
+        def filter(mod, path, entity=None):
+            return 'report'
+        self.cfg.set_filter_py(filter)
+        with self.assertRaises(AssertionError):
+            self.cfg.add_rules({})
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'warning')
+        rv = self.cfg.filter(self.file, entity='one_entity')
+        self.assertEqual(rv, 'warning')
+
+    def test_filter_py_module(self):
+        'Test filter.py to return str("error") for browser or "ignore"'
+        def filter(mod, path, entity=None):
+            return 'error' if mod == 'browser' else 'ignore'
+        self.cfg.set_filter_py(filter)
+        with self.assertRaises(AssertionError):
+            self.cfg.add_rules({})
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'error')
+        rv = self.cfg.filter(self.file, entity='one_entity')
+        self.assertEqual(rv, 'error')
+        rv = self.cfg.filter(self.other_file)
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.other_file, entity='one_entity')
+        self.assertEqual(rv, 'ignore')

third_party/python/compare-locales/compare_locales/tests/paths/test_matcher.py

@@ -0,0 +1,440 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import six
+import unittest
+
+from compare_locales.paths.matcher import Matcher, ANDROID_STANDARD_MAP
+from . import Rooted
+
+
+class TestMatcher(unittest.TestCase):
+
+    def test_matcher(self):
+        one = Matcher('foo/*')
+        self.assertTrue(one.match('foo/baz'))
+        self.assertFalse(one.match('foo/baz/qux'))
+        other = Matcher('bar/*')
+        self.assertTrue(other.match('bar/baz'))
+        self.assertFalse(other.match('bar/baz/qux'))
+        self.assertEqual(one.sub(other, 'foo/baz'), 'bar/baz')
+        self.assertIsNone(one.sub(other, 'bar/baz'))
+        one = Matcher('foo/**')
+        self.assertTrue(one.match('foo/baz'))
+        self.assertTrue(one.match('foo/baz/qux'))
+        other = Matcher('bar/**')
+        self.assertTrue(other.match('bar/baz'))
+        self.assertTrue(other.match('bar/baz/qux'))
+        self.assertEqual(one.sub(other, 'foo/baz'), 'bar/baz')
+        self.assertEqual(one.sub(other, 'foo/baz/qux'), 'bar/baz/qux')
+        one = Matcher('foo/*/one/**')
+        self.assertTrue(one.match('foo/baz/one/qux'))
+        self.assertFalse(one.match('foo/baz/bez/one/qux'))
+        other = Matcher('bar/*/other/**')
+        self.assertTrue(other.match('bar/baz/other/qux'))
+        self.assertFalse(other.match('bar/baz/bez/other/qux'))
+        self.assertEqual(one.sub(other, 'foo/baz/one/qux'),
+                         'bar/baz/other/qux')
+        self.assertEqual(one.sub(other, 'foo/baz/one/qux/zzz'),
+                         'bar/baz/other/qux/zzz')
+        self.assertIsNone(one.sub(other, 'foo/baz/bez/one/qux'))
+        one = Matcher('foo/**/bar/**')
+        self.assertTrue(one.match('foo/bar/baz.qux'))
+        self.assertTrue(one.match('foo/tender/bar/baz.qux'))
+        self.assertFalse(one.match('foo/nobar/baz.qux'))
+        self.assertFalse(one.match('foo/tender/bar'))
+
+    def test_prefix(self):
+        self.assertEqual(
+            Matcher('foo/bar.file').prefix, 'foo/bar.file'
+        )
+        self.assertEqual(
+            Matcher('foo/*').prefix, 'foo/'
+        )
+        self.assertEqual(
+            Matcher('foo/**').prefix, 'foo/'
+        )
+        self.assertEqual(
+            Matcher('foo/*/bar').prefix, 'foo/'
+        )
+        self.assertEqual(
+            Matcher('foo/**/bar').prefix, 'foo/'
+        )
+        self.assertEqual(
+            Matcher('foo/**/bar/*').prefix, 'foo/'
+        )
+        self.assertEqual(
+            Matcher('foo/{v}/bar').prefix,
+            'foo/'
+        )
+        self.assertEqual(
+            Matcher('foo/{v}/bar', {'v': 'expanded'}).prefix,
+            'foo/expanded/bar'
+        )
+        self.assertEqual(
+            Matcher('foo/{v}/*/bar').prefix,
+            'foo/'
+        )
+        self.assertEqual(
+            Matcher('foo/{v}/*/bar', {'v': 'expanded'}).prefix,
+            'foo/expanded/'
+        )
+        self.assertEqual(
+            Matcher('foo/{v}/*/bar', {'v': '{missing}'}).prefix,
+            'foo/'
+        )
+
+    def test_variables(self):
+        self.assertDictEqual(
+            Matcher('foo/bar.file').match('foo/bar.file'),
+            {}
+        )
+        self.assertDictEqual(
+            Matcher('{path}/bar.file').match('foo/bar.file'),
+            {
+                'path': 'foo'
+            }
+        )
+        self.assertDictEqual(
+            Matcher('{ path }/bar.file').match('foo/bar.file'),
+            {
+                'path': 'foo'
+            }
+        )
+        self.assertIsNone(
+            Matcher('{ var }/foopy/{ var }/bears')
+            .match('one/foopy/other/bears')
+        )
+        self.assertDictEqual(
+            Matcher('{ var }/foopy/{ var }/bears')
+            .match('same_value/foopy/same_value/bears'),
+            {
+                'var': 'same_value'
+            }
+        )
+        self.assertIsNone(
+            Matcher('{ var }/foopy/bears', {'var': 'other'})
+            .match('one/foopy/bears')
+        )
+        self.assertDictEqual(
+            Matcher('{ var }/foopy/bears', {'var': 'one'})
+            .match('one/foopy/bears'),
+            {
+                'var': 'one'
+            }
+        )
+        self.assertDictEqual(
+            Matcher('{one}/{two}/something', {
+                'one': 'some/segment',
+                'two': 'with/a/lot/of'
+            }).match('some/segment/with/a/lot/of/something'),
+            {
+                'one': 'some/segment',
+                'two': 'with/a/lot/of'
+            }
+        )
+        self.assertDictEqual(
+            Matcher('{l}**', {
+                'l': 'foo/{locale}/'
+            }).match('foo/it/path'),
+            {
+                'l': 'foo/it/',
+                'locale': 'it',
+                's1': 'path',
+            }
+        )
+        self.assertDictEqual(
+            Matcher('{l}*', {
+                'l': 'foo/{locale}/'
+            }).match('foo/it/path'),
+            {
+                'l': 'foo/it/',
+                'locale': 'it',
+                's1': 'path',
+            }
+        )
+
+    def test_variables_sub(self):
+        one = Matcher('{base}/{loc}/*', {'base': 'ONE_BASE'})
+        other = Matcher('{base}/somewhere/*', {'base': 'OTHER_BASE'})
+        self.assertEqual(
+            one.sub(other, 'ONE_BASE/ab-CD/special'),
+            'OTHER_BASE/somewhere/special'
+        )
+
+    def test_copy(self):
+        one = Matcher('{base}/{loc}/*', {
+            'base': 'ONE_BASE',
+            'generic': 'keep'
+        })
+        other = Matcher(one, {'base': 'OTHER_BASE'})
+        self.assertEqual(
+            one.sub(other, 'ONE_BASE/ab-CD/special'),
+            'OTHER_BASE/ab-CD/special'
+        )
+        self.assertDictEqual(
+            one.env,
+            {
+                'base': ['ONE_BASE'],
+                'generic': ['keep']
+            }
+        )
+        self.assertDictEqual(
+            other.env,
+            {
+                'base': ['OTHER_BASE'],
+                'generic': ['keep']
+            }
+        )
+
+    def test_eq(self):
+        self.assertEqual(
+            Matcher('foo'),
+            Matcher('foo')
+        )
+        self.assertNotEqual(
+            Matcher('foo'),
+            Matcher('bar')
+        )
+        self.assertEqual(
+            Matcher('foo', root='/bar/'),
+            Matcher('foo', root='/bar/')
+        )
+        self.assertNotEqual(
+            Matcher('foo', root='/bar/'),
+            Matcher('foo', root='/baz/')
+        )
+        self.assertNotEqual(
+            Matcher('foo'),
+            Matcher('foo', root='/bar/')
+        )
+        self.assertEqual(
+            Matcher('foo', env={'one': 'two'}),
+            Matcher('foo', env={'one': 'two'})
+        )
+        self.assertEqual(
+            Matcher('foo'),
+            Matcher('foo', env={})
+        )
+        self.assertNotEqual(
+            Matcher('foo', env={'one': 'two'}),
+            Matcher('foo', env={'one': 'three'})
+        )
+        self.assertEqual(
+            Matcher('foo', env={'other': 'val'}),
+            Matcher('foo', env={'one': 'two'})
+        )
+
+
+class ConcatTest(unittest.TestCase):
+    def test_plain(self):
+        left = Matcher('some/path/')
+        right = Matcher('with/file')
+        concatenated = left.concat(right)
+        self.assertEqual(str(concatenated), 'some/path/with/file')
+        self.assertEqual(concatenated.prefix, 'some/path/with/file')
+        pattern_concatenated = left.concat('with/file')
+        self.assertEqual(concatenated, pattern_concatenated)
+
+    def test_stars(self):
+        left = Matcher('some/*/path/')
+        right = Matcher('with/file')
+        concatenated = left.concat(right)
+        self.assertEqual(concatenated.prefix, 'some/')
+        concatenated = right.concat(left)
+        self.assertEqual(concatenated.prefix, 'with/filesome/')
+
+
+class TestAndroid(unittest.TestCase):
+    '''special case handling for `android_locale` to handle the funky
+    locale codes in Android apps
+    '''
+    def test_match(self):
+        # test matches as well as groupdict aliasing.
+        one = Matcher('values-{android_locale}/strings.xml')
+        self.assertEqual(
+            one.match('values-de/strings.xml'),
+            {
+                'android_locale': 'de',
+                'locale': 'de'
+            }
+        )
+        self.assertEqual(
+            one.match('values-de-rDE/strings.xml'),
+            {
+                'android_locale': 'de-rDE',
+                'locale': 'de-DE'
+            }
+        )
+        self.assertEqual(
+            one.match('values-b+sr+Latn/strings.xml'),
+            {
+                'android_locale': 'b+sr+Latn',
+                'locale': 'sr-Latn'
+            }
+        )
+        self.assertEqual(
+            one.with_env(
+                {'locale': 'de'}
+            ).match('values-de/strings.xml'),
+            {
+                'android_locale': 'de',
+                'locale': 'de'
+            }
+        )
+        self.assertEqual(
+            one.with_env(
+                {'locale': 'de-DE'}
+            ).match('values-de-rDE/strings.xml'),
+            {
+                'android_locale': 'de-rDE',
+                'locale': 'de-DE'
+            }
+        )
+        self.assertEqual(
+            one.with_env(
+                {'locale': 'sr-Latn'}
+            ).match('values-b+sr+Latn/strings.xml'),
+            {
+                'android_locale': 'b+sr+Latn',
+                'locale': 'sr-Latn'
+            }
+        )
+
+    def test_repeat(self):
+        self.assertEqual(
+            Matcher('{android_locale}/{android_locale}').match(
+                'b+sr+Latn/b+sr+Latn'
+            ),
+            {
+                'android_locale': 'b+sr+Latn',
+                'locale': 'sr-Latn'
+            }
+        )
+        self.assertEqual(
+            Matcher(
+                '{android_locale}/{android_locale}',
+                env={'locale': 'sr-Latn'}
+            ).match(
+                'b+sr+Latn/b+sr+Latn'
+            ),
+            {
+                'android_locale': 'b+sr+Latn',
+                'locale': 'sr-Latn'
+            }
+        )
+
+    def test_mismatch(self):
+        # test failed matches
+        one = Matcher('values-{android_locale}/strings.xml')
+        self.assertIsNone(
+            one.with_env({'locale': 'de'}).match(
+                'values-fr.xml'
+            )
+        )
+        self.assertIsNone(
+            one.with_env({'locale': 'de-DE'}).match(
+                'values-de-DE.xml'
+            )
+        )
+        self.assertIsNone(
+            one.with_env({'locale': 'sr-Latn'}).match(
+                'values-sr-Latn.xml'
+            )
+        )
+        self.assertIsNone(
+            Matcher('{android_locale}/{android_locale}').match(
+                'b+sr+Latn/de-rDE'
+            )
+        )
+
+    def test_prefix(self):
+        one = Matcher('values-{android_locale}/strings.xml')
+        self.assertEqual(
+            one.with_env({'locale': 'de'}).prefix,
+            'values-de/strings.xml'
+        )
+        self.assertEqual(
+            one.with_env({'locale': 'de-DE'}).prefix,
+            'values-de-rDE/strings.xml'
+        )
+        self.assertEqual(
+            one.with_env({'locale': 'sr-Latn'}).prefix,
+            'values-b+sr+Latn/strings.xml'
+        )
+        self.assertEqual(
+            one.prefix,
+            'values-'
+        )
+
+    def test_aliases(self):
+        # test legacy locale code mapping
+        # he <-> iw, id <-> in, yi <-> ji
+        one = Matcher('values-{android_locale}/strings.xml')
+        for legacy, standard in six.iteritems(ANDROID_STANDARD_MAP):
+            self.assertDictEqual(
+                one.match('values-{}/strings.xml'.format(legacy)),
+                {
+                    'android_locale': legacy,
+                    'locale': standard
+                }
+            )
+            self.assertEqual(
+                one.with_env({'locale': standard}).prefix,
+                'values-{}/strings.xml'.format(legacy)
+            )
+
+
+class TestRootedMatcher(Rooted, unittest.TestCase):
+    def test_root_path(self):
+        one = Matcher('some/path', root=self.root)
+        self.assertIsNone(one.match('some/path'))
+        self.assertIsNotNone(one.match(self.path('/some/path')))
+
+    def test_copy(self):
+        one = Matcher('some/path', root=self.path('/one-root'))
+        other = Matcher(one, root=self.path('/different-root'))
+        self.assertIsNone(other.match('some/path'))
+        self.assertIsNone(
+            other.match(self.path('/one-root/some/path'))
+        )
+        self.assertIsNotNone(
+            other.match(self.path('/different-root/some/path'))
+        )
+
+    def test_rooted(self):
+        r1 = self.path('/one-root')
+        r2 = self.path('/other-root')
+        one = Matcher(self.path('/one-root/full/path'), root=r2)
+        self.assertIsNone(one.match(self.path('/other-root/full/path')))
+        # concat r2 and r1. r1 is absolute, so we gotta trick that
+        concat_root = r2
+        if not r1.startswith('/'):
+            # windows absolute paths don't start with '/', add one
+            concat_root += '/'
+        concat_root += r1
+        self.assertIsNone(one.match(concat_root + '/full/path'))
+        self.assertIsNotNone(one.match(self.path('/one-root/full/path')))
+
+    def test_variable(self):
+        r1 = self.path('/one-root')
+        r2 = self.path('/other-root')
+        one = Matcher(
+            '{var}/path',
+            env={'var': 'relative-dir'},
+            root=r1
+        )
+        self.assertIsNone(one.match('relative-dir/path'))
+        self.assertIsNotNone(
+            one.match(self.path('/one-root/relative-dir/path'))
+        )
+        other = Matcher(one, env={'var': r2})
+        self.assertIsNone(
+            other.match(self.path('/one-root/relative-dir/path'))
+        )
+        self.assertIsNotNone(
+            other.match(self.path('/other-root/path'))
+        )

third_party/python/compare-locales/compare_locales/tests/paths/test_paths.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import unittest
+
+from compare_locales.paths import File
+
+
+class TestFile(unittest.TestCase):
+    def test_hash_and_equality(self):
+        f1 = File('/tmp/full/path/to/file', 'path/to/file')
+        d = {}
+        d[f1] = True
+        self.assertIn(f1, d)
+        f2 = File('/tmp/full/path/to/file', 'path/to/file')
+        self.assertIn(f2, d)
+        f2 = File('/tmp/full/path/to/file', 'path/to/file', locale='en')
+        self.assertNotIn(f2, d)
+        # trigger hash collisions between File and non-File objects
+        self.assertEqual(hash(f1), hash(f1.localpath))
+        self.assertNotIn(f1.localpath, d)
+        f1 = File('/tmp/full/other/path', 'other/path')
+        d[f1.localpath] = True
+        self.assertIn(f1.localpath, d)
+        self.assertNotIn(f1, d)

third_party/python/compare-locales/compare_locales/tests/paths/test_project.py

@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import unittest
+
+from compare_locales.paths import ProjectConfig
+from . import SetupMixin
+
+
+class TestConfigRules(SetupMixin, unittest.TestCase):
+
+    def test_filter_empty(self):
+        'Test that an empty config works'
+        self.cfg.add_paths({
+            'l10n': '/tmp/somedir/{locale}/browser/**'
+        })
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'error')
+        rv = self.cfg.filter(self.file, entity='one_entity')
+        self.assertEqual(rv, 'error')
+        rv = self.cfg.filter(self.other_file)
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.other_file, entity='one_entity')
+        self.assertEqual(rv, 'ignore')
+
+    def test_single_file_rule(self):
+        'Test a single rule for just a single file, no key'
+        self.cfg.add_paths({
+            'l10n': '/tmp/somedir/{locale}/browser/**'
+        })
+        self.cfg.add_rules({
+            'path': '/tmp/somedir/{locale}/browser/one/two/file.ftl',
+            'action': 'ignore'
+        })
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.file, 'one_entity')
+        self.assertEqual(rv, 'error')
+        rv = self.cfg.filter(self.other_file)
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.other_file, 'one_entity')
+        self.assertEqual(rv, 'ignore')
+
+    def test_single_key_rule(self):
+        'Test a single rule with file and key'
+        self.cfg.add_paths({
+            'l10n': '/tmp/somedir/{locale}/browser/**'
+        })
+        self.cfg.add_rules({
+            'path': '/tmp/somedir/{locale}/browser/one/two/file.ftl',
+            'key': 'one_entity',
+            'action': 'ignore'
+        })
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'error')
+        rv = self.cfg.filter(self.file, 'one_entity')
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.other_file)
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.other_file, 'one_entity')
+        self.assertEqual(rv, 'ignore')
+
+    def test_single_non_matching_key_rule(self):
+        'Test a single key rule with regex special chars that should not match'
+        self.cfg.add_paths({
+            'l10n': '/tmp/somedir/{locale}/**'
+        })
+        self.cfg.add_rules({
+            'path': '/tmp/somedir/{locale}/browser/one/two/file.ftl',
+            'key': '.ne_entit.',
+            'action': 'ignore'
+        })
+        rv = self.cfg.filter(self.file, 'one_entity')
+        self.assertEqual(rv, 'error')
+
+    def test_single_matching_re_key_rule(self):
+        'Test a single key with regular expression'
+        self.cfg.add_paths({
+            'l10n': '/tmp/somedir/{locale}/**'
+        })
+        self.cfg.add_rules({
+            'path': '/tmp/somedir/{locale}/browser/one/two/file.ftl',
+            'key': 're:.ne_entit.$',
+            'action': 'ignore'
+        })
+        rv = self.cfg.filter(self.file, 'one_entity')
+        self.assertEqual(rv, 'ignore')
+
+    def test_double_file_rule(self):
+        'Test path shortcut, one for each of our files'
+        self.cfg.add_paths({
+            'l10n': '/tmp/somedir/{locale}/**'
+        })
+        self.cfg.add_rules({
+            'path': [
+                '/tmp/somedir/{locale}/browser/one/two/file.ftl',
+                '/tmp/somedir/{locale}/toolkit/two/one/file.ftl',
+            ],
+            'action': 'ignore'
+        })
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.other_file)
+        self.assertEqual(rv, 'ignore')
+
+    def test_double_file_key_rule(self):
+        'Test path and key shortcut, one key matching, one not'
+        self.cfg.add_paths({
+            'l10n': '/tmp/somedir/{locale}/**'
+        })
+        self.cfg.add_rules({
+            'path': [
+                '/tmp/somedir/{locale}/browser/one/two/file.ftl',
+                '/tmp/somedir/{locale}/toolkit/two/one/file.ftl',
+            ],
+            'key': [
+                'one_entity',
+                'other_entity',
+            ],
+            'action': 'ignore'
+        })
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'error')
+        rv = self.cfg.filter(self.file, 'one_entity')
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.other_file)
+        self.assertEqual(rv, 'error')
+        rv = self.cfg.filter(self.other_file, 'one_entity')
+        self.assertEqual(rv, 'ignore')
+
+    def test_single_wildcard_rule(self):
+        'Test single wildcard'
+        self.cfg.add_paths({
+            'l10n': '/tmp/somedir/{locale}/browser/**'
+        })
+        self.cfg.add_rules({
+            'path': [
+                '/tmp/somedir/{locale}/browser/one/*/*',
+            ],
+            'action': 'ignore'
+        })
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.other_file)
+        self.assertEqual(rv, 'ignore')
+
+    def test_double_wildcard_rule(self):
+        'Test double wildcard'
+        self.cfg.add_paths({
+            'l10n': '/tmp/somedir/{locale}/**'
+        })
+        self.cfg.add_rules({
+            'path': [
+                '/tmp/somedir/{locale}/**',
+            ],
+            'action': 'ignore'
+        })
+        rv = self.cfg.filter(self.file)
+        self.assertEqual(rv, 'ignore')
+        rv = self.cfg.filter(self.other_file)
+        self.assertEqual(rv, 'ignore')
+
+
+class TestProjectConfig(unittest.TestCase):
+    def test_children(self):
+        pc = ProjectConfig(None)
+        child = ProjectConfig(None)
+        pc.add_child(child)
+        self.assertListEqual([pc, child], list(pc.configs))
+
+
+class TestSameConfig(unittest.TestCase):
+
+    def test_path(self):
+        one = ProjectConfig('one.toml')
+        one.set_locales(['ab'])
+        self.assertTrue(one.same(ProjectConfig('one.toml')))
+        self.assertFalse(one.same(ProjectConfig('two.toml')))
+
+    def test_paths(self):
+        one = ProjectConfig('one.toml')
+        one.set_locales(['ab'])
+        one.add_paths({
+            'l10n': '/tmp/somedir/{locale}/**'
+        })
+        other = ProjectConfig('one.toml')
+        self.assertFalse(one.same(other))
+        other.add_paths({
+            'l10n': '/tmp/somedir/{locale}/**'
+        })
+        self.assertTrue(one.same(other))
+
+    def test_children(self):
+        one = ProjectConfig('one.toml')
+        one.add_child(ProjectConfig('inner.toml'))
+        one.set_locales(['ab'])
+        other = ProjectConfig('one.toml')
+        self.assertFalse(one.same(other))
+        other.add_child(ProjectConfig('inner.toml'))
+        self.assertTrue(one.same(other))

third_party/python/compare-locales/compare_locales/tests/po/test_parser.py

@@ -0,0 +1,139 @@
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+import unittest
+
+from compare_locales.tests import ParserTestMixin
+from compare_locales.parser import (
+    BadEntity,
+    Whitespace,
+)
+
+
+class TestPoParser(ParserTestMixin, unittest.TestCase):
+    maxDiff = None
+    filename = 'strings.po'
+
+    def test_parse_string_list(self):
+        self.parser.readUnicode('  ')
+        ctx = self.parser.ctx
+        with self.assertRaises(BadEntity):
+            self.parser._parse_string_list(ctx, 0, 'msgctxt')
+        self.parser.readUnicode('msgctxt   ')
+        ctx = self.parser.ctx
+        with self.assertRaises(BadEntity):
+            self.parser._parse_string_list(ctx, 0, 'msgctxt')
+        self.parser.readUnicode('msgctxt " "')
+        ctx = self.parser.ctx
+        self.assertTupleEqual(
+            self.parser._parse_string_list(ctx, 0, 'msgctxt'),
+            (" ", len(ctx.contents))
+        )
+        self.parser.readUnicode('msgctxt " " \t "A"\r "B"asdf')
+        ctx = self.parser.ctx
+        self.assertTupleEqual(
+            self.parser._parse_string_list(ctx, 0, 'msgctxt'),
+            (" AB", len(ctx.contents)-4)
+        )
+        self.parser.readUnicode('msgctxt "\\\\ " "A" "B"asdf"fo"')
+        ctx = self.parser.ctx
+        self.assertTupleEqual(
+            self.parser._parse_string_list(ctx, 0, 'msgctxt'),
+            ("\\ AB", len(ctx.contents)-8)
+        )
+
+    def test_simple_string(self):
+        source = '''
+msgid "untranslated string"
+msgstr "translated string"
+'''
+        self._test(
+            source,
+            (
+                (Whitespace, '\n'),
+                (('untranslated string', None), 'translated string'),
+                (Whitespace, '\n'),
+            )
+        )
+
+    def test_escapes(self):
+        source = r'''
+msgid "untranslated string"
+msgstr "\\\t\r\n\""
+'''
+        self._test(
+            source,
+            (
+                (Whitespace, '\n'),
+                (('untranslated string', None), '\\\t\r\n"'),
+                (Whitespace, '\n'),
+            )
+        )
+
+    def test_comments(self):
+        source = '''
+#  translator-comments
+#. extracted-comments
+#: reference...
+#, flag...
+#| msgctxt previous-context
+#| msgid previous-untranslated-string
+msgid "untranslated string"
+msgstr "translated string"
+'''
+        self._test(
+            source,
+            (
+                (Whitespace, '\n'),
+                (
+                    ('untranslated string', None),
+                    'translated string',
+                    'extracted-comments',
+                ),
+                (Whitespace, '\n'),
+            )
+        )
+
+    def test_simple_context(self):
+        source = '''
+msgctxt "context to use"
+msgid "untranslated string"
+msgstr "translated string"
+'''
+        self._test(
+            source,
+            (
+                (Whitespace, '\n'),
+                (
+                    ('untranslated string', 'context to use'),
+                    'translated string'
+                ),
+                (Whitespace, '\n'),
+            )
+        )
+
+    def test_translated(self):
+        source = '''
+msgid "reference 1"
+msgstr "translated string"
+
+msgid "reference 2"
+msgstr ""
+'''
+        self._test(
+            source,
+            (
+                (Whitespace, '\n'),
+                (('reference 1', None), 'translated string'),
+                (Whitespace, '\n'),
+                (('reference 2', None), ''),
+                (Whitespace, '\n'),
+            )
+        )
+        entities = self.parser.parse()
+        self.assertListEqual(
+            [e.localized for e in entities],
+            [True, False]
+        )

third_party/python/compare-locales/compare_locales/tests/serializer/__init__.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from compare_locales.parser import getParser
+from compare_locales.serializer import serialize
+
+
+class Helper(object):
+    """Mixin to test serializers.
+
+    Reads the reference_content into self.reference, and uses
+    that to serialize in _test.
+    """
+    name = None
+    reference_content = None
+
+    def setUp(self):
+        p = self.parser = getParser(self.name)
+        p.readUnicode(self.reference_content)
+        self.reference = list(p.walk())
+
+    def _test(self, old_content, new_data, expected):
+        """Test with old content, new data, and the reference data
+        against the expected unicode output.
+        """
+        self.parser.readUnicode(old_content)
+        old_l10n = list(self.parser.walk())
+        output = serialize(self.name, self.reference, old_l10n, new_data)
+        self.assertMultiLineEqual(
+            output.decode(self.parser.encoding),
+            expected
+        )

third_party/python/compare-locales/compare_locales/tests/serializer/test_android.py

@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from . import Helper
+
+
+class TestAndroidSerializer(Helper, unittest.TestCase):
+    name = 'strings.xml'
+    reference_content = """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <!-- The page html title (i.e. the <title> tag content) -->
+    <string name="title">Unable to connect</string>
+    <string name="message"><![CDATA[
+      <ul>
+        <li>The site could be temporarily unavailable or too busy.</li>
+      </ul>
+    ]]></string>
+    <string name="wrapped_message">
+      <![CDATA[
+        <ul>
+          <li>The site could be temporarily unavailable or too busy.</li>
+        </ul>
+      ]]>
+    </string>
+</resources>
+"""
+
+    def test_nothing_new_or_old(self):
+        self._test(
+            "",
+            {},
+            """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    </resources>
+"""
+        )
+
+    def test_new_string(self):
+        self._test(
+            "",
+            {
+                "title": "Cannot connect"
+            },
+            """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <!-- The page html title (i.e. the <title> tag content) -->
+    <string name="title">Cannot connect</string>
+    </resources>
+"""
+        )
+
+    def test_new_cdata(self):
+        self._test(
+            "",
+            {
+                "message": """
+<ul>
+  <li>Something else</li>
+</ul>
+"""
+            },
+            """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <string name="message"><![CDATA[
+<ul>
+  <li>Something else</li>
+</ul>
+]]></string>
+    </resources>
+"""
+        )
+
+    def test_new_cdata_wrapped(self):
+        self._test(
+            "",
+            {
+                "wrapped_message": """
+<ul>
+  <li>Something else</li>
+</ul>
+"""
+            },
+            """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <string name="wrapped_message">
+      <![CDATA[
+<ul>
+  <li>Something else</li>
+</ul>
+]]>
+    </string>
+</resources>
+"""
+        )
+
+    def test_remove_string(self):
+        self._test(
+            """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <string name="first_old_title">Unable to connect</string>
+    <string name="title">Unable to connect</string>
+    <string name="last_old_title">Unable to connect</string>
+</resources>
+""",
+            {},
+            """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <string name="title">Unable to connect</string>
+    </resources>
+"""
+        )
+
+    def test_same_string(self):
+        self._test(
+            """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <string name="title">Unable to connect</string>
+</resources>
+""",
+            {
+                "title": "Unable to connect"
+            },
+            """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <!-- The page html title (i.e. the <title> tag content) -->
+    <string name="title">Unable to connect</string>
+    </resources>
+"""
+        )
+
+
+class TestAndroidDuplicateComment(Helper, unittest.TestCase):
+    name = 'strings.xml'
+    reference_content = """\
+<?xml version="1.0" encoding="utf-8"?>
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<resources>
+    <!-- Label used in the contextmenu shown when long-pressing on a link -->
+    <string name="contextmenu_open_in_app">Open with app</string>
+    <!-- Label used in the contextmenu shown when long-pressing on a link -->
+    <string name="contextmenu_link_share">Share link</string>
+</resources>
+"""
+
+    def test_missing_translation(self):
+        self._test(
+            """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+
+    <!-- Label used in the contextmenu shown when long-pressing on a link -->
+    <!-- Label used in the contextmenu shown when long-pressing on a link -->
+    <string name="contextmenu_link_share"/>
+  </resources>
+""",
+            {
+                "contextmenu_link_share": "translation"
+            },
+            """\
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+
+    <!-- Label used in the contextmenu shown when long-pressing on a link -->
+    <string name="contextmenu_link_share">translation</string>
+  </resources>
+"""
+        )

third_party/python/compare-locales/compare_locales/tests/serializer/test_fluent.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from compare_locales.serializer import serialize
+from . import Helper
+
+
+class TestFluentSerializer(Helper, unittest.TestCase):
+    name = "foo.ftl"
+    reference_content = """\
+this = is English
+
+# another one bites
+another = message
+"""
+
+    def test_nothing_new_or_old(self):
+        output = serialize(self.name, self.reference, [], {})
+        self.assertMultiLineEqual(output.decode(self.parser.encoding), '\n\n')
+
+    def test_obsolete_old_string(self):
+        self._test(
+            """\
+# we used to have this
+old = stuff with comment
+""",
+            {},
+            """\
+
+
+""")
+
+    def test_nothing_old_new_translation(self):
+        self._test(
+            "",
+            {
+                "another": "another = localized message"
+            },
+            """\
+
+
+# another one bites
+another = localized message
+"""
+        )
+
+    def test_old_message_new_other_translation(self):
+        self._test(
+            """\
+this = is localized
+""",
+            {
+                "another": "another = localized message"
+            },
+            """\
+this = is localized
+
+# another one bites
+another = localized message
+"""
+        )
+
+    def test_old_message_new_same_translation(self):
+        self._test(
+            """\
+this = is localized
+""",
+            {
+                "this": "this = has a better message"
+            },
+            """\
+this = has a better message
+
+"""
+        )

third_party/python/compare-locales/compare_locales/tests/serializer/test_properties.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from compare_locales.serializer import serialize
+from . import Helper
+
+
+class TestPropertiesSerializer(Helper, unittest.TestCase):
+    name = 'foo.properties'
+    reference_content = """\
+this = is English
+
+# another one bites
+another = message
+"""
+
+    def test_nothing_new_or_old(self):
+        output = serialize(self.name, self.reference, [], {})
+        self.assertMultiLineEqual(output.decode(self.parser.encoding), '\n\n')
+
+    def test_obsolete_old_string(self):
+        self._test(
+            """\
+# we used to have this
+old = stuff with comment
+""",
+            {},
+            """\
+
+
+""")
+
+    def test_nothing_old_new_translation(self):
+        self._test(
+            "",
+            {
+                "another": "localized message"
+            },
+            """\
+
+
+# another one bites
+another = localized message
+"""
+        )
+
+    def test_old_message_new_other_translation(self):
+        self._test(
+            """\
+this = is localized
+""",
+            {
+                "another": "localized message"
+            },
+            """\
+this = is localized
+
+# another one bites
+another = localized message
+"""
+        )
+
+    def test_old_message_new_same_translation(self):
+        self._test(
+            """\
+this = is localized
+""",
+            {
+                "this": "has a better message"
+            },
+            """\
+this = has a better message
+
+"""
+        )
+
+
+class TestPropertiesDuplicateComment(Helper, unittest.TestCase):
+    name = 'foo.properties'
+    reference_content = """\
+# repetitive
+one = one
+# repetitive
+two = two
+"""
+
+    def test_missing_translation(self):
+        self._test(
+            """\
+# repetitive
+
+# repetitive
+two = two
+""",
+            {},
+            """\
+# repetitive
+
+# repetitive
+two = two
+"""
+        )

third_party/python/compare-locales/compare_locales/tests/test_checks.py

@@ -21,7 +21,7 @@ class BaseHelper(unittest.TestCase):
     def setUp(self):
         p = getParser(self.file.file)
         p.readContents(self.refContent)
-        self.refList, self.refMap = p.parse()
+        self.refList = p.parse()
 
     def _test(self, content, refWarnOrErrors):
         p = getParser(self.file.file)
@@ -32,7 +32,7 @@ class BaseHelper(unittest.TestCase):
         checker = getChecker(self.file)
         if checker.needs_reference:
             checker.set_reference(self.refList)
-        ref = self.refList[self.refMap[l10n.key]]
+        ref = self.refList[l10n.key]
         found = tuple(checker.check(ref, l10n))
         self.assertEqual(found, refWarnOrErrors)
 
@@ -265,7 +265,7 @@ class TestAndroid(unittest.TestCase):
     def getNext(self, v):
         ctx = Parser.Context(v)
         return DTDEntity(
-            ctx, '', (0, len(v)), (), (0, len(v)))
+            ctx, None, None, (0, len(v)), (), (0, len(v)))
 
     def getDTDEntity(self, v):
         if isinstance(v, six.binary_type):
@@ -273,7 +273,7 @@ class TestAndroid(unittest.TestCase):
         v = v.replace('"', '"')
         ctx = Parser.Context('<!ENTITY foo "%s">' % v)
         return DTDEntity(
-            ctx, '', (0, len(v) + 16), (9, 12), (14, len(v) + 14))
+            ctx, None, None, (0, len(v) + 16), (9, 12), (14, len(v) + 14))
 
     def test_android_dtd(self):
         """Testing the actual android checks. The logic is involved,
@@ -406,7 +406,7 @@ class TestAndroid(unittest.TestCase):
         p.readContents(b'<!ENTITY other "some &good.ref;">')
         ref = p.parse()
         checker = getChecker(f)
-        checker.set_reference(ref[0])
+        checker.set_reference(ref)
         # good string
         ref = self.getDTDEntity("plain string")
         l10n = self.getDTDEntity("plain localized string")

third_party/python/compare-locales/compare_locales/tests/test_compare.py

@@ -6,7 +6,6 @@ from __future__ import absolute_import
 import unittest
 
 from compare_locales import compare, paths
-from six.moves.cPickle import loads, dumps
 
 
 class TestTree(unittest.TestCase):
@@ -85,7 +84,7 @@ two/other
 class TestObserver(unittest.TestCase):
     def test_simple(self):
         obs = compare.Observer()
-        f = paths.File('/some/real/sub/path', 'sub/path', locale='de')
+        f = paths.File('/some/real/sub/path', 'de/sub/path', locale='de')
         obs.notify('missingEntity', f, 'one')
         obs.notify('missingEntity', f, 'two')
         obs.updateStats(f, {'missing': 15})
@@ -101,13 +100,9 @@ class TestObserver(unittest.TestCase):
                      {'missingEntity': 'two'}]
             }
         })
-        clone = loads(dumps(obs))
-        self.assertDictEqual(clone.summary, obs.summary)
-        self.assertDictEqual(clone.details.toJSON(), obs.details.toJSON())
-        self.assertIsNone(clone.file_stats)
 
     def test_module(self):
-        obs = compare.Observer(file_stats=True)
+        obs = compare.Observer()
         f = paths.File('/some/real/sub/path', 'path',
                        module='sub', locale='de')
         obs.notify('missingEntity', f, 'one')
@@ -129,57 +124,6 @@ class TestObserver(unittest.TestCase):
                     ]
             }
         })
-        self.assertDictEqual(obs.file_stats, {
-            'de': {
-                'sub/path': {
-                    'missing': 15
-                }
-            }
-        })
-        self.assertEqual(obs.serialize(), '''\
-de/sub/path
-    +one
-    -bar
-    +two
-de:
-missing: 15
-0% of entries changed''')
-        clone = loads(dumps(obs))
-        self.assertDictEqual(clone.summary, obs.summary)
-        self.assertDictEqual(clone.details.toJSON(), obs.details.toJSON())
-        self.assertDictEqual(clone.file_stats, obs.file_stats)
-
-    def test_file_stats(self):
-        obs = compare.Observer(file_stats=True)
-        f = paths.File('/some/real/sub/path', 'sub/path', locale='de')
-        obs.notify('missingEntity', f, 'one')
-        obs.notify('missingEntity', f, 'two')
-        obs.updateStats(f, {'missing': 15})
-        self.assertDictEqual(obs.toJSON(), {
-            'summary': {
-                'de': {
-                    'missing': 15
-                }
-            },
-            'details': {
-                'de/sub/path':
-                    [
-                     {'missingEntity': 'one'},
-                     {'missingEntity': 'two'},
-                    ]
-            }
-        })
-        self.assertDictEqual(obs.file_stats, {
-            'de': {
-                'sub/path': {
-                    'missing': 15
-                }
-            }
-        })
-        clone = loads(dumps(obs))
-        self.assertDictEqual(clone.summary, obs.summary)
-        self.assertDictEqual(clone.details.toJSON(), obs.details.toJSON())
-        self.assertDictEqual(clone.file_stats, obs.file_stats)
 
 
 class TestAddRemove(unittest.TestCase):

third_party/python/compare-locales/compare_locales/tests/test_defines.py

@@ -69,10 +69,11 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
             (Whitespace, '\n\n'),
             ('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
             (Whitespace, '\n\n'),
-            (Comment, 'non-English'),
-            (Whitespace, '\n'),
-            ('MOZ_LANGPACK_CONTRIBUTORS',
-             '<em:contributor>Joe Solon</em:contributor>'),
+            (
+                'MOZ_LANGPACK_CONTRIBUTORS',
+                '<em:contributor>Joe Solon</em:contributor>',
+                'non-English',
+            ),
             (Whitespace, '\n\n'),
             (DefinesInstruction, 'unfilter emptyLines'),
             (Junk, '\n\n')))
@@ -91,9 +92,7 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
             (Whitespace, '\n'),
             (DefinesInstruction, 'filter emptyLines'),
             (Whitespace, '\n\n'),
-            (Comment, u'češtině'),
-            (Whitespace, '\n'),
-            ('seamonkey_l10n_long', ''),
+            ('seamonkey_l10n_long', '', 'češtině'),
             (Whitespace, '\n\n'),
             (DefinesInstruction, 'unfilter emptyLines'),
             (Junk, '\n\n')))
@@ -194,6 +193,36 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
             ('tre', '  '),
             (Whitespace, '\n'),))
 
+    def test_standalone_comments(self):
+        self._test(
+            '''\
+#filter emptyLines
+# One comment
+
+# Second comment
+
+#define foo
+# bar comment
+#define bar
+
+#unfilter emptyLines
+''',
+            (
+                (DefinesInstruction, 'filter emptyLines'),
+                (Whitespace, '\n'),
+                (Comment, 'One comment'),
+                (Whitespace, '\n\n'),
+                (Comment, 'Second comment'),
+                (Whitespace, '\n\n'),
+                ('foo', ''),
+                (Whitespace, '\n'),
+                ('bar', '', 'bar comment'),
+                (Whitespace, '\n\n'),
+                (DefinesInstruction, 'unfilter emptyLines'),
+                (Whitespace, '\n'),
+            )
+        )
+
 
 if __name__ == '__main__':
     unittest.main()

third_party/python/compare-locales/compare_locales/tests/test_dtd.py

@@ -26,6 +26,10 @@ class TestDTD(ParserTestMixin, unittest.TestCase):
     def test_one_entity(self):
         self._test('''<!ENTITY foo.label "stuff">''',
                    (('foo.label', 'stuff'),))
+        self.assertListEqual(
+            [e.localized for e in self.parser],
+            [True]
+        )
 
     quoteContent = '''<!ENTITY good.one "one">
 <!ENTITY bad.one "bad " quote">
@@ -101,6 +105,22 @@ class TestDTD(ParserTestMixin, unittest.TestCase):
 <!-- This Source Code Form is subject to the terms of the Mozilla Public
    - License, v. 2.0. If a copy of the MPL was not distributed with this file,
    - You can obtain one at http://mozilla.org/MPL/2.0/.  -->
+
+<!ENTITY foo "value">
+''')
+        entities = list(p.walk())
+        self.assertIsInstance(entities[0], parser.Comment)
+        self.assertIn('MPL', entities[0].all)
+        e = entities[2]
+        self.assertIsInstance(e, parser.Entity)
+        self.assertEqual(e.key, 'foo')
+        self.assertEqual(e.val, 'value')
+        self.assertEqual(len(entities), 4)
+        # Test again without empty line after licence header, and with BOM.
+        p.readContents(b'''\xEF\xBB\xBF\
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this file,
+   - You can obtain one at http://mozilla.org/MPL/2.0/.  -->
 <!ENTITY foo "value">
 ''')
         entities = list(p.walk())
@@ -215,6 +235,37 @@ spanning lines -->  <!--
         entity = next(entities)
         self.assertIsInstance(entity, parser.Whitespace)
 
+    def test_pre_comment(self):
+        self.parser.readContents(b'''\
+<!-- comment -->
+<!ENTITY one "string">
+
+<!-- standalone -->
+
+<!-- glued --><!ENTITY second "string">
+''')
+        entities = self.parser.walk()
+
+        entity = next(entities)
+        self.assertIsInstance(entity.pre_comment, parser.Comment)
+        self.assertEqual(entity.pre_comment.val, ' comment ')
+        entity = next(entities)
+        self.assertIsInstance(entity, parser.Whitespace)
+
+        entity = next(entities)
+        self.assertIsInstance(entity, parser.Comment)
+        self.assertEqual(entity.val, ' standalone ')
+        entity = next(entities)
+        self.assertIsInstance(entity, parser.Whitespace)
+
+        entity = next(entities)
+        self.assertIsInstance(entity.pre_comment, parser.Comment)
+        self.assertEqual(entity.pre_comment.val, ' glued ')
+        entity = next(entities)
+        self.assertIsInstance(entity, parser.Whitespace)
+        with self.assertRaises(StopIteration):
+            next(entities)
+
 
 if __name__ == '__main__':
     unittest.main()

third_party/python/compare-locales/compare_locales/tests/test_ftl.py

@@ -24,6 +24,7 @@ class TestFluentParser(ParserTestMixin, unittest.TestCase):
         [ent2] = list(self.parser)
 
         self.assertTrue(ent1.equals(ent2))
+        self.assertTrue(ent1.localized)
 
     def test_equality_different_whitespace(self):
         source1 = b'foo = { $arg }'
@@ -343,3 +344,42 @@ baz = Baz
 
         with self.assertRaises(StopIteration):
             next(entities)
+
+    def test_junk(self):
+        self.parser.readUnicode('''\
+# Comment
+
+Line of junk
+
+# Comment
+msg = value
+''')
+        entities = self.parser.walk()
+
+        entity = next(entities)
+        self.assertTrue(isinstance(entity,  parser.FluentComment))
+        self.assertEqual(entity.val, 'Comment')
+
+        entity = next(entities)
+        self.assertTrue(isinstance(entity, parser.Whitespace))
+        self.assertEqual(entity.val, '\n\n')
+
+        entity = next(entities)
+        self.assertTrue(isinstance(entity,  parser.Junk))
+        self.assertEqual(entity.val, 'Line of junk')
+
+        entity = next(entities)
+        self.assertTrue(isinstance(entity, parser.Whitespace))
+        self.assertEqual(entity.val, '\n\n')
+
+        entity = next(entities)
+        self.assertTrue(isinstance(entity, parser.FluentEntity))
+        self.assertEqual(entity.val, 'value')
+        self.assertEqual(entity.entry.comment.content, 'Comment')
+
+        entity = next(entities)
+        self.assertTrue(isinstance(entity, parser.Whitespace))
+        self.assertEqual(entity.val, '\n')
+
+        with self.assertRaises(StopIteration):
+            next(entities)

third_party/python/compare-locales/compare_locales/tests/test_ini.py

@@ -67,6 +67,18 @@ TitleText=Some Title
             ('TitleText', 'Some Title'),
             (Whitespace, '\n')))
 
+    def testMPL2_no_space(self):
+        self._test(mpl2 + '''
+[Strings]
+TitleText=Some Title
+''', (
+            (Comment, mpl2),
+            (Whitespace, '\n'),
+            (IniSection, 'Strings'),
+            (Whitespace, '\n'),
+            ('TitleText', 'Some Title'),
+            (Whitespace, '\n')))
+
     def testMPL2_MultiSpace(self):
         self._test(mpl2 + '''
 

third_party/python/compare-locales/compare_locales/tests/test_keyedtuple.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+from collections import namedtuple
+import unittest
+
+from compare_locales.keyedtuple import KeyedTuple
+
+
+KeyedThing = namedtuple('KeyedThing', ['key', 'val'])
+
+
+class TestKeyedTuple(unittest.TestCase):
+    def test_constructor(self):
+        keyedtuple = KeyedTuple([])
+        self.assertEqual(keyedtuple, tuple())
+
+    def test_contains(self):
+        things = [KeyedThing('one', 'thing'), KeyedThing('two', 'things')]
+        keyedtuple = KeyedTuple(things)
+        self.assertNotIn(1, keyedtuple)
+        self.assertIn('one', keyedtuple)
+        self.assertIn(things[0], keyedtuple)
+        self.assertIn(things[1], keyedtuple)
+        self.assertNotIn(KeyedThing('three', 'stooges'), keyedtuple)
+
+    def test_getitem(self):
+        things = [KeyedThing('one', 'thing'), KeyedThing('two', 'things')]
+        keyedtuple = KeyedTuple(things)
+        self.assertEqual(keyedtuple[0], things[0])
+        self.assertEqual(keyedtuple[1], things[1])
+        self.assertEqual(keyedtuple['one'], things[0])
+        self.assertEqual(keyedtuple['two'], things[1])
+
+    def test_items(self):
+        things = [KeyedThing('one', 'thing'), KeyedThing('two', 'things')]
+        things.extend(things)
+        keyedtuple = KeyedTuple(things)
+        self.assertEqual(len(keyedtuple), 4)
+        items = list(keyedtuple.items())
+        self.assertEqual(len(items), 4)
+        self.assertEqual(
+            keyedtuple,
+            tuple((v for k, v in items))
+        )
+        self.assertEqual(
+            ('one', 'two', 'one', 'two',),
+            tuple((k for k, v in items))
+        )

third_party/python/compare-locales/compare_locales/tests/test_merge.py

@@ -11,7 +11,8 @@ import shutil
 
 from compare_locales.parser import getParser
 from compare_locales.paths import File
-from compare_locales.compare import ContentComparer, Observer
+from compare_locales.compare.content import ContentComparer
+from compare_locales.compare.observer import Observer
 from compare_locales import mozpath
 
 
@@ -51,12 +52,13 @@ class TestNonSupported(unittest.TestCase, ContentMixin):
         self.assertTrue(os.path.isdir(self.tmp))
         self.reference("""foo = 'fooVal';""")
         self.localized("""foo = 'lfoo';""")
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.js", ""),
                    File(self.l10n, "l10n.js", ""),
                    mozpath.join(self.tmp, "merge", "l10n.js"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary': {},
              'details': {}
              }
@@ -69,12 +71,13 @@ class TestNonSupported(unittest.TestCase, ContentMixin):
     def test_missing(self):
         self.assertTrue(os.path.isdir(self.tmp))
         self.reference("""foo = 'fooVal';""")
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.add(File(self.ref, "en-reference.js", ""),
                File(self.l10n, "l10n.js", ""),
                mozpath.join(self.tmp, "merge", "l10n.js"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary': {},
              'details': {'l10n.js': [{'missingFile': 'error'}]}
              }
@@ -90,12 +93,13 @@ class TestNonSupported(unittest.TestCase, ContentMixin):
             return 'ignore'
         self.assertTrue(os.path.isdir(self.tmp))
         self.reference("""foo = 'fooVal';""")
-        cc = ContentComparer([Observer(filter=ignore)])
+        cc = ContentComparer()
+        cc.observers.append(Observer(filter=ignore))
         cc.add(File(self.ref, "en-reference.js", ""),
                File(self.l10n, "l10n.js", ""),
                mozpath.join(self.tmp, "merge", "l10n.js"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary': {},
              'details': {}
              }
@@ -137,12 +141,13 @@ class TestDefines(unittest.TestCase, ContentMixin):
 
 #unfilter emptyLines
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.inc", ""),
                    File(self.l10n, "l10n.inc", ""),
                    mozpath.join(self.tmp, "merge", "l10n.inc"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'changed': 1,
@@ -174,12 +179,13 @@ class TestDefines(unittest.TestCase, ContentMixin):
 
 #unfilter emptyLines
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.inc", ""),
                    File(self.l10n, "l10n.inc", ""),
                    mozpath.join(self.tmp, "merge", "l10n.inc"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'summary':
                     {None: {
@@ -223,12 +229,13 @@ eff = effVal""")
 bar = lBar
 eff = lEff word
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.properties", ""),
                    File(self.l10n, "l10n.properties", ""),
                    mozpath.join(self.tmp, "merge", "l10n.properties"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'changed': 3,
@@ -249,12 +256,13 @@ bar = barVal
 eff = effVal""")
         self.localized("""bar = lBar
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.properties", ""),
                    File(self.l10n, "l10n.properties", ""),
                    mozpath.join(self.tmp, "merge", "l10n.properties"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'changed': 1,
@@ -272,20 +280,21 @@ eff = effVal""")
         self.assertTrue(os.path.isfile(mergefile))
         p = getParser(mergefile)
         p.readFile(mergefile)
-        [m, n] = p.parse()
-        self.assertEqual([e.key for e in m], ["bar", "foo", "eff"])
+        entities = p.parse()
+        self.assertEqual(list(entities.keys()), ["bar", "foo", "eff"])
 
     def test_missing_file(self):
         self.assertTrue(os.path.isdir(self.tmp))
         self.reference("""foo = fooVal
 bar = barVal
 eff = effVal""")
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.add(File(self.ref, "en-reference.properties", ""),
                File(self.l10n, "l10n.properties", ""),
                mozpath.join(self.tmp, "merge", "l10n.properties"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'missingInFiles': 3,
@@ -308,12 +317,13 @@ eff = effVal""")
 bar = %S lBar
 eff = leffVal
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.properties", ""),
                    File(self.l10n, "l10n.properties", ""),
                    mozpath.join(self.tmp, "merge", "l10n.properties"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'changed': 2,
@@ -333,9 +343,9 @@ eff = leffVal
         self.assertTrue(os.path.isfile(mergefile))
         p = getParser(mergefile)
         p.readFile(mergefile)
-        [m, n] = p.parse()
-        self.assertEqual([e.key for e in m], ["eff", "foo", "bar"])
-        self.assertEqual(m[n['bar']].val, '%d barVal')
+        entities = p.parse()
+        self.assertEqual(list(entities.keys()), ["eff", "foo", "bar"])
+        self.assertEqual(entities['bar'].val, '%d barVal')
 
     def testObsolete(self):
         self.assertTrue(os.path.isdir(self.tmp))
@@ -345,12 +355,13 @@ eff = effVal""")
 other = obsolete
 eff = leffVal
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.properties", ""),
                    File(self.l10n, "l10n.properties", ""),
                    mozpath.join(self.tmp, "merge", "l10n.properties"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'changed': 1,
@@ -372,12 +383,13 @@ eff = leffVal
         self.localized("""foo = fooVal
 eff = leffVal
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.remove(File(self.ref, "en-reference.properties", ""),
                   File(self.l10n, "l10n.properties", ""),
                   mozpath.join(self.tmp, "merge", "l10n.properties"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {},
              'details': {
@@ -399,12 +411,13 @@ bar = lBar
 eff = localized eff
 bar = duplicated bar
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.properties", ""),
                    File(self.l10n, "l10n.properties", ""),
                    mozpath.join(self.tmp, "merge", "l10n.properties"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'errors': 1,
@@ -443,12 +456,13 @@ class TestDTD(unittest.TestCase, ContentMixin):
 <!ENTITY bar 'lBar'>
 <!ENTITY eff 'lEff'>
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.dtd", ""),
                    File(self.l10n, "l10n.dtd", ""),
                    mozpath.join(self.tmp, "merge", "l10n.dtd"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'changed': 3,
@@ -469,12 +483,13 @@ class TestDTD(unittest.TestCase, ContentMixin):
 <!ENTITY eff 'effVal'>""")
         self.localized("""<!ENTITY bar 'lBar'>
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.dtd", ""),
                    File(self.l10n, "l10n.dtd", ""),
                    mozpath.join(self.tmp, "merge", "l10n.dtd"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'changed': 1,
@@ -492,8 +507,8 @@ class TestDTD(unittest.TestCase, ContentMixin):
         self.assertTrue(os.path.isfile(mergefile))
         p = getParser(mergefile)
         p.readFile(mergefile)
-        [m, n] = p.parse()
-        self.assertEqual([e.key for e in m], ["bar", "foo", "eff"])
+        entities = p.parse()
+        self.assertEqual(list(entities.keys()), ["bar", "foo", "eff"])
 
     def testJunk(self):
         self.assertTrue(os.path.isdir(self.tmp))
@@ -504,12 +519,13 @@ class TestDTD(unittest.TestCase, ContentMixin):
 <!ENTY bar 'gimmick'>
 <!ENTITY eff 'effVal'>
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.dtd", ""),
                    File(self.l10n, "l10n.dtd", ""),
                    mozpath.join(self.tmp, "merge", "l10n.dtd"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'errors': 1,
@@ -531,8 +547,8 @@ class TestDTD(unittest.TestCase, ContentMixin):
         self.assertTrue(os.path.isfile(mergefile))
         p = getParser(mergefile)
         p.readFile(mergefile)
-        [m, n] = p.parse()
-        self.assertEqual([e.key for e in m], ["foo", "eff", "bar"])
+        entities = p.parse()
+        self.assertEqual(list(entities.keys()), ["foo", "eff", "bar"])
 
     def test_reference_junk(self):
         self.assertTrue(os.path.isdir(self.tmp))
@@ -542,12 +558,13 @@ class TestDTD(unittest.TestCase, ContentMixin):
         self.localized("""<!ENTITY foo 'fooVal'>
 <!ENTITY eff 'effVal'>
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.dtd", ""),
                    File(self.l10n, "l10n.dtd", ""),
                    mozpath.join(self.tmp, "merge", "l10n.dtd"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'warnings': 1,
@@ -571,12 +588,13 @@ class TestDTD(unittest.TestCase, ContentMixin):
 <!ENTITY bar 'good val'>
 <!ENTITY eff 'effVal'>
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.dtd", ""),
                    File(self.l10n, "l10n.dtd", ""),
                    mozpath.join(self.tmp, "merge", "l10n.dtd"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'warnings': 1,
@@ -630,13 +648,14 @@ foo = lFoo
 bar = lBar
 -eff = lEff
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'changed': 3,
@@ -661,13 +680,14 @@ eff = effVal
 foo = lFoo
 eff = lEff
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'details': {
                     'l10n.ftl': [
@@ -702,13 +722,14 @@ foo = lFoo
 bar lBar
 eff = lEff {
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'details': {
                     'l10n.ftl': [
@@ -743,14 +764,14 @@ eff = lEff {
 
         p = getParser(mergepath)
         p.readFile(mergepath)
-        merged_entities, merged_map = p.parse()
-        self.assertEqual([e.key for e in merged_entities], ["foo"])
-        merged_foo = merged_entities[merged_map['foo']]
+        merged_entities = p.parse()
+        self.assertEqual(list(merged_entities.keys()), ["foo"])
+        merged_foo = merged_entities['foo']
 
         # foo should be l10n
         p.readFile(self.l10n)
-        l10n_entities, l10n_map = p.parse()
-        l10n_foo = l10n_entities[l10n_map['foo']]
+        l10n_entities = p.parse()
+        l10n_foo = l10n_entities['foo']
         self.assertTrue(merged_foo.equals(l10n_foo))
 
     def testMatchingReferences(self):
@@ -760,13 +781,14 @@ foo = Reference { bar }
         self.localized("""\
 foo = Localized { bar }
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'details': {},
                 'summary': {
@@ -793,13 +815,14 @@ foo = Localized { qux }
 bar = Localized
 baz = Localized { qux }
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'details': {
                     'l10n.ftl': [
@@ -852,13 +875,14 @@ foo = lFoo
 bar = lBar
 eff = lEff
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'details': {
                     'l10n.ftl': [
@@ -886,14 +910,14 @@ eff = lEff
 
         p = getParser(mergepath)
         p.readFile(mergepath)
-        merged_entities, merged_map = p.parse()
-        self.assertEqual([e.key for e in merged_entities], ["eff"])
-        merged_eff = merged_entities[merged_map['eff']]
+        merged_entities = p.parse()
+        self.assertEqual(list(merged_entities.keys()), ["eff"])
+        merged_eff = merged_entities['eff']
 
         # eff should be l10n
         p.readFile(self.l10n)
-        l10n_entities, l10n_map = p.parse()
-        l10n_eff = l10n_entities[l10n_map['eff']]
+        l10n_entities = p.parse()
+        l10n_eff = l10n_entities['eff']
         self.assertTrue(merged_eff.equals(l10n_eff))
 
     def test_term_attributes(self):
@@ -915,13 +939,14 @@ eff = lEff
 -qux = Localized Qux
     .other = Locale-specific Qux Attribute
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'details': {
                     'l10n.ftl': [
@@ -956,13 +981,14 @@ foo
 bar = lBar
   .tender = localized
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'details': {
                     'l10n.ftl': [
@@ -988,8 +1014,8 @@ bar = lBar
 
         p = getParser(mergepath)
         p.readFile(mergepath)
-        merged_entities, _ = p.parse()
-        self.assertEqual([e.key for e in merged_entities], [])
+        merged_entities = p.parse()
+        self.assertEqual(merged_entities, tuple())
 
     def testMissingGroupComment(self):
         self.reference("""\
@@ -1002,13 +1028,14 @@ bar = barVal
 foo = lFoo
 bar = lBar
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'details': {},
                 'summary': {
@@ -1035,13 +1062,14 @@ bar = barVal
 foo = lFoo
 bar = barVal
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'details': {},
                 'summary': {
@@ -1071,13 +1099,14 @@ foo = lFoo
 
 bar = lBar
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
 
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {
                 'details': {},
                 'summary': {
@@ -1104,12 +1133,13 @@ bar = lBar
 eff = localized eff
 bar = duplicated bar
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'errors': 1,
@@ -1135,12 +1165,13 @@ bar = duplicated bar
     .attr = so
     .attr = good
 """)
-        cc = ContentComparer([Observer()])
+        cc = ContentComparer()
+        cc.observers.append(Observer())
         cc.compare(File(self.ref, "en-reference.ftl", ""),
                    File(self.l10n, "l10n.ftl", ""),
                    mozpath.join(self.tmp, "merge", "l10n.ftl"))
         self.assertDictEqual(
-            cc.observers[0].toJSON(),
+            cc.observers.toJSON(),
             {'summary':
                 {None: {
                     'warnings': 1,

third_party/python/compare-locales/compare_locales/tests/test_merge_comments.py

@@ -18,8 +18,8 @@ bar = Bar 1
 foo = Foo 2
 bar = Bar 2
 """)
-        self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode("utf-8"), """
 foo = Foo 1
 # Bar Comment 1
 bar = Bar 1
@@ -34,10 +34,9 @@ foo = Foo 2
 # Bar Comment 2
 bar = Bar 2
 """)
-        self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode("utf-8"), """
 foo = Foo 1
-# Bar Comment 2
 bar = Bar 1
 """)
 
@@ -51,8 +50,8 @@ foo = Foo 2
 # Bar Comment 2
 bar = Bar 2
 """)
-        self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode("utf-8"), """
 foo = Foo 1
 # Bar Comment 1
 bar = Bar 1
@@ -72,8 +71,8 @@ foo = Foo 1
 # Foo Comment 2
 foo = Foo 2
 """)
-        self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode("utf-8"), """
 # Standalone Comment 1
 
 # Foo Comment 1
@@ -90,8 +89,8 @@ foo = Foo 1
 # Foo Comment 2
 foo = Foo 2
 """)
-        self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode("utf-8"), """
 # Standalone Comment 2
 
 # Foo Comment 1
@@ -110,8 +109,8 @@ foo = Foo 1
 # Foo Comment 2
 foo = Foo 2
 """)
-        self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode("utf-8"), """
 # Standalone Comment 2
 
 # Standalone Comment 1
@@ -132,8 +131,8 @@ foo = Foo 1
 # Foo Comment 2
 foo = Foo 2
 """)
-        self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode("utf-8"), """
 # Standalone Comment
 
 # Foo Comment 1
@@ -153,10 +152,11 @@ bar = Bar 1
 # Bar Comment 2
 bar = Bar 2
 """)
-        self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode("utf-8"), """
 # Ambiguous Comment
 
+# Ambiguous Comment
 foo = Foo 1
 
 # Bar Comment 1
@@ -176,11 +176,13 @@ foo = Foo 1
 # Bar Comment 2
 bar = Bar 2
 """)
-        self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode("utf-8"), """
 # Ambiguous Comment
 foo = Foo 1
 
+# Ambiguous Comment
+
 # Bar Comment 1
 bar = Bar 1
 """)

third_party/python/compare-locales/compare_locales/tests/test_merge_dtd.py

@@ -17,7 +17,7 @@ class TestMergeDTD(unittest.TestCase):
 <!ENTITY foo "Foo 2">
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 <!ENTITY foo "Foo 1">
 """)
 
@@ -27,7 +27,7 @@ class TestMergeDTD(unittest.TestCase):
 """, b"""
 <!ENTITY foo "Foo 2"> \n""")
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 <!ENTITY foo "Foo 1"> \n""")
 
     def test_browser_dtd(self):
@@ -63,8 +63,8 @@ class TestMergeDTD(unittest.TestCase):
 <!ENTITY mainWindow.privatebrowsing "(Private Browsing)">
 """)
 
-        self.assertEqual(
-            merge_channels(self.name, *channels), b"""\
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode("utf-8"), """\
 <!-- This Source Code Form is subject to the terms of the Mozilla Public
    - License, v. 2.0. If a copy of the MPL was not distributed with this
    - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
@@ -117,7 +117,7 @@ class TestMergeDTD(unittest.TestCase):
 """)
 
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""\
+            merge_channels(self.name, channels), b"""\
 <!-- This Source Code Form is subject to the terms of the Mozilla Public
    - License, v. 2.0. If a copy of the MPL was not distributed with this
    - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->

third_party/python/compare-locales/compare_locales/tests/test_merge_ftl.py

@@ -16,7 +16,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 """)
 
@@ -28,7 +28,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
     .attr = Attr 1
 """)
@@ -41,10 +41,38 @@ foo = Foo 2
     .attr = Attr 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 """)
 
+    def test_junk_in_first(self):
+        channels = (b"""\
+line of junk
+""", b"""\
+one = entry
+""")
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode('utf-8'),
+            """\
+one = entry
+line of junk
+"""
+        )
+
+    def test_junk_in_last(self):
+        channels = (b"""\
+one = entry
+""", b"""\
+line of junk
+""")
+        self.assertMultiLineEqual(
+            merge_channels(self.name, channels).decode('utf-8'),
+            """\
+line of junk
+one = entry
+"""
+        )
+
     def test_attribute_changed(self):
         channels = (b"""
 foo = Foo 1
@@ -54,7 +82,7 @@ foo = Foo 2
     .attr = Attr 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
     .attr = Attr 1
 """)
@@ -67,7 +95,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 ## Group Comment 1
 foo = Foo 1
 """)
@@ -80,7 +108,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 ## Group Comment 2
 foo = Foo 1
 """)
@@ -94,7 +122,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 ## Group Comment 2
 ## Group Comment 1
 foo = Foo 1
@@ -110,7 +138,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 // Section Comment
 [[ Section ]]
 ## Group Comment
@@ -125,7 +153,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 # Comment 1
 foo = Foo 1
 """)
@@ -138,7 +166,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 """)
 
@@ -151,7 +179,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 # Comment 1
 foo = Foo 1
 """)
@@ -165,7 +193,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 
 # Comment 1
@@ -180,7 +208,7 @@ foo = Foo 2
 # Comment 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 
 # Comment 2
@@ -197,7 +225,7 @@ foo = Foo 2
 # Comment 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 
 # Comment 2
@@ -214,7 +242,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 ### Resource Comment 1
 
 foo = Foo 1
@@ -229,7 +257,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 ### Resource Comment 1
 
 foo = Foo 1
@@ -246,7 +274,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 ### Resource Comment 2
 
 ### Resource Comment 1
@@ -265,7 +293,7 @@ foo
     .attr = Attribute 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 # Comment 1
 foo =
     .attr = Attribute 1
@@ -285,7 +313,7 @@ foo
     .attr = Attribute 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 # Same comment
 
 foo =

third_party/python/compare-locales/compare_locales/tests/test_merge_messages.py

@@ -16,7 +16,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 """)
 
@@ -28,7 +28,7 @@ bar = Bar 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 bar = Bar 1
 """)
@@ -41,7 +41,7 @@ foo = Foo 2
 bar = Bar 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 bar = Bar 2
 """)
@@ -55,7 +55,7 @@ bar = Bar 2
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 bar = Bar 1
 """)
@@ -73,7 +73,7 @@ foo = Foo 2
 foo = Foo 3
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 """)
 
@@ -87,7 +87,7 @@ foo = Foo 3
 bar = Bar 3
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 bar = Bar 3
 """)

third_party/python/compare-locales/compare_locales/tests/test_merge_properties.py

@@ -20,7 +20,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 """)
 
@@ -30,7 +30,7 @@ foo = Foo 1…
 """, "utf8"), encode(u"""
 foo = Foo 2…
 """, "utf8"))
-        output = merge_channels(self.name, *channels)
+        output = merge_channels(self.name, channels)
         self.assertEqual(output, encode(u"""
 foo = Foo 1…
 """, "utf8"))
@@ -39,3 +39,30 @@ foo = Foo 1…
         self.assertEqual(u_output, u"""
 foo = Foo 1…
 """)
+
+    def test_repetitive(self):
+        channels = (b"""\
+# comment
+one = one
+# comment
+three = three
+""", b"""\
+# comment
+one = one
+# comment
+two = two
+# comment
+three = three
+""")
+        output = merge_channels(self.name, channels)
+        self.assertMultiLineEqual(
+            decode(output, "utf-8"),
+            """\
+# comment
+one = one
+# comment
+two = two
+# comment
+three = three
+"""
+        )

third_party/python/compare-locales/compare_locales/tests/test_merge_unknown.py

@@ -19,4 +19,4 @@ foo = Foo 2
 """)
         pattern = "Unsupported file format \(foo\.unknown\)\."
         with six.assertRaisesRegex(self, MergeNotSupportedError, pattern):
-            merge_channels(self.name, *channels)
+            merge_channels(self.name, channels)

third_party/python/compare-locales/compare_locales/tests/test_merge_whitespace.py

@@ -16,7 +16,7 @@ foo = Foo 1
 foo = Foo 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
       """)
 
@@ -30,7 +30,7 @@ foo = Foo 2
 bar = Bar 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 
 bar = Bar 1
@@ -43,7 +43,7 @@ foo = Foo 2
 bar = Bar 2
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 foo = Foo 1
 bar = Bar 2
 """)
@@ -65,7 +65,7 @@ baz = Baz 2
 
 """)
         self.assertEqual(
-            merge_channels(self.name, *channels), b"""
+            merge_channels(self.name, channels), b"""
 
 foo = Foo 1
 

third_party/python/compare-locales/compare_locales/tests/test_parser.py

@@ -41,14 +41,10 @@ third line
 
     def test_empty_parser(self):
         p = parser.Parser()
-        entities, _map = p.parse()
-        self.assertListEqual(
+        entities = p.parse()
+        self.assertTupleEqual(
             entities,
-            []
-        )
-        self.assertDictEqual(
-            _map,
-            {}
+            tuple()
         )
 
 

third_party/python/compare-locales/compare_locales/tests/test_paths.py

@@ -1,609 +0,0 @@
-# -*- coding: utf-8 -*-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import
-import unittest
-
-from compare_locales.paths import (
-    ProjectConfig, File, ProjectFiles, Matcher, TOMLParser
-)
-from compare_locales import mozpath
-import pytoml as toml
-
-
-class TestMatcher(unittest.TestCase):
-
-    def test_matcher(self):
-        one = Matcher('foo/*')
-        self.assertTrue(one.match('foo/baz'))
-        self.assertFalse(one.match('foo/baz/qux'))
-        other = Matcher('bar/*')
-        self.assertTrue(other.match('bar/baz'))
-        self.assertFalse(other.match('bar/baz/qux'))
-        self.assertEqual(one.sub(other, 'foo/baz'), 'bar/baz')
-        self.assertIsNone(one.sub(other, 'bar/baz'))
-        one = Matcher('foo/**')
-        self.assertTrue(one.match('foo/baz'))
-        self.assertTrue(one.match('foo/baz/qux'))
-        other = Matcher('bar/**')
-        self.assertTrue(other.match('bar/baz'))
-        self.assertTrue(other.match('bar/baz/qux'))
-        self.assertEqual(one.sub(other, 'foo/baz'), 'bar/baz')
-        self.assertEqual(one.sub(other, 'foo/baz/qux'), 'bar/baz/qux')
-        one = Matcher('foo/*/one/**')
-        self.assertTrue(one.match('foo/baz/one/qux'))
-        self.assertFalse(one.match('foo/baz/bez/one/qux'))
-        other = Matcher('bar/*/other/**')
-        self.assertTrue(other.match('bar/baz/other/qux'))
-        self.assertFalse(other.match('bar/baz/bez/other/qux'))
-        self.assertEqual(one.sub(other, 'foo/baz/one/qux'),
-                         'bar/baz/other/qux')
-        self.assertEqual(one.sub(other, 'foo/baz/one/qux/zzz'),
-                         'bar/baz/other/qux/zzz')
-        self.assertIsNone(one.sub(other, 'foo/baz/bez/one/qux'))
-        one = Matcher('foo/**/bar/**')
-        self.assertTrue(one.match('foo/bar/baz.qux'))
-        self.assertTrue(one.match('foo/tender/bar/baz.qux'))
-        self.assertFalse(one.match('foo/nobar/baz.qux'))
-        self.assertFalse(one.match('foo/tender/bar'))
-
-    def test_prefix(self):
-        self.assertEqual(
-            Matcher('foo/bar.file').prefix, 'foo/bar.file'
-        )
-        self.assertEqual(
-            Matcher('foo/*').prefix, 'foo/'
-        )
-        self.assertEqual(
-            Matcher('foo/**').prefix, 'foo'
-        )
-        self.assertEqual(
-            Matcher('foo/*/bar').prefix, 'foo/'
-        )
-        self.assertEqual(
-            Matcher('foo/**/bar').prefix, 'foo'
-        )
-
-
-class SetupMixin(object):
-    def setUp(self):
-        self.cfg = ProjectConfig()
-        self.file = File(
-            '/tmp/somedir/de/browser/one/two/file.ftl',
-            'file.ftl',
-            module='browser', locale='de')
-        self.other_file = File(
-            '/tmp/somedir/de/toolkit/two/one/file.ftl',
-            'file.ftl',
-            module='toolkit', locale='de')
-
-
-class TestConfigLegacy(SetupMixin, unittest.TestCase):
-
-    def test_filter_py_true(self):
-        'Test filter.py just return bool(True)'
-        def filter(mod, path, entity=None):
-            return True
-        self.cfg.set_filter_py(filter)
-        with self.assertRaises(AssertionError):
-            self.cfg.add_rules({})
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'error')
-        rv = self.cfg.filter(self.file, entity='one_entity')
-        self.assertEqual(rv, 'error')
-
-    def test_filter_py_false(self):
-        'Test filter.py just return bool(False)'
-        def filter(mod, path, entity=None):
-            return False
-        self.cfg.set_filter_py(filter)
-        with self.assertRaises(AssertionError):
-            self.cfg.add_rules({})
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.file, entity='one_entity')
-        self.assertEqual(rv, 'ignore')
-
-    def test_filter_py_error(self):
-        'Test filter.py just return str("error")'
-        def filter(mod, path, entity=None):
-            return 'error'
-        self.cfg.set_filter_py(filter)
-        with self.assertRaises(AssertionError):
-            self.cfg.add_rules({})
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'error')
-        rv = self.cfg.filter(self.file, entity='one_entity')
-        self.assertEqual(rv, 'error')
-
-    def test_filter_py_ignore(self):
-        'Test filter.py just return str("ignore")'
-        def filter(mod, path, entity=None):
-            return 'ignore'
-        self.cfg.set_filter_py(filter)
-        with self.assertRaises(AssertionError):
-            self.cfg.add_rules({})
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.file, entity='one_entity')
-        self.assertEqual(rv, 'ignore')
-
-    def test_filter_py_report(self):
-        'Test filter.py just return str("report") and match to "warning"'
-        def filter(mod, path, entity=None):
-            return 'report'
-        self.cfg.set_filter_py(filter)
-        with self.assertRaises(AssertionError):
-            self.cfg.add_rules({})
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'warning')
-        rv = self.cfg.filter(self.file, entity='one_entity')
-        self.assertEqual(rv, 'warning')
-
-    def test_filter_py_module(self):
-        'Test filter.py to return str("error") for browser or "ignore"'
-        def filter(mod, path, entity=None):
-            return 'error' if mod == 'browser' else 'ignore'
-        self.cfg.set_filter_py(filter)
-        with self.assertRaises(AssertionError):
-            self.cfg.add_rules({})
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'error')
-        rv = self.cfg.filter(self.file, entity='one_entity')
-        self.assertEqual(rv, 'error')
-        rv = self.cfg.filter(self.other_file)
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.other_file, entity='one_entity')
-        self.assertEqual(rv, 'ignore')
-
-
-class TestConfigRules(SetupMixin, unittest.TestCase):
-
-    def test_filter_empty(self):
-        'Test that an empty config works'
-        self.cfg.add_paths({
-            'l10n': '/tmp/somedir/{locale}/browser/**'
-        })
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'error')
-        rv = self.cfg.filter(self.file, entity='one_entity')
-        self.assertEqual(rv, 'error')
-        rv = self.cfg.filter(self.other_file)
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.other_file, entity='one_entity')
-        self.assertEqual(rv, 'ignore')
-
-    def test_single_file_rule(self):
-        'Test a single rule for just a single file, no key'
-        self.cfg.add_paths({
-            'l10n': '/tmp/somedir/{locale}/browser/**'
-        })
-        self.cfg.add_rules({
-            'path': '/tmp/somedir/{locale}/browser/one/two/file.ftl',
-            'action': 'ignore'
-        })
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.file, 'one_entity')
-        self.assertEqual(rv, 'error')
-        rv = self.cfg.filter(self.other_file)
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.other_file, 'one_entity')
-        self.assertEqual(rv, 'ignore')
-
-    def test_single_key_rule(self):
-        'Test a single rule with file and key'
-        self.cfg.add_paths({
-            'l10n': '/tmp/somedir/{locale}/browser/**'
-        })
-        self.cfg.add_rules({
-            'path': '/tmp/somedir/{locale}/browser/one/two/file.ftl',
-            'key': 'one_entity',
-            'action': 'ignore'
-        })
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'error')
-        rv = self.cfg.filter(self.file, 'one_entity')
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.other_file)
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.other_file, 'one_entity')
-        self.assertEqual(rv, 'ignore')
-
-    def test_single_non_matching_key_rule(self):
-        'Test a single key rule with regex special chars that should not match'
-        self.cfg.add_paths({
-            'l10n': '/tmp/somedir/{locale}/**'
-        })
-        self.cfg.add_rules({
-            'path': '/tmp/somedir/{locale}/browser/one/two/file.ftl',
-            'key': '.ne_entit.',
-            'action': 'ignore'
-        })
-        rv = self.cfg.filter(self.file, 'one_entity')
-        self.assertEqual(rv, 'error')
-
-    def test_single_matching_re_key_rule(self):
-        'Test a single key with regular expression'
-        self.cfg.add_paths({
-            'l10n': '/tmp/somedir/{locale}/**'
-        })
-        self.cfg.add_rules({
-            'path': '/tmp/somedir/{locale}/browser/one/two/file.ftl',
-            'key': 're:.ne_entit.$',
-            'action': 'ignore'
-        })
-        rv = self.cfg.filter(self.file, 'one_entity')
-        self.assertEqual(rv, 'ignore')
-
-    def test_double_file_rule(self):
-        'Test path shortcut, one for each of our files'
-        self.cfg.add_paths({
-            'l10n': '/tmp/somedir/{locale}/**'
-        })
-        self.cfg.add_rules({
-            'path': [
-                '/tmp/somedir/{locale}/browser/one/two/file.ftl',
-                '/tmp/somedir/{locale}/toolkit/two/one/file.ftl',
-            ],
-            'action': 'ignore'
-        })
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.other_file)
-        self.assertEqual(rv, 'ignore')
-
-    def test_double_file_key_rule(self):
-        'Test path and key shortcut, one key matching, one not'
-        self.cfg.add_paths({
-            'l10n': '/tmp/somedir/{locale}/**'
-        })
-        self.cfg.add_rules({
-            'path': [
-                '/tmp/somedir/{locale}/browser/one/two/file.ftl',
-                '/tmp/somedir/{locale}/toolkit/two/one/file.ftl',
-            ],
-            'key': [
-                'one_entity',
-                'other_entity',
-            ],
-            'action': 'ignore'
-        })
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'error')
-        rv = self.cfg.filter(self.file, 'one_entity')
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.other_file)
-        self.assertEqual(rv, 'error')
-        rv = self.cfg.filter(self.other_file, 'one_entity')
-        self.assertEqual(rv, 'ignore')
-
-    def test_single_wildcard_rule(self):
-        'Test single wildcard'
-        self.cfg.add_paths({
-            'l10n': '/tmp/somedir/{locale}/browser/**'
-        })
-        self.cfg.add_rules({
-            'path': [
-                '/tmp/somedir/{locale}/browser/one/*/*',
-            ],
-            'action': 'ignore'
-        })
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.other_file)
-        self.assertEqual(rv, 'ignore')
-
-    def test_double_wildcard_rule(self):
-        'Test double wildcard'
-        self.cfg.add_paths({
-            'l10n': '/tmp/somedir/{locale}/**'
-        })
-        self.cfg.add_rules({
-            'path': [
-                '/tmp/somedir/{locale}/**',
-            ],
-            'action': 'ignore'
-        })
-        rv = self.cfg.filter(self.file)
-        self.assertEqual(rv, 'ignore')
-        rv = self.cfg.filter(self.other_file)
-        self.assertEqual(rv, 'ignore')
-
-
-class MockProjectFiles(ProjectFiles):
-    def __init__(self, mocks, locale, projects, mergebase=None):
-        (super(MockProjectFiles, self)
-            .__init__(locale, projects, mergebase=mergebase))
-        self.mocks = mocks
-
-    def _files(self, matcher):
-        base = matcher.prefix
-        for path in self.mocks.get(base, []):
-            p = mozpath.join(base, path)
-            if matcher.match(p):
-                yield p
-
-
-class TestProjectPaths(unittest.TestCase):
-    def test_l10n_path(self):
-        cfg = ProjectConfig()
-        cfg.add_environment(l10n_base='/tmp')
-        cfg.locales.append('de')
-        cfg.add_paths({
-            'l10n': '{l10n_base}/{locale}/*'
-        })
-        mocks = {
-            '/tmp/de/': [
-                'good.ftl',
-                'not/subdir/bad.ftl'
-            ],
-            '/tmp/fr/': [
-                'good.ftl',
-                'not/subdir/bad.ftl'
-            ],
-        }
-        files = MockProjectFiles(mocks, 'de', [cfg])
-        self.assertListEqual(
-            list(files), [('/tmp/de/good.ftl', None, None, set())])
-        self.assertTupleEqual(
-            files.match('/tmp/de/something.ftl'),
-            ('/tmp/de/something.ftl', None, None, set()))
-        self.assertIsNone(files.match('/tmp/fr/something.ftl'))
-        files = MockProjectFiles(mocks, 'de', [cfg], mergebase='merging')
-        self.assertListEqual(
-            list(files),
-            [('/tmp/de/good.ftl', None, 'merging/de/good.ftl', set())])
-        self.assertTupleEqual(
-            files.match('/tmp/de/something.ftl'),
-            ('/tmp/de/something.ftl', None, 'merging/de/something.ftl', set()))
-        # 'fr' is not in the locale list, should return no files
-        files = MockProjectFiles(mocks, 'fr', [cfg])
-        self.assertListEqual(list(files), [])
-
-    def test_reference_path(self):
-        cfg = ProjectConfig()
-        cfg.add_environment(l10n_base='/tmp/l10n')
-        cfg.locales.append('de')
-        cfg.add_paths({
-            'l10n': '{l10n_base}/{locale}/*',
-            'reference': '/tmp/reference/*'
-        })
-        mocks = {
-            '/tmp/l10n/de/': [
-                'good.ftl',
-                'not/subdir/bad.ftl'
-            ],
-            '/tmp/l10n/fr/': [
-                'good.ftl',
-                'not/subdir/bad.ftl'
-            ],
-            '/tmp/reference/': [
-                'ref.ftl',
-                'not/subdir/bad.ftl'
-            ],
-        }
-        files = MockProjectFiles(mocks, 'de', [cfg])
-        self.assertListEqual(
-            list(files),
-            [
-                ('/tmp/l10n/de/good.ftl', '/tmp/reference/good.ftl', None,
-                 set()),
-                ('/tmp/l10n/de/ref.ftl', '/tmp/reference/ref.ftl', None,
-                 set()),
-            ])
-        self.assertTupleEqual(
-            files.match('/tmp/l10n/de/good.ftl'),
-            ('/tmp/l10n/de/good.ftl', '/tmp/reference/good.ftl', None,
-             set()),
-            )
-        self.assertTupleEqual(
-            files.match('/tmp/reference/good.ftl'),
-            ('/tmp/l10n/de/good.ftl', '/tmp/reference/good.ftl', None,
-             set()),
-            )
-        self.assertIsNone(files.match('/tmp/l10n/de/subdir/bad.ftl'))
-        self.assertIsNone(files.match('/tmp/reference/subdir/bad.ftl'))
-        files = MockProjectFiles(mocks, 'de', [cfg], mergebase='merging')
-        self.assertListEqual(
-            list(files),
-            [
-                ('/tmp/l10n/de/good.ftl', '/tmp/reference/good.ftl',
-                 'merging/de/good.ftl', set()),
-                ('/tmp/l10n/de/ref.ftl', '/tmp/reference/ref.ftl',
-                 'merging/de/ref.ftl', set()),
-            ])
-        self.assertTupleEqual(
-            files.match('/tmp/l10n/de/good.ftl'),
-            ('/tmp/l10n/de/good.ftl', '/tmp/reference/good.ftl',
-             'merging/de/good.ftl', set()),
-            )
-        self.assertTupleEqual(
-            files.match('/tmp/reference/good.ftl'),
-            ('/tmp/l10n/de/good.ftl', '/tmp/reference/good.ftl',
-             'merging/de/good.ftl', set()),
-            )
-        # 'fr' is not in the locale list, should return no files
-        files = MockProjectFiles(mocks, 'fr', [cfg])
-        self.assertListEqual(list(files), [])
-
-    def test_partial_l10n(self):
-        cfg = ProjectConfig()
-        cfg.locales.extend(['de', 'fr'])
-        cfg.add_paths({
-            'l10n': '/tmp/{locale}/major/*'
-        }, {
-            'l10n': '/tmp/{locale}/minor/*',
-            'locales': ['de']
-        })
-        mocks = {
-            '/tmp/de/major/': [
-                'good.ftl',
-                'not/subdir/bad.ftl'
-            ],
-            '/tmp/de/minor/': [
-                'good.ftl',
-            ],
-            '/tmp/fr/major/': [
-                'good.ftl',
-                'not/subdir/bad.ftl'
-            ],
-            '/tmp/fr/minor/': [
-                'good.ftl',
-            ],
-        }
-        files = MockProjectFiles(mocks, 'de', [cfg])
-        self.assertListEqual(
-            list(files),
-            [
-                ('/tmp/de/major/good.ftl', None, None, set()),
-                ('/tmp/de/minor/good.ftl', None, None, set()),
-            ])
-        self.assertTupleEqual(
-            files.match('/tmp/de/major/some.ftl'),
-            ('/tmp/de/major/some.ftl', None, None, set()))
-        self.assertIsNone(files.match('/tmp/de/other/some.ftl'))
-        # 'fr' is not in the locale list of minor, should only return major
-        files = MockProjectFiles(mocks, 'fr', [cfg])
-        self.assertListEqual(
-            list(files),
-            [
-                ('/tmp/fr/major/good.ftl', None, None, set()),
-            ])
-        self.assertIsNone(files.match('/tmp/fr/minor/some.ftl'))
-
-    def test_validation_mode(self):
-        cfg = ProjectConfig()
-        cfg.add_environment(l10n_base='/tmp/l10n')
-        cfg.locales.append('de')
-        cfg.add_paths({
-            'l10n': '{l10n_base}/{locale}/*',
-            'reference': '/tmp/reference/*'
-        })
-        mocks = {
-            '/tmp/l10n/de/': [
-                'good.ftl',
-                'not/subdir/bad.ftl'
-            ],
-            '/tmp/l10n/fr/': [
-                'good.ftl',
-                'not/subdir/bad.ftl'
-            ],
-            '/tmp/reference/': [
-                'ref.ftl',
-                'not/subdir/bad.ftl'
-            ],
-        }
-        # `None` switches on validation mode
-        files = MockProjectFiles(mocks, None, [cfg])
-        self.assertListEqual(
-            list(files),
-            [
-                ('/tmp/reference/ref.ftl', '/tmp/reference/ref.ftl', None,
-                 set()),
-            ])
-
-
-class MockTOMLParser(TOMLParser):
-    def __init__(self, path_data, env=None, ignore_missing_includes=False):
-        # mock, use the path as data. Yeah, not nice
-        super(MockTOMLParser, self).__init__(
-            '/tmp/base.toml',
-            env=env, ignore_missing_includes=ignore_missing_includes
-        )
-        self.data = toml.loads(path_data)
-
-    def load(self):
-        # we mocked this
-        pass
-
-
-class TestL10nMerge(unittest.TestCase):
-    # need to go through TOMLParser, as that's handling most of the
-    # environment
-    def test_merge_paths(self):
-        cfg = MockTOMLParser.parse(
-            '''\
-basepath = "."
-locales = [
-    "de",
-]
-[env]
-    l = "{l10n_base}/{locale}/"
-[[paths]]
-    reference = "reference/*"
-    l10n = "{l}*"
-''',
-            env={'l10n_base': '/tmp/l10n'}
-        )
-        mocks = {
-            '/tmp/l10n/de/': [
-                'good.ftl',
-                'not/subdir/bad.ftl'
-            ],
-            '/tmp/l10n/fr/': [
-                'good.ftl',
-                'not/subdir/bad.ftl'
-            ],
-            '/tmp/reference/': [
-                'ref.ftl',
-                'not/subdir/bad.ftl'
-            ],
-        }
-        cfg.add_global_environment(l10n_base='/tmp/l10n')
-        files = MockProjectFiles(mocks, 'de', [cfg], '/tmp/mergers')
-        self.assertListEqual(
-            list(files),
-            [
-                ('/tmp/l10n/de/good.ftl', '/tmp/reference/good.ftl',
-                 '/tmp/mergers/de/good.ftl',
-                 set()),
-                ('/tmp/l10n/de/ref.ftl', '/tmp/reference/ref.ftl',
-                 '/tmp/mergers/de/ref.ftl',
-                 set()),
-            ])
-
-
-class TestProjectConfig(unittest.TestCase):
-    def test_expand_paths(self):
-        pc = ProjectConfig()
-        pc.add_environment(one="first_path")
-        self.assertEqual(pc.expand('foo'), 'foo')
-        self.assertEqual(pc.expand('foo{one}bar'), 'foofirst_pathbar')
-        pc.add_environment(l10n_base='../tmp/localizations')
-        self.assertEqual(
-            pc.expand('{l}dir', {'l': '{l10n_base}/{locale}/'}),
-            '../tmp/localizations/{locale}/dir')
-        self.assertEqual(
-            pc.expand('{l}dir', {
-                'l': '{l10n_base}/{locale}/',
-                'l10n_base': '../merge-base'
-            }),
-            '../merge-base/{locale}/dir')
-
-    def test_children(self):
-        pc = ProjectConfig()
-        child = ProjectConfig()
-        pc.add_child(child)
-        self.assertListEqual([pc, child], list(pc.configs))
-
-
-class TestFile(unittest.TestCase):
-    def test_hash_and_equality(self):
-        f1 = File('/tmp/full/path/to/file', 'path/to/file')
-        d = {}
-        d[f1] = True
-        self.assertIn(f1, d)
-        f2 = File('/tmp/full/path/to/file', 'path/to/file')
-        self.assertIn(f2, d)
-        f2 = File('/tmp/full/path/to/file', 'path/to/file', locale='en')
-        self.assertNotIn(f2, d)
-        # trigger hash collisions between File and non-File objects
-        self.assertEqual(hash(f1), hash(f1.localpath))
-        self.assertNotIn(f1.localpath, d)
-        f1 = File('/tmp/full/other/path', 'other/path')
-        d[f1.localpath] = True
-        self.assertIn(f1.localpath, d)
-        self.assertNotIn(f1, d)

third_party/python/compare-locales/compare_locales/tests/test_properties.py

@@ -48,6 +48,7 @@ and still has another line coming
                'and here is the 2nd part']
         i = iter(self.parser)
         for r, e in zip(ref, i):
+            self.assertTrue(e.localized)
             self.assertEqual(e.val, r)
 
     def test_bug121341(self):
@@ -155,6 +156,18 @@ foo = bar
             (Comment, 'LOCALIZATION NOTE'),
             (Whitespace, '\n\n\n')))
 
+    def test_standalone_license(self):
+        self._test('''\
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+foo = value
+''', (
+            (Comment, 'MPL'),
+            (Whitespace, '\n'),
+            ('foo', 'value'),
+            (Whitespace, '\n')))
+
     def test_empty_file(self):
         self._test('', tuple())
         self._test('\n', ((Whitespace, '\n'),))
@@ -207,6 +220,24 @@ t\xa0e = three\xa0''', (
             ('t\xa0e', 'three\xa0'),
         ))
 
+    def test_pre_comment(self):
+        self._test('''\
+# comment
+one = string
+
+# standalone
+
+# glued
+second = string
+''', (
+            ('one', 'string', 'comment'),
+            (Whitespace, '\n\n'),
+            (Comment, 'standalone'),
+            (Whitespace, '\n\n'),
+            ('second', 'string', 'glued'),
+            (Whitespace, '\n'),
+        ))
+
 
 if __name__ == '__main__':
     unittest.main()

third_party/python/fluent/fluent/migrate/blame.py

@@ -7,6 +7,7 @@ import json
 import os
 
 from compare_locales.parser import getParser, Junk
+from compare_locales import mozpath
 import hglib
 from hglib.util import b, cmdbuilder
 
@@ -31,7 +32,7 @@ class Blame(object):
                 'blame': self.blame}
 
     def handleFile(self, file_blame):
-        path = file_blame['path']
+        path = mozpath.normsep(file_blame['path'])
 
         try:
             parser = getParser(path)
@@ -41,7 +42,7 @@ class Blame(object):
         self.blame[path] = {}
 
         self.readFile(parser, path)
-        entities, emap = parser.parse()
+        entities = parser.parse()
         for e in entities:
             if isinstance(e, Junk):
                 continue

third_party/python/fluent/fluent/migrate/helpers.py

@@ -63,6 +63,11 @@ def transforms_from(ftl, **substitutions):
     def into_argument(node):
         """Convert AST node into an argument to migration transforms."""
         if isinstance(node, FTL.StringLiteral):
+            # Special cases for booleans which don't exist in Fluent.
+            if node.value == "True":
+                return True
+            if node.value == "False":
+                return False
             return node.value
         if isinstance(node, FTL.MessageReference):
             try:
@@ -88,7 +93,10 @@ def transforms_from(ftl, **substitutions):
             name = node.callee.name
             if name == "COPY":
                 args = (into_argument(arg) for arg in node.positional)
-                return COPY(*args)
+                kwargs = {
+                    arg.name.name: into_argument(arg.value)
+                    for arg in node.named}
+                return COPY(*args, **kwargs)
             if name in IMPLICIT_TRANSFORMS:
                 raise NotSupportedError(
                     "{} may not be used with transforms_from(). It runs "

third_party/python/fluent/fluent/migrate/tool.py

@@ -39,9 +39,9 @@ def main(lang, reference_dir, localization_dir, migrations, dry_run):
         try:
             # Add the migration spec.
             migration.migrate(ctx)
-        except MigrationError:
-            print('  Skipping migration {} for {}'.format(
-                migration.__name__, lang))
+        except MigrationError as e:
+            print('  Skipping migration {} for {}:\n    {}'.format(
+                migration.__name__, lang, e))
             continue
 
         # Keep track of how many changesets we're committing.

third_party/python/fluent/fluent/migrate/transforms.py

@@ -108,8 +108,14 @@ def chain_elements(elements):
                 'Expected Pattern, PatternElement or Expression')
 
 
-re_leading_ws = re.compile(r'^(?P<whitespace>\s+)(?P<text>.*?)$')
-re_trailing_ws = re.compile(r'^(?P<text>.*?)(?P<whitespace>\s+)$')
+re_leading_ws = re.compile(
+    r'\A(?:(?P<whitespace> +)(?P<text>.*?)|(?P<block_text>\n.*?))\Z',
+    re.S,
+)
+re_trailing_ws = re.compile(
+    r'\A(?:(?P<text>.*?)(?P<whitespace> +)|(?P<block_text>.*\n))\Z',
+    re.S
+)
 
 
 def extract_whitespace(regex, element):
@@ -119,15 +125,21 @@ def extract_whitespace(regex, element):
     encodes the extracted whitespace as a StringLiteral and the
     TextElement has the same amount of whitespace removed. The
     Placeable with the extracted whitespace is always returned first.
+    If the element starts or ends with a newline, add an empty
+    StringLiteral.
     '''
     match = re.search(regex, element.value)
     if match:
-        whitespace = match.group('whitespace')
+        # If white-space is None, we're a newline. Add an
+        # empty { "" }
+        whitespace = match.group('whitespace') or ''
         placeable = FTL.Placeable(FTL.StringLiteral(whitespace))
         if whitespace == element.value:
             return placeable, None
         else:
-            return placeable, FTL.TextElement(match.group('text'))
+            # Either text or block_text matched the rest.
+            text = match.group('text') or match.group('block_text')
+            return placeable, FTL.TextElement(text)
     else:
         return None, element
 
@@ -198,7 +210,7 @@ class Source(Transform):
 
     """
 
-    def __init__(self, path, key):
+    def __init__(self, path, key, trim=False):
         if path.endswith('.ftl'):
             raise NotSupportedError(
                 'Migrating translations from Fluent files is not supported '
@@ -206,9 +218,25 @@ class Source(Transform):
 
         self.path = path
         self.key = key
+        self.trim = trim
+
+    def get_text(self, ctx):
+        return ctx.get_source(self.path, self.key)
+
+    @staticmethod
+    def trim_text(text):
+        # strip leading white-space
+        text = re.sub('^[ \t]+', '', text, flags=re.M)
+        # strip trailing white-space
+        text = re.sub('[ \t]+$', '', text, flags=re.M)
+        # strip leading and trailing empty lines
+        text = text.strip('\r\n')
+        return text
 
     def __call__(self, ctx):
-        text = ctx.get_source(self.path, self.key)
+        text = self.get_text(ctx)
+        if self.trim:
+            text = self.trim_text(text)
         return FTL.TextElement(text)
 
 
@@ -220,47 +248,97 @@ class COPY(Source):
         return Transform.pattern_of(element)
 
 
+PRINTF = re.compile(
+    r'%(?P<good>%|'
+    r'(?:(?P<number>[1-9][0-9]*)\$)?'
+    r'(?P<width>\*|[0-9]+)?'
+    r'(?P<prec>\.(?:\*|[0-9]+)?)?'
+    r'(?P<spec>[duxXosScpfg]))'
+)
+
+
+def number():
+    i = 1
+    while True:
+        yield i
+        i += 1
+
+
+def normalize_printf(text):
+    """Normalize printf arguments so that they're all numbered.
+    Gecko forbids mixing unnumbered and numbered ones, so
+    we just need to convert unnumbered to numbered ones.
+    Also remove ones that have zero width, as they're intended
+    to be removed from the output by the localizer.
+    """
+    next_number = number()
+
+    def normalized(match):
+        if match.group('good') == '%':
+            return '%'
+        hidden = match.group('width') == '0'
+        if match.group('number'):
+            return '' if hidden else match.group()
+        num = next(next_number)
+        return '' if hidden else '%{}${}'.format(num, match.group('spec'))
+
+    return PRINTF.sub(normalized, text)
+
+
 class REPLACE_IN_TEXT(Transform):
     """Create a Pattern from a TextElement and replace legacy placeables.
 
     The original placeables are defined as keys on the `replacements` dict.
-    For each key the value is defined as a FTL Pattern, Placeable,
-    TextElement or Expressions to be interpolated.
+    For each key the value must be defined as a FTL Pattern, Placeable,
+    TextElement or Expression to be interpolated.
     """
 
-    def __init__(self, element, replacements):
+    def __init__(self, element, replacements, normalize_printf=False):
         self.element = element
         self.replacements = replacements
+        self.normalize_printf = normalize_printf
 
     def __call__(self, ctx):
-        # Only replace placeables which are present in the translation.
-        replacements = {
-            key: evaluate(ctx, repl)
-            for key, repl in self.replacements.items()
-            if key in self.element.value
+        # For each specified replacement, find all indices of the original
+        # placeable in the source translation. If missing, the list of indices
+        # will be empty.
+        value = self.element.value
+        if self.normalize_printf:
+            value = normalize_printf(value)
+        key_indices = {
+            key: [m.start() for m in re.finditer(re.escape(key), value)]
+            for key in self.replacements.keys()
         }
 
-        # Order the original placeables by their position in the translation.
-        keys_in_order = sorted(
-            replacements.keys(),
-            key=lambda x: self.element.value.find(x)
+        # Build a dict of indices to replacement keys.
+        keys_indexed = {}
+        for key, indices in key_indices.items():
+            for index in indices:
+                keys_indexed[index] = key
+
+        # Order the replacements by the position of the original placeable in
+        # the translation.
+        replacements = (
+            (key, evaluate(ctx, self.replacements[key]))
+            for index, key
+            in sorted(keys_indexed.items(), key=lambda x: x[0])
         )
 
         # A list of PatternElements built from the legacy translation and the
         # FTL replacements. It may contain empty or adjacent TextElements.
         elements = []
-        tail = self.element.value
+        tail = value
 
         # Convert original placeables and text into FTL Nodes. For each
         # original placeable the translation will be partitioned around it and
         # the text before it will be converted into an `FTL.TextElement` and
         # the placeable will be replaced with its replacement.
-        for key in keys_in_order:
+        for key, node in replacements:
             before, key, tail = tail.partition(key)
             elements.append(FTL.TextElement(before))
-            elements.append(replacements[key])
+            elements.append(node)
 
-        # Dont' forget about the tail after the loop ends.
+        # Don't forget about the tail after the loop ends.
         elements.append(FTL.TextElement(tail))
         return Transform.pattern_of(*elements)
 
@@ -272,13 +350,20 @@ class REPLACE(Source):
     replaced with FTL placeables using the `REPLACE_IN_TEXT` transform.
     """
 
-    def __init__(self, path, key, replacements):
-        super(REPLACE, self).__init__(path, key)
+    def __init__(
+        self, path, key, replacements,
+        normalize_printf=False, **kwargs
+    ):
+        super(REPLACE, self).__init__(path, key, **kwargs)
         self.replacements = replacements
+        self.normalize_printf = normalize_printf
 
     def __call__(self, ctx):
         element = super(REPLACE, self).__call__(ctx)
-        return REPLACE_IN_TEXT(element, self.replacements)(ctx)
+        return REPLACE_IN_TEXT(
+            element, self.replacements,
+            normalize_printf=self.normalize_printf
+        )(ctx)
 
 
 class PLURALS(Source):
@@ -293,8 +378,9 @@ class PLURALS(Source):
     """
     DEFAULT_ORDER = ('zero', 'one', 'two', 'few', 'many', 'other')
 
-    def __init__(self, path, key, selector, foreach=Transform.pattern_of):
-        super(PLURALS, self).__init__(path, key)
+    def __init__(self, path, key, selector, foreach=Transform.pattern_of,
+                 **kwargs):
+        super(PLURALS, self).__init__(path, key, **kwargs)
         self.selector = selector
         self.foreach = foreach
 
@@ -349,7 +435,7 @@ class PLURALS(Source):
             # variant. Then evaluate it to a migrated FTL node.
             value = evaluate(ctx, self.foreach(form))
             return FTL.Variant(
-                key=FTL.VariantName(key),
+                key=FTL.Identifier(key),
                 value=value,
                 default=key == default_key
             )

third_party/python/fluent/fluent/migrate/util.py

@@ -5,7 +5,7 @@ from __future__ import absolute_import
 import textwrap
 
 import fluent.syntax.ast as FTL
-from fluent.syntax.parser import FluentParser, FTLParserStream
+from fluent.syntax.parser import FluentParser, FluentParserStream
 
 
 fluent_parser = FluentParser(with_spans=False)
@@ -34,7 +34,7 @@ def ftl_resource_to_json(code):
 
 
 def ftl_pattern_to_json(code):
-    ps = FTLParserStream(ftl(code))
+    ps = FluentParserStream(ftl(code))
     return fluent_parser.get_pattern(ps).to_json()
 
 

third_party/python/fluent/fluent/syntax/ast.py

@@ -303,11 +303,6 @@ class Identifier(SyntaxNode):
         self.name = name
 
 
-class VariantName(Identifier):
-    def __init__(self, name, **kwargs):
-        super(VariantName, self).__init__(name, **kwargs)
-
-
 class BaseComment(Entry):
     def __init__(self, content=None, **kwargs):
         super(BaseComment, self).__init__(**kwargs)

third_party/python/fluent/fluent/syntax/ftlstream.py

@@ -1,294 +0,0 @@
-from __future__ import unicode_literals
-from .stream import ParserStream
-from .errors import ParseError
-
-
-INLINE_WS = (' ', '\t')
-SPECIAL_LINE_START_CHARS = ('}', '.', '[', '*')
-
-
-class FTLParserStream(ParserStream):
-    last_comment_zero_four_syntax = False
-
-    def skip_inline_ws(self):
-        while self.ch:
-            if self.ch not in INLINE_WS:
-                break
-            self.next()
-
-    def peek_inline_ws(self):
-        ch = self.current_peek()
-        while ch:
-            if ch not in INLINE_WS:
-                break
-            ch = self.peek()
-
-    def skip_blank_lines(self):
-        line_count = 0
-        while True:
-            self.peek_inline_ws()
-
-            if self.current_peek_is('\n'):
-                self.skip_to_peek()
-                self.next()
-                line_count += 1
-            else:
-                self.reset_peek()
-                return line_count
-
-    def peek_blank_lines(self):
-        while True:
-            line_start = self.get_peek_index()
-
-            self.peek_inline_ws()
-
-            if self.current_peek_is('\n'):
-                self.peek()
-            else:
-                self.reset_peek(line_start)
-                break
-
-    def skip_indent(self):
-        self.skip_blank_lines()
-        self.skip_inline_ws()
-
-    def expect_char(self, ch):
-        if self.ch == ch:
-            self.next()
-            return True
-
-        if ch == '\n':
-            # Unicode Character 'SYMBOL FOR NEWLINE' (U+2424)
-            raise ParseError('E0003', '\u2424')
-
-        raise ParseError('E0003', ch)
-
-    def expect_indent(self):
-        self.expect_char('\n')
-        self.skip_blank_lines()
-        self.expect_char(' ')
-        self.skip_inline_ws()
-
-    def expect_line_end(self):
-        if self.ch is None:
-            # EOF is a valid line end in Fluent.
-            return True
-        return self.expect_char('\n')
-
-    def take_char(self, f):
-        ch = self.ch
-        if ch is not None and f(ch):
-            self.next()
-            return ch
-        return None
-
-    def is_char_id_start(self, ch=None):
-        if ch is None:
-            return False
-
-        cc = ord(ch)
-        return (cc >= 97 and cc <= 122) or \
-               (cc >= 65 and cc <= 90)
-
-    def is_identifier_start(self):
-        ch = self.current_peek()
-        is_id = self.is_char_id_start(ch)
-        self.reset_peek()
-        return is_id
-
-    def is_number_start(self):
-        if self.current_is('-'):
-            self.peek()
-
-        cc = ord(self.current_peek())
-        is_digit = cc >= 48 and cc <= 57
-        self.reset_peek()
-        return is_digit
-
-    def is_char_pattern_continuation(self, ch):
-        if ch is None:
-            return False
-
-        return ch not in SPECIAL_LINE_START_CHARS
-
-    def is_peek_value_start(self):
-        self.peek_inline_ws()
-        ch = self.current_peek()
-
-        # Inline Patterns may start with any char.
-        if ch is not None and ch != '\n':
-            return True
-
-        return self.is_peek_next_line_value()
-
-    def is_peek_next_line_zero_four_style_comment(self):
-        if not self.current_peek_is('\n'):
-            return False
-
-        self.peek()
-
-        if self.current_peek_is('/'):
-            self.peek()
-            if self.current_peek_is('/'):
-                self.reset_peek()
-                return True
-
-        self.reset_peek()
-        return False
-
-    # -1 - any
-    #  0 - comment
-    #  1 - group comment
-    #  2 - resource comment
-    def is_peek_next_line_comment(self, level=-1):
-        if not self.current_peek_is('\n'):
-            return False
-
-        i = 0
-
-        while (i <= level or (level == -1 and i < 3)):
-            self.peek()
-            if not self.current_peek_is('#'):
-                if i <= level and level != -1:
-                    self.reset_peek()
-                    return False
-                break
-            i += 1
-
-        self.peek()
-
-        if self.current_peek() in [' ', '\n']:
-            self.reset_peek()
-            return True
-
-        self.reset_peek()
-        return False
-
-    def is_peek_next_line_variant_start(self):
-        if not self.current_peek_is('\n'):
-            return False
-
-        self.peek()
-
-        self.peek_blank_lines()
-
-        ptr = self.get_peek_index()
-
-        self.peek_inline_ws()
-
-        if (self.get_peek_index() - ptr == 0):
-            self.reset_peek()
-            return False
-
-        if self.current_peek_is('*'):
-            self.peek()
-
-        if self.current_peek_is('[') and not self.peek_char_is('['):
-            self.reset_peek()
-            return True
-
-        self.reset_peek()
-        return False
-
-    def is_peek_next_line_attribute_start(self):
-        if not self.current_peek_is('\n'):
-            return False
-
-        self.peek()
-
-        self.peek_blank_lines()
-
-        ptr = self.get_peek_index()
-
-        self.peek_inline_ws()
-
-        if (self.get_peek_index() - ptr == 0):
-            self.reset_peek()
-            return False
-
-        if self.current_peek_is('.'):
-            self.reset_peek()
-            return True
-
-        self.reset_peek()
-        return False
-
-    def is_peek_next_line_value(self):
-        if not self.current_peek_is('\n'):
-            return False
-
-        self.peek()
-
-        self.peek_blank_lines()
-
-        ptr = self.get_peek_index()
-
-        self.peek_inline_ws()
-
-        if (self.get_peek_index() - ptr == 0):
-            self.reset_peek()
-            return False
-
-        if not self.is_char_pattern_continuation(self.current_peek()):
-            self.reset_peek()
-            return False
-
-        self.reset_peek()
-        return True
-
-    def skip_to_next_entry_start(self):
-        while self.ch:
-            if self.current_is('\n') and not self.peek_char_is('\n'):
-                self.next()
-
-                if self.ch is None or \
-                   self.is_identifier_start() or \
-                   self.current_is('-') or \
-                   self.current_is('#') or \
-                   (self.current_is('/') and self.peek_char_is('/')) or \
-                   (self.current_is('[') and self.peek_char_is('[')):
-                    break
-            self.next()
-
-    def take_id_start(self):
-        if self.is_char_id_start(self.ch):
-            ret = self.ch
-            self.next()
-            return ret
-
-        raise ParseError('E0004', 'a-zA-Z')
-
-    def take_id_char(self):
-        def closure(ch):
-            cc = ord(ch)
-            return ((cc >= 97 and cc <= 122) or
-                    (cc >= 65 and cc <= 90) or
-                    (cc >= 48 and cc <= 57) or
-                    cc == 95 or cc == 45)
-        return self.take_char(closure)
-
-    def take_variant_name_char(self):
-        def closure(ch):
-            if ch is None:
-                return False
-            cc = ord(ch)
-            return (cc >= 97 and cc <= 122) or \
-                   (cc >= 65 and cc <= 90) or \
-                   (cc >= 48 and cc <= 57) or \
-                cc == 95 or cc == 45 or cc == 32
-        return self.take_char(closure)
-
-    def take_digit(self):
-        def closure(ch):
-            cc = ord(ch)
-            return (cc >= 48 and cc <= 57)
-        return self.take_char(closure)
-
-    def take_hex_digit(self):
-        def closure(ch):
-            cc = ord(ch)
-            return (
-                (cc >= 48 and cc <= 57)   # 0-9
-                or (cc >= 65 and cc <= 70)  # A-F
-                or (cc >= 97 and cc <= 102))  # a-f
-        return self.take_char(closure)

third_party/python/fluent/fluent/syntax/parser.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 import re
-from .ftlstream import FTLParserStream
 from . import ast
+from .stream import EOF, EOL, FluentParserStream
 from .errors import ParseError
 
 
@@ -10,7 +10,7 @@ def with_span(fn):
         if not self.with_spans:
             return fn(self, ps, *args)
 
-        start = ps.get_index()
+        start = ps.index
         node = fn(self, ps, *args)
 
         # Don't re-add the span if the node already has it.  This may happen
@@ -18,7 +18,7 @@ def with_span(fn):
         if node.span is not None:
             return node
 
-        end = ps.get_index()
+        end = ps.index
         node.add_span(start, end)
         return node
 
@@ -30,15 +30,15 @@ class FluentParser(object):
         self.with_spans = with_spans
 
     def parse(self, source):
-        ps = FTLParserStream(source)
-        ps.skip_blank_lines()
+        ps = FluentParserStream(source)
+        ps.skip_blank_block()
 
         entries = []
         last_comment = None
 
-        while ps.current():
+        while ps.current_char:
             entry = self.get_entry_or_junk(ps)
-            blank_lines = ps.skip_blank_lines()
+            blank_lines = ps.skip_blank_block()
 
             # Regular Comments require special logic. Comments may be attached
             # to Messages or Terms if they are followed immediately by them.
@@ -47,7 +47,7 @@ class FluentParser(object):
             # Message or the Term parsed successfully.
             if (
                 isinstance(entry, ast.Comment)
-                and blank_lines == 0 and ps.current()
+                and blank_lines == 0 and ps.current_char
             ):
                 # Stash the comment and decide what to do with it
                 # in the next pass.
@@ -79,7 +79,7 @@ class FluentParser(object):
         res = ast.Resource(entries)
 
         if self.with_spans:
-            res.add_span(0, ps.get_index())
+            res.add_span(0, ps.index)
 
         return res
 
@@ -92,32 +92,35 @@ class FluentParser(object):
         Preceding comments are ignored unless they contain syntax errors
         themselves, in which case Junk for the invalid comment is returned.
         """
-        ps = FTLParserStream(source)
-        ps.skip_blank_lines()
+        ps = FluentParserStream(source)
+        ps.skip_blank_block()
 
-        while ps.current_is('#'):
+        while ps.current_char == '#':
             skipped = self.get_entry_or_junk(ps)
             if isinstance(skipped, ast.Junk):
                 # Don't skip Junk comments.
                 return skipped
-            ps.skip_blank_lines()
+            ps.skip_blank_block()
 
         return self.get_entry_or_junk(ps)
 
     def get_entry_or_junk(self, ps):
-        entry_start_pos = ps.get_index()
+        entry_start_pos = ps.index
 
         try:
             entry = self.get_entry(ps)
             ps.expect_line_end()
             return entry
         except ParseError as err:
-            error_index = ps.get_index()
-            ps.skip_to_next_entry_start()
-            next_entry_start = ps.get_index()
+            error_index = ps.index
+            ps.skip_to_next_entry_start(entry_start_pos)
+            next_entry_start = ps.index
+            if next_entry_start < error_index:
+                # The position of the error must be inside of the Junk's span.
+                error_index = next_entry_start
 
             # Create a Junk instance
-            slice = ps.get_slice(entry_start_pos, next_entry_start)
+            slice = ps.string[entry_start_pos:next_entry_start]
             junk = ast.Junk(slice)
             if self.with_spans:
                 junk.add_span(entry_start_pos, next_entry_start)
@@ -127,16 +130,16 @@ class FluentParser(object):
             return junk
 
     def get_entry(self, ps):
-        if ps.current_is('#'):
+        if ps.current_char == '#':
             return self.get_comment(ps)
 
-        if ps.current_is('/'):
+        if ps.current_char == '/':
             return self.get_zero_four_style_comment(ps)
 
-        if ps.current_is('['):
+        if ps.current_char == '[':
             return self.get_group_comment_from_section(ps)
 
-        if ps.current_is('-'):
+        if ps.current_char == '-':
             return self.get_term(ps)
 
         if ps.is_identifier_start():
@@ -153,13 +156,13 @@ class FluentParser(object):
         content = ''
 
         while True:
-            ch = ps.take_char(lambda x: x != '\n')
+            ch = ps.take_char(lambda x: x != EOL)
             while ch:
                 content += ch
-                ch = ps.take_char(lambda x: x != '\n')
+                ch = ps.take_char(lambda x: x != EOL)
 
-            if ps.is_peek_next_line_zero_four_style_comment():
-                content += ps.current()
+            if ps.is_next_line_zero_four_comment(skip=False):
+                content += ps.current_char
                 ps.next()
                 ps.expect_char('/')
                 ps.expect_char('/')
@@ -168,8 +171,7 @@ class FluentParser(object):
                 break
 
         # Comments followed by Sections become GroupComments.
-        ps.peek()
-        if ps.current_peek_is('['):
+        if ps.peek() == '[':
             ps.skip_to_peek()
             self.get_group_comment_from_section(ps)
             return ast.GroupComment(content)
@@ -188,22 +190,23 @@ class FluentParser(object):
 
         while True:
             i = -1
-            while ps.current_is('#') and (i < (2 if level == -1 else level)):
+            while ps.current_char == '#' \
+                    and (i < (2 if level == -1 else level)):
                 ps.next()
                 i += 1
 
             if level == -1:
                 level = i
 
-            if not ps.current_is('\n'):
+            if ps.current_char != EOL:
                 ps.expect_char(' ')
-                ch = ps.take_char(lambda x: x != '\n')
+                ch = ps.take_char(lambda x: x != EOL)
                 while ch:
                     content += ch
-                    ch = ps.take_char(lambda x: x != '\n')
+                    ch = ps.take_char(lambda x: x != EOL)
 
-            if ps.is_peek_next_line_comment(level):
-                content += ps.current()
+            if ps.is_next_line_comment(skip=False, level=level):
+                content += ps.current_char
                 ps.next()
             else:
                 break
@@ -217,15 +220,13 @@ class FluentParser(object):
 
     @with_span
     def get_group_comment_from_section(self, ps):
+        def until_closing_bracket_or_eol(ch):
+            return ch not in (']', EOL)
+
         ps.expect_char('[')
         ps.expect_char('[')
-
-        ps.skip_inline_ws()
-
-        self.get_variant_name(ps)
-
-        ps.skip_inline_ws()
-
+        while ps.take_char(until_closing_bracket_or_eol):
+            pass
         ps.expect_char(']')
         ps.expect_char(']')
 
@@ -237,20 +238,17 @@ class FluentParser(object):
     def get_message(self, ps):
         id = self.get_identifier(ps)
 
-        ps.skip_inline_ws()
+        ps.skip_blank_inline()
         pattern = None
 
         # XXX Syntax 0.4 compat
-        if ps.current_is('='):
+        if ps.current_char == '=':
             ps.next()
 
-            if ps.is_peek_value_start():
-                ps.skip_indent()
+            if ps.is_value_start(skip=True):
                 pattern = self.get_pattern(ps)
-            else:
-                ps.skip_inline_ws()
 
-        if ps.is_peek_next_line_attribute_start():
+        if ps.is_next_line_attribute_start(skip=True):
             attrs = self.get_attributes(ps)
         else:
             attrs = None
@@ -264,16 +262,15 @@ class FluentParser(object):
     def get_term(self, ps):
         id = self.get_term_identifier(ps)
 
-        ps.skip_inline_ws()
+        ps.skip_blank_inline()
         ps.expect_char('=')
 
-        if ps.is_peek_value_start():
-            ps.skip_indent()
+        if ps.is_value_start(skip=True):
             value = self.get_value(ps)
         else:
             raise ParseError('E0006', id.name)
 
-        if ps.is_peek_next_line_attribute_start():
+        if ps.is_next_line_attribute_start(skip=True):
             attrs = self.get_attributes(ps)
         else:
             attrs = None
@@ -286,11 +283,10 @@ class FluentParser(object):
 
         key = self.get_identifier(ps)
 
-        ps.skip_inline_ws()
+        ps.skip_blank_inline()
         ps.expect_char('=')
 
-        if ps.is_peek_value_start():
-            ps.skip_indent()
+        if ps.is_value_start(skip=True):
             value = self.get_pattern(ps)
             return ast.Attribute(key, value)
 
@@ -300,11 +296,10 @@ class FluentParser(object):
         attrs = []
 
         while True:
-            ps.expect_indent()
             attr = self.get_attribute(ps)
             attrs.append(attr)
 
-            if not ps.is_peek_next_line_attribute_start():
+            if not ps.is_next_line_attribute_start(skip=True):
                 break
         return attrs
 
@@ -325,35 +320,36 @@ class FluentParser(object):
         return ast.Identifier('-{}'.format(id.name))
 
     def get_variant_key(self, ps):
-        ch = ps.current()
+        ch = ps.current_char
 
-        if ch is None:
+        if ch is EOF:
             raise ParseError('E0013')
 
         cc = ord(ch)
         if ((cc >= 48 and cc <= 57) or cc == 45):  # 0-9, -
             return self.get_number(ps)
 
-        return self.get_variant_name(ps)
+        return self.get_identifier(ps)
 
     @with_span
     def get_variant(self, ps, has_default):
         default_index = False
 
-        if ps.current_is('*'):
+        if ps.current_char == '*':
             if has_default:
                 raise ParseError('E0015')
             ps.next()
             default_index = True
 
         ps.expect_char('[')
+        ps.skip_blank()
 
         key = self.get_variant_key(ps)
 
+        ps.skip_blank()
         ps.expect_char(']')
 
-        if ps.is_peek_value_start():
-            ps.skip_indent()
+        if ps.is_value_start(skip=True):
             value = self.get_value(ps)
             return ast.Variant(key, value, default_index)
 
@@ -364,7 +360,6 @@ class FluentParser(object):
         has_default = False
 
         while True:
-            ps.expect_indent()
             variant = self.get_variant(ps, has_default)
 
             if variant.default:
@@ -372,26 +367,16 @@ class FluentParser(object):
 
             variants.append(variant)
 
-            if not ps.is_peek_next_line_variant_start():
+            if not ps.is_next_line_variant_start(skip=False):
                 break
 
+            ps.skip_blank()
+
         if not has_default:
             raise ParseError('E0010')
 
         return variants
 
-    @with_span
-    def get_variant_name(self, ps):
-        name = ps.take_id_start()
-        while True:
-            ch = ps.take_variant_name_char()
-            if ch:
-                name += ch
-            else:
-                break
-
-        return ast.VariantName(name.rstrip(' \t\n\r'))
-
     def get_digits(self, ps):
         num = ''
 
@@ -409,13 +394,13 @@ class FluentParser(object):
     def get_number(self, ps):
         num = ''
 
-        if ps.current_is('-'):
+        if ps.current_char == '-':
             num += '-'
             ps.next()
 
         num += self.get_digits(ps)
 
-        if ps.current_is('.'):
+        if ps.current_char == '.':
             num += '.'
             ps.next()
             num += self.get_digits(ps)
@@ -424,34 +409,37 @@ class FluentParser(object):
 
     @with_span
     def get_value(self, ps):
-        if ps.current_is('{'):
+        if ps.current_char == '{':
             ps.peek()
-            ps.peek_inline_ws()
-            if ps.is_peek_next_line_variant_start():
+            ps.peek_blank_inline()
+            if ps.is_next_line_variant_start(skip=False):
                 return self.get_variant_list(ps)
+            ps.reset_peek()
 
         return self.get_pattern(ps)
 
     @with_span
     def get_variant_list(self, ps):
         ps.expect_char('{')
-        ps.skip_inline_ws()
+        ps.skip_blank_inline()
+        ps.expect_line_end()
+        ps.skip_blank()
         variants = self.get_variants(ps)
-        ps.expect_indent()
+        ps.expect_line_end()
+        ps.skip_blank()
         ps.expect_char('}')
         return ast.VariantList(variants)
 
     @with_span
     def get_pattern(self, ps):
         elements = []
-        ps.skip_inline_ws()
 
-        while ps.current():
-            ch = ps.current()
+        while ps.current_char:
+            ch = ps.current_char
 
             # The end condition for get_pattern's while loop is a newline
             # which is not followed by a valid pattern continuation.
-            if ch == '\n' and not ps.is_peek_next_line_value():
+            if ch == EOL and not ps.is_next_line_value(skip=False):
                 break
 
             if ch == '{':
@@ -464,6 +452,8 @@ class FluentParser(object):
         last_element = elements[-1]
         if isinstance(last_element, ast.TextElement):
             last_element.value = last_element.value.rstrip(' \t\n\r')
+            if last_element.value == "":
+                elements.pop()
 
         return ast.Pattern(elements)
 
@@ -471,34 +461,34 @@ class FluentParser(object):
     def get_text_element(self, ps):
         buf = ''
 
-        while ps.current():
-            ch = ps.current()
+        while ps.current_char:
+            ch = ps.current_char
 
             if ch == '{':
                 return ast.TextElement(buf)
 
-            if ch == '\n':
-                if not ps.is_peek_next_line_value():
+            if ch == EOL:
+                if not ps.is_next_line_value(skip=False):
                     return ast.TextElement(buf)
 
                 ps.next()
-                ps.skip_inline_ws()
+                ps.skip_blank_inline()
 
-                # Add the new line to the buffer
-                buf += ch
+                buf += EOL
                 continue
 
             if ch == '\\':
                 ps.next()
                 buf += self.get_escape_sequence(ps)
-            else:
-                buf += ch
-                ps.next()
+                continue
+
+            buf += ch
+            ps.next()
 
         return ast.TextElement(buf)
 
     def get_escape_sequence(self, ps, specials=('{', '\\')):
-        next = ps.current()
+        next = ps.current_char
 
         if next in specials:
             ps.next()
@@ -510,8 +500,8 @@ class FluentParser(object):
 
             for _ in range(4):
                 ch = ps.take_hex_digit()
-                if ch is None:
-                    raise ParseError('E0026', sequence + ps.current())
+                if not ch:
+                    raise ParseError('E0026', sequence + ps.current_char)
                 sequence += ch
 
             return '\\u{}'.format(sequence)
@@ -527,16 +517,14 @@ class FluentParser(object):
 
     @with_span
     def get_expression(self, ps):
-        ps.skip_inline_ws()
+        ps.skip_blank()
 
         selector = self.get_selector_expression(ps)
 
-        ps.skip_inline_ws()
+        ps.skip_blank()
 
-        if ps.current_is('-'):
-            ps.peek()
-
-            if not ps.current_peek_is('>'):
+        if ps.current_char == '-':
+            if ps.peek() != '>':
                 ps.reset_peek()
                 return selector
 
@@ -553,9 +541,12 @@ class FluentParser(object):
             ps.next()
             ps.next()
 
-            ps.skip_inline_ws()
+            ps.skip_blank_inline()
+            ps.expect_line_end()
+            ps.skip_blank()
 
             variants = self.get_variants(ps)
+            ps.skip_blank()
 
             if len(variants) == 0:
                 raise ParseError('E0011')
@@ -564,8 +555,6 @@ class FluentParser(object):
             if any(isinstance(v.value, ast.VariantList) for v in variants):
                 raise ParseError('E0023')
 
-            ps.expect_indent()
-
             return ast.SelectExpression(selector, variants)
         elif (
             isinstance(selector, ast.AttributeExpression)
@@ -573,11 +562,13 @@ class FluentParser(object):
         ):
             raise ParseError('E0019')
 
+        ps.skip_blank()
+
         return selector
 
     @with_span
     def get_selector_expression(self, ps):
-        if ps.current_is('{'):
+        if ps.current_char == '{':
             return self.get_placeable(ps)
 
         literal = self.get_literal(ps)
@@ -585,7 +576,7 @@ class FluentParser(object):
         if not isinstance(literal, (ast.MessageReference, ast.TermReference)):
             return literal
 
-        ch = ps.current()
+        ch = ps.current_char
 
         if (ch == '.'):
             ps.next()
@@ -623,16 +614,16 @@ class FluentParser(object):
     def get_call_arg(self, ps):
         exp = self.get_selector_expression(ps)
 
-        ps.skip_inline_ws()
+        ps.skip_blank()
 
-        if not ps.current_is(':'):
+        if ps.current_char != ':':
             return exp
 
         if not isinstance(exp, ast.MessageReference):
             raise ParseError('E0009')
 
         ps.next()
-        ps.skip_inline_ws()
+        ps.skip_blank()
 
         val = self.get_arg_val(ps)
 
@@ -643,11 +634,10 @@ class FluentParser(object):
         named = []
         argument_names = set()
 
-        ps.skip_inline_ws()
-        ps.skip_indent()
+        ps.skip_blank()
 
         while True:
-            if ps.current_is(')'):
+            if ps.current_char == ')':
                 break
 
             arg = self.get_call_arg(ps)
@@ -661,13 +651,11 @@ class FluentParser(object):
             else:
                 positional.append(arg)
 
-            ps.skip_inline_ws()
-            ps.skip_indent()
+            ps.skip_blank()
 
-            if ps.current_is(','):
+            if ps.current_char == ',':
                 ps.next()
-                ps.skip_inline_ws()
-                ps.skip_indent()
+                ps.skip_blank()
                 continue
             else:
                 break
@@ -677,7 +665,7 @@ class FluentParser(object):
     def get_arg_val(self, ps):
         if ps.is_number_start():
             return self.get_number(ps)
-        elif ps.current_is('"'):
+        elif ps.current_char == '"':
             return self.get_string(ps)
         raise ParseError('E0012')
 
@@ -687,26 +675,26 @@ class FluentParser(object):
 
         ps.expect_char('"')
 
-        ch = ps.take_char(lambda x: x != '"' and x != '\n')
+        ch = ps.take_char(lambda x: x != '"' and x != EOL)
         while ch:
             if ch == '\\':
                 val += self.get_escape_sequence(ps, ('{', '\\', '"'))
             else:
                 val += ch
-            ch = ps.take_char(lambda x: x != '"' and x != '\n')
+            ch = ps.take_char(lambda x: x != '"' and x != EOL)
 
-        if ps.current_is('\n'):
+        if ps.current_char == EOL:
             raise ParseError('E0020')
 
-        ps.next()
+        ps.expect_char('"')
 
         return ast.StringLiteral(val)
 
     @with_span
     def get_literal(self, ps):
-        ch = ps.current()
+        ch = ps.current_char
 
-        if ch is None:
+        if ch is EOF:
             raise ParseError('E0014')
 
         if ch == '$':

third_party/python/fluent/fluent/syntax/serializer.py

@@ -276,8 +276,8 @@ def serialize_variant_name(symbol):
 
 
 def serialize_variant_key(key):
-    if isinstance(key, ast.VariantName):
-        return serialize_variant_name(key)
+    if isinstance(key, ast.Identifier):
+        return serialize_identifier(key)
     if isinstance(key, ast.NumberLiteral):
         return serialize_number_literal(key)
     raise Exception('Unknown variant key type: {}'.format(type(key)))

third_party/python/fluent/fluent/syntax/stream.py

@@ -1,125 +1,332 @@
 from __future__ import unicode_literals
+from .errors import ParseError
 
 
-class StringIter():
-    def __init__(self, source):
-        self.source = source
-        self.len = len(source)
-        self.i = 0
-
-    def next(self):
-        if self.i < self.len:
-            ret = self.source[self.i]
-            self.i += 1
-            return ret
-        return None
-
-    def get_slice(self, start, end):
-        return self.source[start:end]
-
-
-class ParserStream():
+class ParserStream(object):
     def __init__(self, string):
-        self.iter = StringIter(string)
-        self.buf = []
-        self.peek_index = 0
+        self.string = string
         self.index = 0
+        self.peek_offset = 0
 
-        self.ch = None
+    def get(self, offset):
+        try:
+            return self.string[offset]
+        except IndexError:
+            return None
 
-        self.iter_end = False
-        self.peek_end = False
+    def char_at(self, offset):
+        # When the cursor is at CRLF, return LF but don't move the cursor. The
+        # cursor still points to the EOL position, which in this case is the
+        # beginning of the compound CRLF sequence. This ensures slices of
+        # [inclusive, exclusive) continue to work properly.
+        if self.get(offset) == '\r' \
+                and self.get(offset + 1) == '\n':
+            return '\n'
 
-        self.ch = self.iter.next()
+        return self.get(offset)
+
+    @property
+    def current_char(self):
+        return self.char_at(self.index)
+
+    @property
+    def current_peek(self):
+        return self.char_at(self.index + self.peek_offset)
 
     def next(self):
-        if self.iter_end:
-            return None
-
-        if len(self.buf) == 0:
-            self.ch = self.iter.next()
-        else:
-            self.ch = self.buf.pop(0)
-
+        self.peek_offset = 0
+        # Skip over CRLF as if it was a single character.
+        if self.get(self.index) == '\r' \
+                and self.get(self.index + 1) == '\n':
+            self.index += 1
         self.index += 1
-
-        if self.ch is None:
-            self.iter_end = True
-            self.peek_end = True
-
-        self.peek_index = self.index
-
-        return self.ch
-
-    def current(self):
-        return self.ch
-
-    def current_is(self, ch):
-        return self.ch == ch
-
-    def current_peek(self):
-        if self.peek_end:
-            return None
-
-        diff = self.peek_index - self.index
-
-        if diff == 0:
-            return self.ch
-        return self.buf[diff - 1]
-
-    def current_peek_is(self, ch):
-        return self.current_peek() == ch
+        return self.get(self.index)
 
     def peek(self):
-        if self.peek_end:
-            return None
+        # Skip over CRLF as if it was a single character.
+        if self.get(self.index + self.peek_offset) == '\r' \
+                and self.get(self.index + self.peek_offset + 1) == '\n':
+            self.peek_offset += 1
+        self.peek_offset += 1
+        return self.get(self.index + self.peek_offset)
 
-        self.peek_index += 1
-
-        diff = self.peek_index - self.index
-
-        if diff > len(self.buf):
-            ch = self.iter.next()
-            if ch is not None:
-                self.buf.append(ch)
-            else:
-                self.peek_end = True
-                return None
-
-        return self.buf[diff - 1]
-
-    def get_index(self):
-        return self.index
-
-    def get_peek_index(self):
-        return self.peek_index
-
-    def peek_char_is(self, ch):
-        if self.peek_end:
-            return False
-
-        ret = self.peek()
-
-        self.peek_index -= 1
-
-        return ret == ch
-
-    def reset_peek(self, pos=False):
-        if pos:
-            if pos < self.peek_index:
-                self.peek_end = False
-            self.peek_index = pos
-        else:
-            self.peek_index = self.index
-            self.peek_end = self.iter_end
+    def reset_peek(self, offset=0):
+        self.peek_offset = offset
 
     def skip_to_peek(self):
-        diff = self.peek_index - self.index
+        self.index += self.peek_offset
+        self.peek_offset = 0
 
-        for i in range(0, diff):
-            self.ch = self.buf.pop(0)
 
-        self.index = self.peek_index
+EOL = '\n'
+EOF = None
+SPECIAL_LINE_START_CHARS = ('}', '.', '[', '*')
 
-    def get_slice(self, start, end):
-        return self.iter.get_slice(start, end)
+
+class FluentParserStream(ParserStream):
+    last_comment_zero_four_syntax = False
+
+    def skip_blank_inline(self):
+        while self.current_char == ' ':
+            self.next()
+
+    def peek_blank_inline(self):
+        while self.current_peek == ' ':
+            self.peek()
+
+    def skip_blank_block(self):
+        line_count = 0
+        while True:
+            self.peek_blank_inline()
+
+            if self.current_peek == EOL:
+                self.skip_to_peek()
+                self.next()
+                line_count += 1
+            else:
+                self.reset_peek()
+                return line_count
+
+    def peek_blank_block(self):
+        while True:
+            line_start = self.peek_offset
+
+            self.peek_blank_inline()
+
+            if self.current_peek == EOL:
+                self.peek()
+            else:
+                self.reset_peek(line_start)
+                break
+
+    def skip_blank(self):
+        while self.current_char in (" ", EOL):
+            self.next()
+
+    def peek_blank(self):
+        while self.current_peek in (" ", EOL):
+            self.peek()
+
+    def expect_char(self, ch):
+        if self.current_char == ch:
+            self.next()
+            return True
+
+        raise ParseError('E0003', ch)
+
+    def expect_line_end(self):
+        if self.current_char is EOF:
+            # EOF is a valid line end in Fluent.
+            return True
+
+        if self.current_char == EOL:
+            self.next()
+            return True
+
+        # Unicode Character 'SYMBOL FOR NEWLINE' (U+2424)
+        raise ParseError('E0003', '\u2424')
+
+    def take_char(self, f):
+        ch = self.current_char
+        if ch is EOF:
+            return EOF
+        if f(ch):
+            self.next()
+            return ch
+        return False
+
+    def is_char_id_start(self, ch):
+        if ch is EOF:
+            return False
+
+        cc = ord(ch)
+        return (cc >= 97 and cc <= 122) or \
+               (cc >= 65 and cc <= 90)
+
+    def is_identifier_start(self):
+        return self.is_char_id_start(self.current_peek)
+
+    def is_number_start(self):
+        ch = self.peek() if self.current_char == '-' else self.current_char
+        if ch is EOF:
+            self.reset_peek()
+            return False
+
+        cc = ord(ch)
+        is_digit = cc >= 48 and cc <= 57
+        self.reset_peek()
+        return is_digit
+
+    def is_char_pattern_continuation(self, ch):
+        if ch is EOF:
+            return False
+
+        return ch not in SPECIAL_LINE_START_CHARS
+
+    def is_value_start(self, skip):
+        if skip is False:
+            raise NotImplementedError()
+
+        self.peek_blank_inline()
+        ch = self.current_peek
+
+        # Inline Patterns may start with any char.
+        if ch is not EOF and ch != EOL:
+            self.skip_to_peek()
+            return True
+
+        return self.is_next_line_value(skip)
+
+    def is_next_line_zero_four_comment(self, skip):
+        if skip is True:
+            raise NotImplementedError()
+
+        if self.current_peek != EOL:
+            return False
+
+        is_comment = (self.peek(), self.peek()) == ('/', '/')
+        self.reset_peek()
+        return is_comment
+
+    # -1 - any
+    #  0 - comment
+    #  1 - group comment
+    #  2 - resource comment
+    def is_next_line_comment(self, skip, level=-1):
+        if skip is True:
+            raise NotImplementedError()
+
+        if self.current_peek != EOL:
+            return False
+
+        i = 0
+
+        while (i <= level or (level == -1 and i < 3)):
+            if self.peek() != '#':
+                if i <= level and level != -1:
+                    self.reset_peek()
+                    return False
+                break
+            i += 1
+
+        # The first char after #, ## or ###.
+        if self.peek() in (' ', EOL):
+            self.reset_peek()
+            return True
+
+        self.reset_peek()
+        return False
+
+    def is_next_line_variant_start(self, skip):
+        if skip is True:
+            raise NotImplementedError()
+
+        if self.current_peek != EOL:
+            return False
+
+        self.peek_blank()
+
+        if self.current_peek == '*':
+            self.peek()
+
+        if self.current_peek == '[' and self.peek() != '[':
+            self.reset_peek()
+            return True
+
+        self.reset_peek()
+        return False
+
+    def is_next_line_attribute_start(self, skip):
+        if skip is False:
+            raise NotImplementedError()
+
+        self.peek_blank()
+
+        if self.current_peek == '.':
+            self.skip_to_peek()
+            return True
+
+        self.reset_peek()
+        return False
+
+    def is_next_line_value(self, skip):
+        if self.current_peek != EOL:
+            return False
+
+        self.peek_blank_block()
+
+        ptr = self.peek_offset
+
+        self.peek_blank_inline()
+
+        if self.current_peek != "{":
+            if (self.peek_offset - ptr == 0):
+                self.reset_peek()
+                return False
+
+            if not self.is_char_pattern_continuation(self.current_peek):
+                self.reset_peek()
+                return False
+
+        if skip:
+            self.skip_to_peek()
+        else:
+            self.reset_peek()
+
+        return True
+
+    def skip_to_next_entry_start(self, junk_start):
+        last_newline = self.string.rfind(EOL, 0, self.index)
+        if junk_start < last_newline:
+            # Last seen newline is _after_ the junk start. It's safe to rewind
+            # without the risk of resuming at the same broken entry.
+            self.index = last_newline
+
+        while self.current_char:
+            # We're only interested in beginnings of line.
+            if self.current_char != EOL:
+                self.next()
+                continue
+
+            # Break if the first char in this line looks like an entry start.
+            first = self.next()
+            if self.is_char_id_start(first) or first == '-' or first == '#':
+                break
+
+            # Syntax 0.4 compatibility
+            peek = self.peek()
+            self.reset_peek()
+            if (first, peek) == ('/', '/') or (first, peek) == ('[', '['):
+                break
+
+    def take_id_start(self):
+        if self.is_char_id_start(self.current_char):
+            ret = self.current_char
+            self.next()
+            return ret
+
+        raise ParseError('E0004', 'a-zA-Z')
+
+    def take_id_char(self):
+        def closure(ch):
+            cc = ord(ch)
+            return ((cc >= 97 and cc <= 122) or
+                    (cc >= 65 and cc <= 90) or
+                    (cc >= 48 and cc <= 57) or
+                    cc == 95 or cc == 45)
+        return self.take_char(closure)
+
+    def take_digit(self):
+        def closure(ch):
+            cc = ord(ch)
+            return (cc >= 48 and cc <= 57)
+        return self.take_char(closure)
+
+    def take_hex_digit(self):
+        def closure(ch):
+            cc = ord(ch)
+            return (
+                (cc >= 48 and cc <= 57)   # 0-9
+                or (cc >= 65 and cc <= 70)  # A-F
+                or (cc >= 97 and cc <= 102))  # a-f
+        return self.take_char(closure)