Bug 1637254, vendor l10n python libraries updates, r=stas

This updates compare-locales to 8.0. The major version bump is due
to the changes to the json output.
This also updates fluent.syntax to 0.17.
Vendor in fluent.migrate 0.9. This is the first formal vendoring,
the version we had in-tree wasn't released on pypi before.

Differential Revision: https://phabricator.services.mozilla.com/D75127

third_party/python/compare-locales/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: compare-locales
-Version: 7.2.5
+Version: 8.0.0
 Summary: Lint Mozilla localizations
 Home-page: UNKNOWN
 Author: Axel Hecht

third_party/python/compare-locales/compare_locales/__init__.py

@@ -1 +1 @@
-version = "7.2.5"
+version = "8.0.0"

third_party/python/compare-locales/compare_locales/checks/base.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import re
+import six
 
 
 class EntityPos(int):
@@ -54,3 +55,73 @@ class Checker(object):
         Only do this if self.needs_reference is True.
         '''
         self.reference = reference
+
+
+class CSSCheckMixin(object):
+    def maybe_style(self, ref_value, l10n_value):
+        ref_map, _ = self.parse_css_spec(ref_value)
+        if not ref_map:
+            return
+        l10n_map, errors = self.parse_css_spec(l10n_value)
+        for t in self.check_style(ref_map, l10n_map, errors):
+            yield t
+
+    def check_style(self, ref_map, l10n_map, errors):
+        if not l10n_map:
+            yield ('error', 0, 'reference is a CSS spec', 'css')
+            return
+        if errors:
+            yield ('error', 0, 'reference is a CSS spec', 'css')
+            return
+        msgs = []
+        for prop, unit in l10n_map.items():
+            if prop not in ref_map:
+                msgs.insert(0, '%s only in l10n' % prop)
+                continue
+            else:
+                ref_unit = ref_map.pop(prop)
+                if unit != ref_unit:
+                    msgs.append("units for %s don't match "
+                                "(%s != %s)" % (prop, unit, ref_unit))
+        for prop in six.iterkeys(ref_map):
+            msgs.insert(0, '%s only in reference' % prop)
+        if msgs:
+            yield ('warning', 0, ', '.join(msgs), 'css')
+
+    def parse_css_spec(self, val):
+        if not hasattr(self, '_css_spec'):
+            self._css_spec = re.compile(
+                r'(?:'
+                r'(?P<prop>(?:min\-|max\-)?(?:width|height))'
+                r'[ \t\r\n]*:[ \t\r\n]*'
+                r'(?P<length>[0-9]+|[0-9]*\.[0-9]+)'
+                r'(?P<unit>ch|em|ex|rem|px|cm|mm|in|pc|pt)'
+                r')'
+                r'|\Z'
+            )
+            self._css_sep = re.compile(r'[ \t\r\n]*(?P<semi>;)?[ \t\r\n]*$')
+        refMap = errors = None
+        end = 0
+        for m in self._css_spec.finditer(val):
+            if end == 0 and m.start() == m.end():
+                # no CSS spec found, just immediately end of string
+                return None, None
+            if m.start() > end:
+                split = self._css_sep.match(val, end, m.start())
+                if split is None:
+                    errors = errors or []
+                    errors.append({
+                        'pos': end,
+                        'code': 'css-bad-content',
+                    })
+                elif end > 0 and split.group('semi') is None:
+                    errors = errors or []
+                    errors.append({
+                        'pos': end,
+                        'code': 'css-missing-semicolon',
+                    })
+            if m.group('prop'):
+                refMap = refMap or {}
+                refMap[m.group('prop')] = m.group('unit')
+            end = m.end()
+        return refMap, errors

third_party/python/compare-locales/compare_locales/checks/dtd.py

@@ -9,10 +9,10 @@ from xml import sax
 import six
 
 from compare_locales.parser import DTDParser
-from .base import Checker
+from .base import Checker, CSSCheckMixin
 
 
-class DTDChecker(Checker):
+class DTDChecker(Checker, CSSCheckMixin):
     """Tests to run on DTD files.
 
     Uses xml.sax for the heavy lifting of xml parsing.
@@ -68,11 +68,6 @@ class DTDChecker(Checker):
     num = re.compile('^%s$' % numPattern)
     lengthPattern = '%s(em|px|ch|cm|in)' % numPattern
     length = re.compile('^%s$' % lengthPattern)
-    spec = re.compile(r'((?:min\-)?(?:width|height))[ \t\r\n]*:[ \t\r\n]*%s' %
-                      lengthPattern)
-    style = re.compile(
-        r'^%(spec)s[ \t\r\n]*(;[ \t\r\n]*%(spec)s[ \t\r\n]*)*;?$' %
-        {'spec': spec.pattern})
 
     def check(self, refEnt, l10nEnt):
         """Try to parse the refvalue inside a dummy element, and keep
@@ -177,28 +172,9 @@ class DTDChecker(Checker):
         # just a length, width="100em"
         if self.length.match(refValue) and not self.length.match(l10nValue):
             yield ('error', 0, 'reference is a CSS length', 'css')
-        # real CSS spec, style="width:100px;"
-        if self.style.match(refValue):
-            if not self.style.match(l10nValue):
-                yield ('error', 0, 'reference is a CSS spec', 'css')
-            else:
-                # warn if different properties or units
-                refMap = dict((s, u) for s, _, u in
-                              self.spec.findall(refValue))
-                msgs = []
-                for s, _, u in self.spec.findall(l10nValue):
-                    if s not in refMap:
-                        msgs.insert(0, '%s only in l10n' % s)
-                        continue
-                    else:
-                        ru = refMap.pop(s)
-                        if u != ru:
-                            msgs.append("units for %s don't match "
-                                        "(%s != %s)" % (s, u, ru))
-                for s in six.iterkeys(refMap):
-                    msgs.insert(0, '%s only in reference' % s)
-                if msgs:
-                    yield ('warning', 0, ', '.join(msgs), 'css')
+        # Check for actual CSS style attribute values
+        for t in self.maybe_style(refValue, l10nValue):
+            yield t
 
         if self.extra_tests is not None and 'android-dtd' in self.extra_tests:
             for t in self.processAndroidContent(self.texthandler.textcontent):

third_party/python/compare-locales/compare_locales/checks/fluent.py

@@ -10,7 +10,7 @@ from collections import defaultdict
 from fluent.syntax import ast as ftl
 from fluent.syntax.serializer import serialize_variant_key
 
-from .base import Checker
+from .base import Checker, CSSCheckMixin
 from compare_locales import plurals
 
 
@@ -26,10 +26,26 @@ MSGS = {
     'obsolete-attribute': 'Obsolete attribute: {name}',
     'duplicate-variant': 'Variant key "{name}" is duplicated',
     'missing-plural': 'Plural categories missing: {categories}',
+    'plain-message': '{message}',
 }
 
 
-class ReferenceMessageVisitor(ftl.Visitor):
+def pattern_variants(pattern):
+    """Get variants of plain text of a pattern.
+
+    For now, just return simple text patterns.
+    This can be improved to allow for SelectExpressions
+    of simple text patterns, or even nested expressions, and Literals.
+    Variants with Variable-, Message-, or TermReferences should be ignored.
+    """
+    elements = pattern.elements
+    if len(elements) == 1:
+        if isinstance(elements[0], ftl.TextElement):
+            return [elements[0].value]
+    return []
+
+
+class ReferenceMessageVisitor(ftl.Visitor, CSSCheckMixin):
     def __init__(self):
         # References to Messages, their Attributes, and Terms
         # Store reference name and type
@@ -42,6 +58,9 @@ class ReferenceMessageVisitor(ftl.Visitor):
         self.message_has_value = False
         # Map attribute names to positions
         self.attribute_positions = {}
+        # Map of CSS style attribute properties and units
+        self.css_styles = None
+        self.css_errors = None
 
     def generic_visit(self, node):
         if isinstance(
@@ -62,6 +81,14 @@ class ReferenceMessageVisitor(ftl.Visitor):
         self.refs = self.entry_refs[node.id.name]
         super(ReferenceMessageVisitor, self).generic_visit(node)
         self.refs = old_refs
+        if node.id.name != 'style':
+            return
+        text_values = pattern_variants(node.value)
+        if not text_values:
+            self.css_styles = 'skip'
+            return
+        # right now, there's just one possible text value
+        self.css_styles, self.css_errors = self.parse_css_spec(text_values[0])
 
     def visit_SelectExpression(self, node):
         # optimize select expressions to only go through the variants
@@ -142,8 +169,9 @@ class GenericL10nChecks(object):
                         )
                     )
         # Check for plural categories
-        if self.locale in plurals.CATEGORIES_BY_LOCALE:
-            known_plurals = set(plurals.CATEGORIES_BY_LOCALE[self.locale])
+        known_plurals = plurals.get_plural(self.locale)
+        if known_plurals:
+            known_plurals = set(known_plurals)
             # Ask for known plurals, but check for plurals w/out `other`.
             # `other` is used for all kinds of things.
             check_plurals = known_plurals.copy()
@@ -214,6 +242,19 @@ class L10nMessageVisitor(GenericL10nChecks, ReferenceMessageVisitor):
         self.reference_refs = self.reference.entry_refs[node.id.name]
         super(L10nMessageVisitor, self).visit_Attribute(node)
         self.reference_refs = old_reference_refs
+        if node.id.name != 'style' or self.css_styles == 'skip':
+            return
+        ref_styles = self.reference.css_styles
+        if ref_styles in ('skip', None):
+            # Reference is complex, l10n isn't.
+            # Let's still validate the css spec.
+            ref_styles = {}
+        for cat, msg, pos, _ in self.check_style(
+            ref_styles,
+            self.css_styles,
+            self.css_errors
+        ):
+            self.messages.append((cat, msg, pos))
 
     def visit_SelectExpression(self, node):
         super(L10nMessageVisitor, self).visit_SelectExpression(node)

third_party/python/compare-locales/compare_locales/checks/properties.py

@@ -71,8 +71,9 @@ class PropertiesChecker(Checker):
         '''Check for the stringbundle plurals logic.
         The common variable pattern is #1.
         '''
-        if self.locale in plurals.CATEGORIES_BY_LOCALE:
-            expected_forms = len(plurals.CATEGORIES_BY_LOCALE[self.locale])
+        known_plurals = plurals.get_plural(self.locale)
+        if known_plurals:
+            expected_forms = len(known_plurals)
             found_forms = l10nValue.count(';') + 1
             msg = 'expecting {} plurals, found {}'.format(
                 expected_forms,

third_party/python/compare-locales/compare_locales/compare/content.py

@@ -244,19 +244,17 @@ class ContentComparer:
                 l10n, merge_file, missings, skips, l10n_ctx,
                 p.capabilities, p.encoding)
 
-        stats = {}
-        for cat, value in (
-                ('missing', missing),
-                ('missing_w', missing_w),
-                ('report', report),
-                ('obsolete', obsolete),
-                ('changed', changed),
-                ('changed_w', changed_w),
-                ('unchanged', unchanged),
-                ('unchanged_w', unchanged_w),
-                ('keys', keys)):
-            if value:
-                stats[cat] = value
+        stats = {
+            'missing': missing,
+            'missing_w': missing_w,
+            'report': report,
+            'obsolete': obsolete,
+            'changed': changed,
+            'changed_w': changed_w,
+            'unchanged': unchanged,
+            'unchanged_w': unchanged_w,
+            'keys': keys,
+        }
         self.observers.updateStats(l10n, stats)
         pass
 
@@ -294,7 +292,7 @@ class ContentComparer:
             return
         # strip parse errors
         entities = [e for e in entities if not isinstance(e, parser.Junk)]
-        self.observers.updateStats(missing, {'missingInFiles': len(entities)})
+        self.observers.updateStats(missing, {'missing': len(entities)})
         missing_w = 0
         for e in entities:
             missing_w += e.count_words()

third_party/python/compare-locales/compare_locales/compare/observer.py

@@ -20,7 +20,19 @@ class Observer(object):
         for quiet=2, skip missing and obsolete files. For quiet=3,
         skip warnings and errors.
         '''
-        self.summary = defaultdict(lambda: defaultdict(int))
+        self.summary = defaultdict(lambda: {
+            "errors": 0,
+            "warnings": 0,
+            "missing": 0,
+            "missing_w": 0,
+            "report": 0,
+            "obsolete": 0,
+            "changed": 0,
+            "changed_w": 0,
+            "unchanged": 0,
+            "unchanged_w": 0,
+            "keys": 0,
+        })
         self.details = Tree(list)
         self.quiet = quiet
         self.filter = filter
@@ -164,20 +176,12 @@ class ObserverList(Observer):
         if len(self.observers) > 1:
             # add ourselves if there's more than one project
             for loc, lst in summaries.items():
-                lst.append(self.summary.get(loc, {}))
-        # normalize missing and missingInFiles -> missing
-        for summarylist in summaries.values():
-            for summary in summarylist:
-                if 'missingInFiles' in summary:
-                    summary['missing'] = (
-                        summary.get('missing', 0)
-                        + summary.pop('missingInFiles')
-                    )
+                lst.append(self.summary[loc])
         keys = (
             'errors',
             'warnings',
             'missing', 'missing_w',
-            'obsolete', 'obsolete_w',
+            'obsolete',
             'changed', 'changed_w',
             'unchanged', 'unchanged_w',
             'keys',
@@ -192,7 +196,7 @@ class ObserverList(Observer):
             segment = [''] * len(keys)
             for summary in summaries:
                 for row, key in enumerate(keys):
-                    segment[row] += ' {:6}'.format(summary.get(key, ''))
+                    segment[row] += ' {:6}'.format(summary.get(key) or '')
 
             out += [
                 lead + row
@@ -201,8 +205,7 @@ class ObserverList(Observer):
             ]
 
             total = sum([summaries[-1].get(k, 0)
-                         for k in ['changed', 'unchanged', 'report', 'missing',
-                                   'missingInFiles']
+                         for k in ['changed', 'unchanged', 'report', 'missing']
                          ])
             rate = 0
             if total:

third_party/python/compare-locales/compare_locales/integration_tests/test_plurals.py

@@ -3,14 +3,13 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import ast
 import json
-import os
 import unittest
-import six
 from six.moves.urllib.error import URLError
 from six.moves.urllib.request import urlopen
 
+from compare_locales import plurals
+
 
 TRANSVISION_URL = (
     'https://transvision.mozfr.org/'
@@ -30,24 +29,16 @@ class TestPlural(unittest.TestCase):
 
     def test_valid_forms(self):
         reference_form_map = self._load_transvision()
-        compare_locales_map = self._parse_plurals_py()
-        # Notify locales in compare-locales but not in Transvision
-        # Might be incubator locales
-        extra_locales = set()
-        extra_locales.update(compare_locales_map)
-        extra_locales.difference_update(reference_form_map)
-        for locale in sorted(extra_locales):
-            print("{} not in Transvision, OK".format(locale))
-            compare_locales_map.pop(locale)
         # Strip matches from dicts, to make diff for test small
-        locales = set()
-        locales.update(compare_locales_map)
-        locales.intersection_update(reference_form_map)
+        locales = list(reference_form_map)
+        cl_form_map = {}
         for locale in locales:
-            if compare_locales_map[locale] == reference_form_map[locale]:
-                compare_locales_map.pop(locale)
+            cl_form = str(plurals.get_plural_rule(locale))
+            if cl_form == reference_form_map[locale]:
                 reference_form_map.pop(locale)
-        self.assertDictEqual(reference_form_map, compare_locales_map)
+            else:
+                cl_form_map[locale] = cl_form
+        self.assertDictEqual(reference_form_map, cl_form_map)
 
     def _load_transvision(self):
         '''Use the Transvision API to load all values of pluralRule
@@ -59,26 +50,3 @@ class TestPlural(unittest.TestCase):
         except URLError:
             raise unittest.SkipTest("Couldn't load Transvision API.")
         return json.loads(data)
-
-    def _parse_plurals_py(self):
-        '''Load compare_locales.plurals, parse the AST, and inspect
-        the dictionary assigned to CATEGORIES_BY_LOCALE to find
-        the actual plural number.
-        Convert both number and locale code to unicode for comparing
-        to json.
-        '''
-        path = os.path.join(os.path.dirname(__file__), '..', 'plurals.py')
-        with open(path) as source_file:
-            plurals_ast = ast.parse(source_file.read())
-        assign_cats_statement = [
-            s for s in plurals_ast.body
-            if isinstance(s, ast.Assign)
-            and any(t.id == 'CATEGORIES_BY_LOCALE' for t in s.targets)
-        ][0]
-        return dict(
-            (six.text_type(k.s), six.text_type(v.slice.value.n))
-            for k, v in zip(
-                assign_cats_statement.value.keys,
-                assign_cats_statement.value.values
-            )
-        )

third_party/python/compare-locales/compare_locales/parser/__init__.py

@@ -54,6 +54,14 @@ def getParser(path):
     for item in __constructors:
         if re.search(item[0], path):
             return item[1]
+    try:
+        from pkg_resources import iter_entry_points
+        for entry_point in iter_entry_points('compare_locales.parsers'):
+            p = entry_point.resolve()()
+            if p.use(path):
+                return p
+    except (ImportError, IOError):
+        pass
     raise UserWarning("Cannot find Parser")
 
 

third_party/python/compare-locales/compare_locales/parser/po.py

@@ -24,7 +24,11 @@ class PoEntityMixin(object):
 
     @property
     def val(self):
-        return self.stringlist_val
+        return (
+            self.stringlist_val
+            if self.stringlist_val
+            else self.stringlist_key[0]
+        )
 
     @property
     def key(self):
@@ -33,7 +37,7 @@ class PoEntityMixin(object):
     @property
     def localized(self):
         # gettext denotes a non-localized string by an empty value
-        return bool(self.val)
+        return bool(self.stringlist_val)
 
     def __repr__(self):
         return self.key[0]

third_party/python/compare-locales/compare_locales/paths/configparser.py

@@ -40,6 +40,7 @@ class TOMLParser(object):
         self.processPaths(ctx)
         self.processFilters(ctx)
         self.processIncludes(ctx)
+        self.processExcludes(ctx)
         self.processLocales(ctx)
         return self.asConfig(ctx)
 
@@ -101,15 +102,23 @@ class TOMLParser(object):
             ctx.pc.add_rules(rule)
 
     def processIncludes(self, ctx):
+        for child in self._processChild(ctx, 'includes'):
+            ctx.pc.add_child(child)
+
+    def processExcludes(self, ctx):
+        for child in self._processChild(ctx, 'excludes'):
+            ctx.pc.exclude(child)
+
+    def _processChild(self, ctx, field):
         assert ctx.data is not None
-        if 'includes' not in ctx.data:
+        if field not in ctx.data:
             return
-        for include in ctx.data['includes']:
-            # resolve include['path'] against our root and env
+        for child_config in ctx.data[field]:
+            # resolve child_config['path'] against our root and env
             p = mozpath.normpath(
                 expand(
                     ctx.pc.root,
-                    include['path'],
+                    child_config['path'],
                     ctx.pc.environ
                 )
             )
@@ -125,7 +134,7 @@ class TOMLParser(object):
                     .getLogger('compare-locales.io')
                     .error('%s: %s', e.strerror, e.filename))
                 continue
-            ctx.pc.add_child(child)
+            yield child
 
     def asConfig(self, ctx):
         return ctx.pc

third_party/python/compare-locales/compare_locales/paths/files.py

@@ -10,6 +10,16 @@ from compare_locales import mozpath
 REFERENCE_LOCALE = 'en-x-moz-reference'
 
 
+class ConfigList(list):
+    def maybe_extend(self, other):
+        '''Add configs from other list if this list doesn't have this path yet.
+        '''
+        for config in other:
+            if any(mine.path == config.path for mine in self):
+                continue
+            self.append(config)
+
+
 class ProjectFiles(object):
     '''Iterable object to get all files and tests for a locale and a
     list of ProjectConfigs.
@@ -20,14 +30,26 @@ class ProjectFiles(object):
     def __init__(self, locale, projects, mergebase=None):
         self.locale = locale
         self.matchers = []
+        self.exclude = None
         self.mergebase = mergebase
-        configs = []
+        configs = ConfigList()
+        excludes = ConfigList()
         for project in projects:
             # Only add this project if we're not in validation mode,
             # and the given locale is enabled for the project.
             if locale is not None and locale not in project.all_locales:
                 continue
-            configs.extend(project.configs)
+            configs.maybe_extend(project.configs)
+            excludes.maybe_extend(project.excludes)
+        # If an excluded config is explicitly included, drop if from the
+        # excludes.
+        excludes = [
+            exclude
+            for exclude in excludes
+            if not any(c.path == exclude.path for c in configs)
+        ]
+        if excludes:
+            self.exclude = ProjectFiles(locale, excludes)
         for pc in configs:
             if locale and pc.locales is not None and locale not in pc.locales:
                 continue
@@ -123,6 +145,9 @@ class ProjectFiles(object):
 
     def iter_reference(self):
         '''Iterate over reference files.'''
+        # unset self.exclude, as we don't want that for our reference files
+        exclude = self.exclude
+        self.exclude = None
         known = {}
         for matchers in self.matchers:
             if 'reference' not in matchers:
@@ -137,6 +162,7 @@ class ProjectFiles(object):
                     }
         for path, d in sorted(known.items()):
             yield (path, d.get('reference'), None, d['test'])
+        self.exclude = exclude
 
     def _files(self, matcher):
         '''Base implementation of getting all files in a hierarchy
@@ -146,12 +172,16 @@ class ProjectFiles(object):
         '''
         base = matcher.prefix
         if self._isfile(base):
+            if self.exclude and self.exclude.match(base) is not None:
+                return
             if matcher.match(base) is not None:
                 yield base
             return
         for d, dirs, files in self._walk(base):
             for f in files:
                 p = mozpath.join(d, f)
+                if self.exclude and self.exclude.match(p) is not None:
+                    continue
                 if matcher.match(p) is not None:
                     yield p
 
@@ -168,9 +198,14 @@ class ProjectFiles(object):
 
         This routine doesn't check that the files actually exist.
         '''
+        if (
+            self.locale is not None and
+            self.exclude and self.exclude.match(path) is not None
+        ):
+            return
         for matchers in self.matchers:
             matcher = matchers['l10n']
-            if matcher.match(path) is not None:
+            if self.locale is not None and matcher.match(path) is not None:
                 ref = merge = None
                 if 'reference' in matchers:
                     ref = matcher.sub(matchers['reference'], path)

third_party/python/compare-locales/compare_locales/paths/matcher.py

@@ -33,7 +33,7 @@ class Matcher(object):
     uses that to transform l10n and en-US paths back and forth.
     '''
 
-    def __init__(self, pattern_or_other, env={}, root=None):
+    def __init__(self, pattern_or_other, env={}, root=None, encoding=None):
         '''Create regular expression similar to mozpath.match().
         '''
         parser = PatternParser()
@@ -50,12 +50,14 @@ class Matcher(object):
             self.env.update(real_env)
             if root is not None:
                 self.pattern.root = root
+            self.encoding = other.encoding
             return
         self.env = real_env
         pattern = pattern_or_other
         self.pattern = parser.parse(pattern)
         if root is not None:
             self.pattern.root = root
+        self.encoding = encoding
 
     def with_env(self, environ):
         return Matcher(self, environ)
@@ -64,7 +66,10 @@ class Matcher(object):
     def prefix(self):
         subpattern = Pattern(self.pattern[:self.pattern.prefix_length])
         subpattern.root = self.pattern.root
-        return subpattern.expand(self.env)
+        prefix = subpattern.expand(self.env)
+        if self.encoding is not None:
+            prefix = prefix.encode(self.encoding)
+        return prefix
 
     def match(self, path):
         '''Test the given path against this matcher and its environment.
@@ -77,6 +82,8 @@ class Matcher(object):
         if m is None:
             return None
         d = m.groupdict()
+        if self.encoding is not None:
+            d = {key: value.decode(self.encoding) for key, value in d.items()}
         if 'android_locale' in d and 'locale' not in d:
             # map android_locale to locale code
             locale = d['android_locale']
@@ -94,9 +101,10 @@ class Matcher(object):
     def _cache_regex(self):
         if self._cached_re is not None:
             return
-        self._cached_re = re.compile(
-            self.pattern.regex_pattern(self.env) + '$'
-        )
+        pattern = self.pattern.regex_pattern(self.env) + '$'
+        if self.encoding is not None:
+            pattern = pattern.encode(self.encoding)
+        self._cached_re = re.compile(pattern)
 
     def sub(self, other, path):
         '''
@@ -108,11 +116,14 @@ class Matcher(object):
             return None
         env = {}
         env.update(
-            (key, Literal(value))
+            (key, Literal(value if value is not None else ''))
             for key, value in m.items()
         )
         env.update(other.env)
-        return other.pattern.expand(env)
+        path = other.pattern.expand(env)
+        if self.encoding is not None:
+            path = path.encode(self.encoding)
+        return path
 
     def concat(self, other):
         '''Concat two Matcher objects.
@@ -164,6 +175,8 @@ class Matcher(object):
                     continue
                 if self.env[k] != other.env[k]:
                     return False
+        if self.encoding != other.encoding:
+            return False
         return True
 
 

third_party/python/compare-locales/compare_locales/paths/project.py

@@ -9,6 +9,10 @@ from .matcher import Matcher
 import six
 
 
+class ExcludeError(ValueError):
+    pass
+
+
 class ProjectConfig(object):
     '''Abstraction of l10n project configuration data.
     '''
@@ -30,6 +34,7 @@ class ProjectConfig(object):
         self._all_locales = None
         self.environ = {}
         self.children = []
+        self.excludes = []
         self._cache = None
 
     def same(self, other):
@@ -112,8 +117,20 @@ class ProjectConfig(object):
 
     def add_child(self, child):
         self._all_locales = None  # clear cache
+        if child.excludes:
+            raise ExcludeError(
+                'Included configs cannot declare their own excludes.'
+            )
         self.children.append(child)
 
+    def exclude(self, child):
+        for config in child.configs:
+            if config.excludes:
+                raise ExcludeError(
+                    'Excluded configs cannot declare their own excludes.'
+                )
+        self.excludes.append(child)
+
     def set_locales(self, locales, deep=False):
         self._all_locales = None  # clear cache
         self.locales = locales
@@ -168,6 +185,8 @@ class ProjectConfig(object):
             return self._cache
         self._cache = self.FilterCache(locale)
         for paths in self.paths:
+            if 'locales' in paths and locale not in paths['locales']:
+                continue
             self._cache.l10n_paths.append(paths['l10n'].with_env({
                 "locale": locale
             }))
@@ -180,6 +199,11 @@ class ProjectConfig(object):
         return self._cache
 
     def _filter(self, l10n_file, entity=None):
+        if any(
+            exclude.filter(l10n_file) == 'error'
+            for exclude in self.excludes
+        ):
+            return
         actions = set(
             child._filter(l10n_file, entity=entity)
             for child in self.children)

third_party/python/compare-locales/compare_locales/plurals.py

@@ -58,152 +58,160 @@ CATEGORIES_EXCEPTIONS = {
 }
 
 CATEGORIES_BY_LOCALE = {
-    'ace': CATEGORIES_BY_INDEX[0],
-    'ach': CATEGORIES_BY_INDEX[1],
-    'af': CATEGORIES_BY_INDEX[1],
-    'ak': CATEGORIES_BY_INDEX[2],
-    'an': CATEGORIES_BY_INDEX[1],
-    'ar': CATEGORIES_BY_INDEX[12],
-    'arn': CATEGORIES_BY_INDEX[1],
-    'as': CATEGORIES_BY_INDEX[1],
-    'ast': CATEGORIES_BY_INDEX[1],
-    'az': CATEGORIES_BY_INDEX[1],
-    'be': CATEGORIES_BY_INDEX[7],
-    'bg': CATEGORIES_BY_INDEX[1],
-    'bn': CATEGORIES_BY_INDEX[2],
-    'bn-BD': CATEGORIES_BY_INDEX[2],
-    'bn-IN': CATEGORIES_BY_INDEX[2],
-    'br': CATEGORIES_BY_INDEX[16],
-    'brx': CATEGORIES_BY_INDEX[1],
-    'bs': CATEGORIES_BY_INDEX[19],
-    'ca': CATEGORIES_BY_INDEX[1],
-    'cak': CATEGORIES_BY_INDEX[1],
-    'crh': CATEGORIES_BY_INDEX[1],
-    'cs': CATEGORIES_BY_INDEX[8],
-    'csb': CATEGORIES_BY_INDEX[9],
-    'cv': CATEGORIES_BY_INDEX[1],
-    'cy': CATEGORIES_BY_INDEX[18],
-    'da': CATEGORIES_BY_INDEX[1],
-    'de': CATEGORIES_BY_INDEX[1],
-    'dsb': CATEGORIES_BY_INDEX[10],
-    'el': CATEGORIES_BY_INDEX[1],
-    'en-CA': CATEGORIES_BY_INDEX[1],
-    'en-GB': CATEGORIES_BY_INDEX[1],
-    'en-US': CATEGORIES_BY_INDEX[1],
-    'en-ZA': CATEGORIES_BY_INDEX[1],
-    'en-x-moz-reference': CATEGORIES_BY_INDEX[1],  # for reference validation
-    'eo': CATEGORIES_BY_INDEX[1],
-    'es-AR': CATEGORIES_BY_INDEX[1],
-    'es-CL': CATEGORIES_BY_INDEX[1],
-    'es-ES': CATEGORIES_BY_INDEX[1],
-    'es-MX': CATEGORIES_BY_INDEX[1],
-    'et': CATEGORIES_BY_INDEX[1],
-    'eu': CATEGORIES_BY_INDEX[1],
-    'fa': CATEGORIES_BY_INDEX[2],
-    'ff': CATEGORIES_BY_INDEX[1],
-    'fi': CATEGORIES_BY_INDEX[1],
-    'fr': CATEGORIES_BY_INDEX[2],
-    'frp': CATEGORIES_BY_INDEX[2],
-    'fur': CATEGORIES_BY_INDEX[1],
-    'fy-NL': CATEGORIES_BY_INDEX[1],
-    'ga-IE': CATEGORIES_BY_INDEX[11],
-    'gd': CATEGORIES_BY_INDEX[4],
-    'gl': CATEGORIES_BY_INDEX[1],
-    'gn': CATEGORIES_BY_INDEX[1],
-    'gu-IN': CATEGORIES_BY_INDEX[2],
-    'he': CATEGORIES_BY_INDEX[1],
-    'hi-IN': CATEGORIES_BY_INDEX[2],
-    'hr': CATEGORIES_BY_INDEX[19],
-    'hsb': CATEGORIES_BY_INDEX[10],
-    'hto': CATEGORIES_BY_INDEX[1],
-    'hu': CATEGORIES_BY_INDEX[1],
-    'hy-AM': CATEGORIES_BY_INDEX[1],
-    'ia': CATEGORIES_BY_INDEX[1],
-    'id': CATEGORIES_BY_INDEX[0],
-    'ilo': CATEGORIES_BY_INDEX[0],
-    'is': CATEGORIES_BY_INDEX[15],
-    'it': CATEGORIES_BY_INDEX[1],
-    'ja': CATEGORIES_BY_INDEX[0],
-    'ja-JP-mac': CATEGORIES_BY_INDEX[0],
-    'jiv': CATEGORIES_BY_INDEX[17],
-    'ka': CATEGORIES_BY_INDEX[1],
-    'kab': CATEGORIES_BY_INDEX[1],
-    'kk': CATEGORIES_BY_INDEX[1],
-    'km': CATEGORIES_BY_INDEX[0],
-    'kn': CATEGORIES_BY_INDEX[1],
-    'ko': CATEGORIES_BY_INDEX[0],
-    'ks': CATEGORIES_BY_INDEX[1],
-    'ku': CATEGORIES_BY_INDEX[1],
-    'lb': CATEGORIES_BY_INDEX[1],
-    'lg': CATEGORIES_BY_INDEX[1],
-    'lij': CATEGORIES_BY_INDEX[1],
-    'lo': CATEGORIES_BY_INDEX[0],
-    'lt': CATEGORIES_BY_INDEX[6],
-    'ltg': CATEGORIES_BY_INDEX[3],
-    'lv': CATEGORIES_BY_INDEX[3],
-    'lus': CATEGORIES_BY_INDEX[0],
-    'mai': CATEGORIES_BY_INDEX[1],
-    'meh': CATEGORIES_BY_INDEX[0],
-    'mix': CATEGORIES_BY_INDEX[0],
-    'mk': CATEGORIES_BY_INDEX[15],
-    'ml': CATEGORIES_BY_INDEX[1],
-    'mn': CATEGORIES_BY_INDEX[1],
-    'mr': CATEGORIES_BY_INDEX[1],
-    'ms': CATEGORIES_BY_INDEX[0],
-    'my': CATEGORIES_BY_INDEX[0],
-    'nb-NO': CATEGORIES_BY_INDEX[1],
-    'ne-NP': CATEGORIES_BY_INDEX[1],
-    'nl': CATEGORIES_BY_INDEX[1],
-    'nn-NO': CATEGORIES_BY_INDEX[1],
-    'nr': CATEGORIES_BY_INDEX[1],
-    'nso': CATEGORIES_BY_INDEX[2],
-    'ny': CATEGORIES_BY_INDEX[1],
-    'oc': CATEGORIES_BY_INDEX[2],
-    'or': CATEGORIES_BY_INDEX[1],
-    'pa-IN': CATEGORIES_BY_INDEX[2],
-    'pai': CATEGORIES_BY_INDEX[0],
-    'pl': CATEGORIES_BY_INDEX[9],
-    'pt-BR': CATEGORIES_BY_INDEX[1],
-    'pt-PT': CATEGORIES_BY_INDEX[1],
-    'quy': CATEGORIES_BY_INDEX[1],
-    'qvi': CATEGORIES_BY_INDEX[1],
-    'rm': CATEGORIES_BY_INDEX[1],
-    'ro': CATEGORIES_BY_INDEX[5],
-    'ru': CATEGORIES_BY_INDEX[7],
-    'rw': CATEGORIES_BY_INDEX[1],
-    'sah': CATEGORIES_BY_INDEX[0],
-    'sat': CATEGORIES_BY_INDEX[1],
-    'sc': CATEGORIES_BY_INDEX[1],
-    'scn': CATEGORIES_BY_INDEX[1],
-    'si': CATEGORIES_BY_INDEX[1],
-    'sk': CATEGORIES_BY_INDEX[8],
-    'sl': CATEGORIES_BY_INDEX[10],
-    'son': CATEGORIES_BY_INDEX[1],
-    'sq': CATEGORIES_BY_INDEX[1],
-    'sr': CATEGORIES_BY_INDEX[19],
-    'ss': CATEGORIES_BY_INDEX[1],
-    'st': CATEGORIES_BY_INDEX[1],
-    'sv-SE': CATEGORIES_BY_INDEX[1],
-    'sw': CATEGORIES_BY_INDEX[1],
-    'ta-LK': CATEGORIES_BY_INDEX[1],
-    'ta': CATEGORIES_BY_INDEX[1],
-    'te': CATEGORIES_BY_INDEX[1],
-    'th': CATEGORIES_BY_INDEX[0],
-    'tl': CATEGORIES_BY_INDEX[1],
-    'tn': CATEGORIES_BY_INDEX[1],
-    'tr': CATEGORIES_BY_INDEX[1],
-    'trs': CATEGORIES_BY_INDEX[1],
-    'ts': CATEGORIES_BY_INDEX[1],
-    'tsz': CATEGORIES_BY_INDEX[1],
-    'uk': CATEGORIES_BY_INDEX[7],
-    'ur': CATEGORIES_BY_INDEX[1],
-    'uz': CATEGORIES_BY_INDEX[1],
-    've': CATEGORIES_BY_INDEX[1],
-    'vi': CATEGORIES_BY_INDEX[0],
-    'wo': CATEGORIES_BY_INDEX[0],
-    'xh': CATEGORIES_BY_INDEX[1],
-    'zam': CATEGORIES_BY_INDEX[1],
-    'zh-CN': CATEGORIES_BY_INDEX[0],
-    'zh-TW': CATEGORIES_BY_INDEX[0],
-    'zu': CATEGORIES_BY_INDEX[2],
+    'ace': 0,
+    'ach': 1,
+    'af': 1,
+    'ak': 2,
+    'an': 1,
+    'ar': 12,
+    'arn': 1,
+    'as': 1,
+    'ast': 1,
+    'az': 1,
+    'be': 7,
+    'bg': 1,
+    'bn': 2,
+    'bo': 0,
+    'br': 16,
+    'brx': 1,
+    'bs': 19,
+    'ca': 1,
+    'cak': 1,
+    'ckb': 1,
+    'crh': 1,
+    'cs': 8,
+    'csb': 9,
+    'cv': 1,
+    'cy': 18,
+    'da': 1,
+    'de': 1,
+    'dsb': 10,
+    'el': 1,
+    'en': 1,
+    'eo': 1,
+    'es': 1,
+    'et': 1,
+    'eu': 1,
+    'fa': 2,
+    'ff': 1,
+    'fi': 1,
+    'fr': 2,
+    'frp': 2,
+    'fur': 1,
+    'fy': 1,
+    'ga': 11,
+    'gd': 4,
+    'gl': 1,
+    'gn': 1,
+    'gu': 2,
+    'he': 1,
+    'hi': 2,
+    'hr': 19,
+    'hsb': 10,
+    'hto': 1,
+    'hu': 1,
+    'hy': 1,
+    'hye': 1,
+    'ia': 1,
+    'id': 0,
+    'ilo': 0,
+    'is': 15,
+    'it': 1,
+    'ja': 0,
+    'jiv': 17,
+    'ka': 1,
+    'kab': 1,
+    'kk': 1,
+    'km': 0,
+    'kn': 1,
+    'ko': 0,
+    'ks': 1,
+    'ku': 1,
+    'lb': 1,
+    'lg': 1,
+    'lij': 1,
+    'lo': 0,
+    'lt': 6,
+    'ltg': 3,
+    'lv': 3,
+    'lus': 0,
+    'mai': 1,
+    'meh': 0,
+    'mix': 0,
+    'mk': 15,
+    'ml': 1,
+    'mn': 1,
+    'mr': 1,
+    'ms': 0,
+    'my': 0,
+    'nb': 1,
+    'ne': 1,
+    'nl': 1,
+    'nn': 1,
+    'nr': 1,
+    'nso': 2,
+    'ny': 1,
+    'oc': 2,
+    'or': 1,
+    'pa': 2,
+    'pai': 0,
+    'pl': 9,
+    'pt': 1,
+    'quy': 1,
+    'qvi': 1,
+    'rm': 1,
+    'ro': 5,
+    'ru': 7,
+    'rw': 1,
+    'sah': 0,
+    'sat': 1,
+    'sc': 1,
+    'scn': 1,
+    'si': 1,
+    'sk': 8,
+    'sl': 10,
+    'son': 1,
+    'sq': 1,
+    'sr': 19,
+    'ss': 1,
+    'st': 1,
+    'sv': 1,
+    'sw': 1,
+    'ta': 1,
+    'ta': 1,
+    'te': 1,
+    'th': 0,
+    'tl': 1,
+    'tn': 1,
+    'tr': 1,
+    'trs': 1,
+    'ts': 1,
+    'tsz': 1,
+    'uk': 7,
+    'ur': 1,
+    'uz': 1,
+    've': 1,
+    'vi': 0,
+    'wo': 0,
+    'xh': 1,
+    'zam': 1,
+    'zh-CN': 0,
+    'zh-TW': 0,
+    'zu': 2,
 }
+
+
+def get_plural(locale):
+    plural_form = get_plural_rule(locale)
+    if plural_form is None:
+        return None
+    return CATEGORIES_BY_INDEX[plural_form]
+
+
+def get_plural_rule(locale):
+    if locale is None:
+        return None
+    if locale in CATEGORIES_BY_LOCALE:
+        return CATEGORIES_BY_LOCALE[locale]
+    locale = locale.split('-', 1)[0]
+    return CATEGORIES_BY_LOCALE.get(locale)

third_party/python/compare-locales/compare_locales/tests/fluent/test_checks.py

@@ -403,5 +403,179 @@ msg = { $val ->
         )
 
 
+class CSSStyleTest(BaseHelper):
+    file = File('foo.ftl', 'foo.ftl')
+    refContent = b'''\
+simple =
+    .style = width:1px
+select =
+    .style = {PLATFORM() ->
+        [windows] width:1px
+       *[unix] max-width:1px
+    }
+ref =
+    .style = {simple.style}
+broken =
+    .style = 28em
+'''
+
+    def test_simple(self):
+        self._test(dedent_ftl(
+            '''\
+            simple =
+                .style = width:2px
+            '''),
+            tuple())
+        self._test(dedent_ftl(
+            '''\
+            simple =
+                .style = max-width:2px
+            '''),
+            (
+                (
+                    'warning', 0,
+                    'width only in reference, max-width only in l10n', 'fluent'
+                ),
+            ))
+        self._test(dedent_ftl(
+            '''\
+            simple =
+                .style = stuff
+            '''),
+            (
+                (
+                    'error', 0,
+                    'reference is a CSS spec', 'fluent'
+                ),
+            ))
+        # Cover the current limitations of only plain strings
+        self._test(dedent_ftl(
+            '''\
+            simple =
+                .style = {"width:3px"}
+            '''),
+            tuple())
+
+    def test_select(self):
+        self._test(dedent_ftl(
+            '''\
+            select =
+                .style = width:2px
+            '''),
+            (
+                (
+                    'warning', 0,
+                    'width only in l10n', 'fluent'
+                ),
+            ))
+        self._test(dedent_ftl(
+            '''\
+            select =
+                .style = max-width:2px
+            '''),
+            (
+                (
+                    'warning', 0,
+                    'max-width only in l10n', 'fluent'
+                ),
+            ))
+        self._test(dedent_ftl(
+            '''\
+            select =
+                .style = stuff
+            '''),
+            (
+                (
+                    'error', 0,
+                    'reference is a CSS spec', 'fluent'
+                ),
+            ))
+        # Cover the current limitations of only plain strings
+        self._test(dedent_ftl(
+            '''\
+            select =
+                .style = {"width:1px"}
+            '''),
+            tuple())
+
+    def test_ref(self):
+        self._test(dedent_ftl(
+            '''\
+            ref =
+                .style = width:2px
+            '''),
+            (
+                (
+                    'warning', 0,
+                    'width only in l10n', 'fluent'
+                ),
+                (
+                    'warning', 0,
+                    'Missing message reference: simple.style', 'fluent'
+                ),
+            ))
+        self._test(dedent_ftl(
+            '''\
+            ref =
+                .style = max-width:2px
+            '''),
+            (
+                (
+                    'warning', 0,
+                    'max-width only in l10n', 'fluent'
+                ),
+                (
+                    'warning', 0,
+                    'Missing message reference: simple.style', 'fluent'
+                ),
+            ))
+        self._test(dedent_ftl(
+            '''\
+            ref =
+                .style = stuff
+            '''),
+            (
+                (
+                    'error', 0,
+                    'reference is a CSS spec', 'fluent'
+                ),
+                (
+                    'warning', 0,
+                    'Missing message reference: simple.style', 'fluent'
+                ),
+            ))
+        # Cover the current limitations of only plain strings
+        self._test(dedent_ftl(
+            '''\
+            ref =
+                .style = {"width:1px"}
+            '''),
+            (
+                (
+                    'warning', 0,
+                    'Missing message reference: simple.style', 'fluent'
+                ),
+            ))
+
+    def test_broken(self):
+        self._test(dedent_ftl(
+            '''\
+            broken =
+                .style = 27em
+            '''),
+            (('error', 0, 'reference is a CSS spec', 'fluent'),))
+        self._test(dedent_ftl(
+            '''\
+            broken =
+                .style = width: 27em
+            '''),
+            (
+                (
+                    'warning', 0,
+                    'width only in l10n', 'fluent'
+                ),
+            ))
+
+
 if __name__ == '__main__':
     unittest.main()

third_party/python/compare-locales/compare_locales/tests/paths/__init__.py

@@ -5,6 +5,8 @@
 
 from __future__ import absolute_import
 
+from collections import defaultdict
+import six
 import tempfile
 from compare_locales.paths import (
     ProjectConfig, File, ProjectFiles, TOMLParser
@@ -21,6 +23,9 @@ class Rooted(object):
     def path(self, leaf=''):
         return self.root + leaf
 
+    def leaf(self, path):
+        return mozpath.relpath(path, self.root)
+
 
 class SetupMixin(object):
     def setUp(self):
@@ -36,18 +41,64 @@ class SetupMixin(object):
         self.cfg.set_locales(['de'])
 
 
-class MockNode(object):
-    def __init__(self, name):
-        self.name = name
+class MockOS(object):
+    '''Mock `os.path.isfile` and `os.walk` based on a list of files.
+    '''
+    def __init__(self, root, paths):
+        self.root = root
         self.files = []
         self.dirs = {}
+        if not paths:
+            return
+        if isinstance(paths[0], six.string_types):
+            paths = [
+                mozpath.split(path)
+                for path in sorted(paths)
+            ]
+        child_paths = defaultdict(list)
+        for segs in paths:
+            if len(segs) == 1:
+                self.files.append(segs[0])
+            else:
+                child_paths[segs[0]].append(segs[1:])
+        for root, leafs in child_paths.items():
+            self.dirs[root] = MockOS(mozpath.join(self.root, root), leafs)
 
-    def walk(self):
-        subdirs = sorted(self.dirs)
-        if self.name is not None:
-            yield self.name, subdirs, self.files
+    def find(self, dir_path):
+        relpath = mozpath.relpath(dir_path, self.root)
+        if relpath.startswith('..'):
+            return None
+        if relpath in ('', '.'):
+            return self
+        segs = mozpath.split(relpath)
+        node = self
+        while segs:
+            seg = segs.pop(0)
+            if seg not in node.dirs:
+                return None
+            node = node.dirs[seg]
+        return node
+
+    def isfile(self, path):
+        dirname = mozpath.dirname(path)
+        if dirname:
+            node = self.find(dirname)
+        else:
+            node = self
+        return node and mozpath.basename(path) in node.files
+
+    def walk(self, path=None):
+        if path is None:
+            node = self
+        else:
+            node = self.find(path)
+            if node is None:
+                return
+        subdirs = sorted(node.dirs)
+        if node.root is not None:
+            yield node.root, subdirs, node.files
         for subdir in subdirs:
-            child = self.dirs[subdir]
+            child = node.dirs[subdir]
             for tpl in child.walk():
                 yield tpl
 
@@ -56,25 +107,18 @@ class MockProjectFiles(ProjectFiles):
     def __init__(self, mocks, locale, projects, mergebase=None):
         (super(MockProjectFiles, self)
             .__init__(locale, projects, mergebase=mergebase))
-        self.mocks = mocks
+        root = mozpath.commonprefix(mocks)
+        files = [mozpath.relpath(f, root) for f in mocks]
+        self.mocks = MockOS(root, files)
 
     def _isfile(self, path):
-        return path in self.mocks
+        return self.mocks.isfile(path)
 
     def _walk(self, base):
         base = mozpath.normpath(base)
-        local_files = [
-            mozpath.split(mozpath.relpath(f, base))
-            for f in self.mocks if f.startswith(base)
-        ]
-        root = MockNode(base)
-        for segs in local_files:
-            node = root
-            for n, seg in enumerate(segs[:-1]):
-                if seg not in node.dirs:
-                    node.dirs[seg] = MockNode('/'.join([base] + segs[:n+1]))
-                node = node.dirs[seg]
-            node.files.append(segs[-1])
+        root = self.mocks.find(base)
+        if not root:
+            return
         for tpl in root.walk():
             yield tpl
 

third_party/python/compare-locales/compare_locales/tests/paths/test_configparser.py

@@ -5,15 +5,16 @@
 
 from __future__ import absolute_import, unicode_literals
 import unittest
+import six
 
 from . import MockTOMLParser
 from compare_locales.paths.matcher import Matcher
-from compare_locales.paths.project import ProjectConfig
+from compare_locales.paths.project import ProjectConfig, ExcludeError
 from compare_locales import mozpath
 
 
 class TestConfigParser(unittest.TestCase):
-    def test_imports(self):
+    def test_includes(self):
         parser = MockTOMLParser({
             "root.toml": """
 basepath = "."
@@ -21,9 +22,14 @@ basepath = "."
   o = "toolkit"
 [[includes]]
   path = "{o}/other.toml"
+[[includes]]
+  path = "dom/more.toml"
 """,
             "other.toml": """
 basepath = "."
+""",
+            "more.toml": """
+basepath = "."
 """
         })
         config = parser.parse("root.toml")
@@ -32,9 +38,55 @@ basepath = "."
         self.assertEqual(configs[0], config)
         self.assertListEqual(
             [c.path for c in configs],
-            ["root.toml", mozpath.abspath("toolkit/other.toml")]
+            [
+                "root.toml",
+                mozpath.abspath("toolkit/other.toml"),
+                mozpath.abspath("dom/more.toml"),
+            ]
         )
 
+    def test_excludes(self):
+        parser = MockTOMLParser({
+            "root.toml": """
+basepath = "."
+[[excludes]]
+  path = "exclude.toml"
+[[excludes]]
+  path = "other-exclude.toml"
+  """,
+            "exclude.toml": """
+basepath = "."
+""",
+            "other-exclude.toml": """
+basepath = "."
+""",
+            "grandparent.toml": """
+basepath = "."
+[[includes]]
+  path = "root.toml"
+""",
+            "wrapped.toml": """
+basepath = "."
+[[excludes]]
+  path = "root.toml"
+ """
+        })
+        config = parser.parse("root.toml")
+        self.assertIsInstance(config, ProjectConfig)
+        configs = list(config.configs)
+        self.assertListEqual(configs, [config])
+        self.assertEqual(
+            [c.path for c in config.excludes],
+            [
+                mozpath.abspath("exclude.toml"),
+                mozpath.abspath("other-exclude.toml"),
+            ]
+        )
+        with six.assertRaisesRegex(self, ExcludeError, 'Included configs'):
+            parser.parse("grandparent.toml")
+        with six.assertRaisesRegex(self, ExcludeError, 'Excluded configs'):
+            parser.parse("wrapped.toml")
+
     def test_paths(self):
         parser = MockTOMLParser({
             "l10n.toml": """

third_party/python/compare-locales/compare_locales/tests/paths/test_files.py

@@ -5,17 +5,64 @@
 
 from __future__ import absolute_import
 import unittest
+import mock
 
 from compare_locales.paths import (
-    ProjectConfig
+    File,
+    ProjectConfig,
+    ProjectFiles,
 )
 from . import (
+    MockOS,
     MockProjectFiles,
     MockTOMLParser,
     Rooted,
 )
 
 
+class TestMockOS(Rooted, unittest.TestCase):
+    def setUp(self):
+        self.node = MockOS('jazz', [
+            'one/bit',
+            'two/deep/in/directories/with/file1',
+            'two/deep/in/directories/with/file2',
+            'three/feet',
+        ])
+
+    def test_isfile(self):
+        self.assertTrue(self.node.isfile('jazz/one/bit'))
+        self.assertFalse(self.node.isfile('jazz/one'))
+        self.assertFalse(self.node.isfile('foo'))
+
+    def test_walk(self):
+        self.assertListEqual(
+            list(self.node.walk()),
+            [
+                ('jazz', ['one', 'three', 'two'], []),
+                ('jazz/one', [], ['bit']),
+                ('jazz/three', [], ['feet']),
+                ('jazz/two', ['deep'], []),
+                ('jazz/two/deep', ['in'], []),
+                ('jazz/two/deep/in', ['directories'], []),
+                ('jazz/two/deep/in/directories', ['with'], []),
+                ('jazz/two/deep/in/directories/with', [], [
+                    'file1',
+                    'file2',
+                ]),
+            ]
+        )
+
+    def test_find(self):
+        self.assertIsNone(self.node.find('foo'))
+        self.assertIsNone(self.node.find('jazz/one/bit'))
+        self.assertIsNone(self.node.find('jazz/one/bit/too/much'))
+        self.assertIsNotNone(self.node.find('jazz/one'))
+        self.assertListEqual(list(self.node.find('jazz/one').walk()), [
+                ('jazz/one', [], ['bit']),
+        ])
+        self.assertEqual(self.node.find('jazz'), self.node)
+
+
 class TestProjectPaths(Rooted, unittest.TestCase):
     def test_l10n_path(self):
         cfg = ProjectConfig(None)
@@ -244,6 +291,240 @@ class TestProjectPaths(Rooted, unittest.TestCase):
             ])
 
 
+@mock.patch('os.path.isfile')
+@mock.patch('os.walk')
+class TestExcludes(Rooted, unittest.TestCase):
+    def _list(self, locale, _walk, _isfile):
+        parser = MockTOMLParser({
+            "pontoon.toml":
+            '''\
+basepath = "."
+
+[[includes]]
+    path = "configs-pontoon.toml"
+
+[[excludes]]
+    path = "configs-vendor.toml"
+[[excludes]]
+    path = "configs-special-templates.toml"
+''',
+            "vendor.toml":
+            '''\
+basepath = "."
+
+[[includes]]
+    path = "configs-vendor.toml"
+
+[[excludes]]
+    path = "configs-special-templates.toml"
+''',
+            "configs-pontoon.toml":
+            '''\
+basepath = "."
+
+locales = [
+    "de",
+    "gd",
+    "it",
+]
+
+[[paths]]
+    reference = "en/**/*.ftl"
+    l10n = "{locale}/**/*.ftl"
+''',
+            "configs-vendor.toml":
+            '''\
+basepath = "."
+
+locales = [
+    "de",
+    "it",
+]
+
+[[paths]]
+    reference = "en/firefox/*.ftl"
+    l10n = "{locale}/firefox/*.ftl"
+''',
+            "configs-special-templates.toml":
+            '''\
+basepath = "."
+
+[[paths]]
+    reference = "en/firefox/home.ftl"
+    l10n = "{locale}/firefox/home.ftl"
+    locales = [
+        "de",
+        "fr",
+    ]
+[[paths]]
+    reference = "en/firefox/pagina.ftl"
+    l10n = "{locale}/firefox/pagina.ftl"
+    locales = [
+        "gd",
+    ]
+''',
+            })
+        pontoon = parser.parse(self.path('/pontoon.toml'))
+        vendor = parser.parse(self.path('/vendor.toml'))
+        pc = ProjectFiles(locale, [pontoon, vendor])
+        mock_files = [
+            '{}/{}/{}'.format(locale, dir, f)
+            for locale in ('de', 'en', 'gd', 'it')
+            for dir, files in (
+                ('firefox', ('home.ftl', 'feature.ftl')),
+                ('mozorg', ('mission.ftl',)),
+            )
+            for f in files
+        ]
+        os_ = MockOS(self.root, mock_files)
+        _isfile.side_effect = os_.isfile
+        _walk.side_effect = os_.walk
+        local_files = [self.leaf(p).split('/', 1)[1] for p, _, _, _ in pc]
+        return pontoon, vendor, local_files
+
+    def test_reference(self, _walk, _isfile):
+        pontoon_config, vendor_config, files = self._list(None, _walk, _isfile)
+        pontoon_files = ProjectFiles(None, [pontoon_config])
+        vendor_files = ProjectFiles(None, [vendor_config])
+        self.assertListEqual(
+            files,
+            [
+                'firefox/feature.ftl',
+                'firefox/home.ftl',
+                'mozorg/mission.ftl',
+            ]
+        )
+        ref_path = self.path('/en/firefox/feature.ftl')
+        self.assertIsNotNone(pontoon_files.match(ref_path))
+        self.assertIsNotNone(vendor_files.match(ref_path))
+        ref_path = self.path('/en/firefox/home.ftl')
+        self.assertIsNotNone(pontoon_files.match(ref_path))
+        self.assertIsNotNone(vendor_files.match(ref_path))
+        ref_path = self.path('/en/mozorg/mission.ftl')
+        self.assertIsNotNone(pontoon_files.match(ref_path))
+        self.assertIsNone(vendor_files.match(ref_path))
+
+    def test_de(self, _walk, _isfile):
+        # home.ftl excluded completely by configs-special-templates.toml
+        # firefox/* only in vendor
+        pontoon_config, vendor_config, files = self._list('de', _walk, _isfile)
+        pontoon_files = ProjectFiles('de', [pontoon_config])
+        vendor_files = ProjectFiles('de', [vendor_config])
+        self.assertListEqual(
+            files,
+            [
+                'firefox/feature.ftl',
+                # 'firefox/home.ftl',
+                'mozorg/mission.ftl',
+            ]
+        )
+        l10n_path = self.path('/de/firefox/feature.ftl')
+        ref_path = self.path('/en/firefox/feature.ftl')
+        self.assertEqual(
+            pontoon_config.filter(
+                File(
+                    l10n_path,
+                    'de/firefox/feature.ftl',
+                    locale='de'
+                )
+            ),
+            'ignore'
+        )
+        self.assertIsNone(pontoon_files.match(l10n_path))
+        self.assertIsNone(pontoon_files.match(ref_path))
+        self.assertIsNotNone(vendor_files.match(l10n_path))
+        self.assertIsNotNone(vendor_files.match(ref_path))
+        l10n_path = self.path('/de/firefox/home.ftl')
+        ref_path = self.path('/en/firefox/home.ftl')
+        self.assertEqual(
+            pontoon_config.filter(
+                File(
+                    l10n_path,
+                    'de/firefox/home.ftl',
+                    locale='de'
+                )
+            ),
+            'ignore'
+        )
+        self.assertIsNone(pontoon_files.match(l10n_path))
+        self.assertIsNone(pontoon_files.match(ref_path))
+        self.assertIsNone(vendor_files.match(l10n_path))
+        self.assertIsNone(vendor_files.match(ref_path))
+        l10n_path = self.path('/de/mozorg/mission.ftl')
+        ref_path = self.path('/en/mozorg/mission.ftl')
+        self.assertEqual(
+            pontoon_config.filter(
+                File(
+                    l10n_path,
+                    'de/mozorg/mission.ftl',
+                    locale='de'
+                )
+            ),
+            'error'
+        )
+        self.assertIsNotNone(pontoon_files.match(l10n_path))
+        self.assertIsNotNone(pontoon_files.match(ref_path))
+        self.assertIsNone(vendor_files.match(l10n_path))
+        self.assertIsNone(vendor_files.match(ref_path))
+
+    def test_gd(self, _walk, _isfile):
+        # only community localization
+        pontoon_config, vendor_config, files = self._list('gd', _walk, _isfile)
+        pontoon_files = ProjectFiles('gd', [pontoon_config])
+        vendor_files = ProjectFiles('gd', [vendor_config])
+        self.assertListEqual(
+            files,
+            [
+                'firefox/feature.ftl',
+                'firefox/home.ftl',
+                'mozorg/mission.ftl',
+            ]
+        )
+        l10n_path = self.path('/gd/firefox/home.ftl')
+        ref_path = self.path('/en/firefox/home.ftl')
+        self.assertEqual(
+            pontoon_config.filter(
+                File(
+                    l10n_path,
+                    'gd/firefox/home.ftl',
+                    locale='gd'
+                )
+            ),
+            'error'
+        )
+        self.assertIsNotNone(pontoon_files.match(l10n_path))
+        self.assertIsNotNone(pontoon_files.match(ref_path))
+        self.assertIsNone(vendor_files.match(l10n_path))
+        self.assertIsNone(vendor_files.match(ref_path))
+
+    def test_it(self, _walk, _isfile):
+        # all pages translated, but split between vendor and community
+        pontoon_config, vendor_config, files = self._list('it', _walk, _isfile)
+        pontoon_files = ProjectFiles('it', [pontoon_config])
+        vendor_files = ProjectFiles('it', [vendor_config])
+        self.assertListEqual(
+            files,
+            [
+                'firefox/feature.ftl',
+                'firefox/home.ftl',
+                'mozorg/mission.ftl',
+            ]
+        )
+        l10n_path = self.path('/it/firefox/home.ftl')
+        ref_path = self.path('/en/firefox/home.ftl')
+        file = File(
+            l10n_path,
+            'it/firefox/home.ftl',
+            locale='it'
+        )
+        self.assertEqual(pontoon_config.filter(file), 'ignore')
+        self.assertEqual(vendor_config.filter(file), 'error')
+        self.assertIsNone(pontoon_files.match(l10n_path))
+        self.assertIsNone(pontoon_files.match(ref_path))
+        self.assertIsNotNone(vendor_files.match(l10n_path))
+        self.assertIsNotNone(vendor_files.match(ref_path))
+
+
 class TestL10nMerge(Rooted, unittest.TestCase):
     # need to go through TOMLParser, as that's handling most of the
     # environment

third_party/python/compare-locales/compare_locales/tests/paths/test_matcher.py

@@ -46,6 +46,18 @@ class TestMatcher(unittest.TestCase):
         self.assertTrue(one.match('foo/tender/bar/baz.qux'))
         self.assertFalse(one.match('foo/nobar/baz.qux'))
         self.assertFalse(one.match('foo/tender/bar'))
+        other = Matcher('baz/**/qux/**')
+        self.assertEqual(one.sub(other, 'foo/bar/baz.qux'), 'baz/qux/baz.qux')
+        self.assertEqual(
+            one.sub(other, 'foo/tender/bar/baz.qux'),
+            'baz/tender/qux/baz.qux'
+        )
+
+    def test_encoded_matcher(self):
+        one = Matcher('foo/*', encoding='utf-8')
+        self.assertTrue(one.match(b'foo/bar'))
+        other = Matcher('bar/*', encoding='utf-8')
+        self.assertEqual(one.sub(other, b'foo/baz'), b'bar/baz')
 
     def test_prefix(self):
         self.assertEqual(
@@ -87,6 +99,22 @@ class TestMatcher(unittest.TestCase):
             'foo/'
         )
 
+    def test_encoded_prefix(self):
+        self.assertEqual(
+            Matcher('foo/bar.file', encoding='utf-8').prefix, b'foo/bar.file'
+        )
+        self.assertEqual(
+            Matcher('foo/*', encoding='utf-8').prefix, b'foo/'
+        )
+        self.assertEqual(
+            Matcher('foo/{v}/bar', encoding='utf-8').prefix,
+            b'foo/'
+        )
+        self.assertEqual(
+            Matcher('foo/{v}/bar', {'v': 'expanded'}, encoding='utf-8').prefix,
+            b'foo/expanded/bar'
+        )
+
     def test_variables(self):
         self.assertDictEqual(
             Matcher('foo/bar.file').match('foo/bar.file'),
@@ -157,6 +185,30 @@ class TestMatcher(unittest.TestCase):
             }
         )
 
+    def test_encoded_variables(self):
+        self.assertDictEqual(
+            Matcher('foo/bar.file', encoding='utf-8').match(b'foo/bar.file'),
+            {}
+        )
+        self.assertDictEqual(
+            Matcher(
+                '{path}/bar.file', encoding='utf-8'
+            ).match(b'foo/bar.file'),
+            {
+                'path': 'foo'
+            }
+        )
+        self.assertDictEqual(
+            Matcher('{l}*', {
+                'l': 'foo/{locale}/'
+            }, encoding='utf-8').match(b'foo/it/path'),
+            {
+                'l': 'foo/it/',
+                'locale': 'it',
+                's1': 'path',
+            }
+        )
+
     def test_variables_sub(self):
         one = Matcher('{base}/{loc}/*', {'base': 'ONE_BASE'})
         other = Matcher('{base}/somewhere/*', {'base': 'OTHER_BASE'})
@@ -164,6 +216,14 @@ class TestMatcher(unittest.TestCase):
             one.sub(other, 'ONE_BASE/ab-CD/special'),
             'OTHER_BASE/somewhere/special'
         )
+        one = Matcher('{base}/{loc}/*', {'base': 'ONE_BASE'}, encoding='utf-8')
+        other = Matcher(
+            '{base}/somewhere/*', {'base': 'OTHER_BASE'}, encoding='utf-8'
+        )
+        self.assertEqual(
+            one.sub(other, b'ONE_BASE/ab-CD/special'),
+            b'OTHER_BASE/somewhere/special'
+        )
 
     def test_copy(self):
         one = Matcher('{base}/{loc}/*', {

third_party/python/compare-locales/compare_locales/tests/po/test_parser.py

@@ -128,7 +128,7 @@ msgstr ""
                 (Whitespace, '\n'),
                 (('reference 1', None), 'translated string'),
                 (Whitespace, '\n'),
-                (('reference 2', None), ''),
+                (('reference 2', None), 'reference 2'),
                 (Whitespace, '\n'),
             )
         )

third_party/python/compare-locales/compare_locales/tests/test_checks.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import unittest
+
+from compare_locales.checks.base import CSSCheckMixin
+
+
+class CSSParserTest(unittest.TestCase):
+    def setUp(self):
+        self.mixin = CSSCheckMixin()
+
+    def test_other(self):
+        refMap, errors = self.mixin.parse_css_spec('foo')
+        self.assertIsNone(refMap)
+        self.assertIsNone(errors)
+
+    def test_css_specs(self):
+        for prop in (
+            'min-width', 'width', 'max-width',
+            'min-height', 'height', 'max-height',
+        ):
+            refMap, errors = self.mixin.parse_css_spec('{}:1px;'.format(prop))
+            self.assertDictEqual(
+                refMap, {prop: 'px'}
+            )
+            self.assertIsNone(errors)
+
+    def test_single_whitespace(self):
+        refMap, errors = self.mixin.parse_css_spec('width:15px;')
+        self.assertDictEqual(
+            refMap, {'width': 'px'}
+        )
+        self.assertIsNone(errors)
+        refMap, errors = self.mixin.parse_css_spec('width   : \t 15px  ;  ')
+        self.assertDictEqual(
+            refMap, {'width': 'px'}
+        )
+        self.assertIsNone(errors)
+        refMap, errors = self.mixin.parse_css_spec('width: 15px')
+        self.assertDictEqual(
+            refMap, {'width': 'px'}
+        )
+        self.assertIsNone(errors)
+
+    def test_multiple(self):
+        refMap, errors = self.mixin.parse_css_spec('width:15px;height:20.2em;')
+        self.assertDictEqual(
+            refMap, {'height': 'em', 'width': 'px'}
+        )
+        self.assertIsNone(errors)
+        refMap, errors = self.mixin.parse_css_spec(
+            'width:15px \t\t;  height:20em'
+        )
+        self.assertDictEqual(
+            refMap, {'height': 'em', 'width': 'px'}
+        )
+        self.assertIsNone(errors)
+
+    def test_errors(self):
+        refMap, errors = self.mixin.parse_css_spec('width:15pxfoo')
+        self.assertDictEqual(
+            refMap, {'width': 'px'}
+        )
+        self.assertListEqual(
+            errors, [{'pos': 10, 'code': 'css-bad-content'}]
+        )
+        refMap, errors = self.mixin.parse_css_spec('width:15px height:20em')
+        self.assertDictEqual(
+            refMap, {'height': 'em', 'width': 'px'}
+        )
+        self.assertListEqual(
+            errors, [{'pos': 10, 'code': 'css-missing-semicolon'}]
+        )
+        refMap, errors = self.mixin.parse_css_spec('witdth:15px')
+        self.assertIsNone(refMap)
+        self.assertIsNone(errors)
+        refMap, errors = self.mixin.parse_css_spec('width:1,5px')
+        self.assertIsNone(refMap)
+        self.assertIsNone(errors)
+        refMap, errors = self.mixin.parse_css_spec('width:1.5.1px')
+        self.assertIsNone(refMap)
+        self.assertIsNone(errors)
+        refMap, errors = self.mixin.parse_css_spec('width:1.px')
+        self.assertIsNone(refMap)
+        self.assertIsNone(errors)

third_party/python/compare-locales/compare_locales/tests/test_compare.py

@@ -91,7 +91,17 @@ class TestObserver(unittest.TestCase):
         self.assertDictEqual(obs.toJSON(), {
             'summary': {
                 'de': {
-                    'missing': 15
+                    'errors': 0,
+                    'warnings': 0,
+                    'missing': 15,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
+                    'changed': 0,
+                    'changed_w': 0,
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }
             },
             'details': {
@@ -112,7 +122,17 @@ class TestObserver(unittest.TestCase):
         self.assertDictEqual(obs.toJSON(), {
             'summary': {
                 'de': {
-                    'missing': 15
+                    'errors': 0,
+                    'warnings': 0,
+                    'missing': 15,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
+                    'changed': 0,
+                    'changed_w': 0,
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }
             },
             'details': {

third_party/python/compare-locales/compare_locales/tests/test_merge.py

@@ -150,10 +150,17 @@ class TestDefines(unittest.TestCase, ContentMixin):
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 0,
+                    'warnings': 0,
+                    'missing': 0,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
                     'changed': 1,
                     'changed_w': 2,
                     'unchanged': 1,
-                    'unchanged_w': 1
+                    'unchanged_w': 1,
+                    'keys': 0,
                 }},
              'details': {}
              }
@@ -189,10 +196,17 @@ class TestDefines(unittest.TestCase, ContentMixin):
             {
                 'summary':
                     {None: {
+                        'errors': 0,
+                        'warnings': 0,
                         'missing': 1,
                         'missing_w': 2,
+                        'report': 0,
+                        'obsolete': 0,
+                        'changed': 0,
+                        'changed_w': 0,
                         'unchanged': 1,
-                        'unchanged_w': 1
+                        'unchanged_w': 1,
+                        'keys': 0,
                     }},
                 'details':
                     {
@@ -238,8 +252,18 @@ eff = lEff word
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 0,
+                    'warnings': 0,
+                    'missing': 0,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
                     'changed': 3,
-                    'changed_w': 5
+                    'changed_w': 5,
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
+
                 }},
              'details': {}
              }
@@ -265,10 +289,17 @@ eff = effVal""")
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 0,
+                    'warnings': 0,
+                    'missing': 2,
+                    'missing_w': 2,
+                    'report': 0,
+                    'obsolete': 0,
                     'changed': 1,
                     'changed_w': 1,
-                    'missing': 2,
-                    'missing_w': 2
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.properties': [
@@ -297,8 +328,17 @@ eff = effVal""")
             cc.observers.toJSON(),
             {'summary':
                 {None: {
-                    'missingInFiles': 3,
-                    'missing_w': 3
+                    'errors': 0,
+                    'warnings': 0,
+                    'missing': 3,
+                    'missing_w': 3,
+                    'report': 0,
+                    'obsolete': 0,
+                    'changed': 0,
+                    'changed_w': 0,
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.properties': [
@@ -326,11 +366,17 @@ eff = leffVal
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 1,
+                    'warnings': 0,
+                    'missing': 1,
+                    'missing_w': 1,
+                    'report': 0,
+                    'obsolete': 0,
                     'changed': 2,
                     'changed_w': 3,
-                    'errors': 1,
-                    'missing': 1,
-                    'missing_w': 1
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.properties': [
@@ -364,11 +410,17 @@ eff = leffVal
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 0,
+                    'warnings': 0,
+                    'missing': 0,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 1,
                     'changed': 1,
                     'changed_w': 1,
-                    'obsolete': 1,
                     'unchanged': 1,
-                    'unchanged_w': 1
+                    'unchanged_w': 1,
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.properties': [
@@ -422,8 +474,15 @@ bar = duplicated bar
                 {None: {
                     'errors': 1,
                     'warnings': 1,
+                    'missing': 0,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
                     'changed': 3,
-                    'changed_w': 6
+                    'changed_w': 6,
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.properties': [
@@ -465,8 +524,17 @@ class TestDTD(unittest.TestCase, ContentMixin):
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 0,
+                    'warnings': 0,
+                    'missing': 0,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
                     'changed': 3,
-                    'changed_w': 3
+                    'changed_w': 3,
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }},
              'details': {}
              }
@@ -492,10 +560,17 @@ class TestDTD(unittest.TestCase, ContentMixin):
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 0,
+                    'warnings': 0,
+                    'missing': 2,
+                    'missing_w': 2,
+                    'report': 0,
+                    'obsolete': 0,
                     'changed': 1,
                     'changed_w': 1,
-                    'missing': 2,
-                    'missing_w': 2
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.dtd': [
@@ -529,10 +604,16 @@ class TestDTD(unittest.TestCase, ContentMixin):
             {'summary':
                 {None: {
                     'errors': 1,
+                    'warnings': 0,
                     'missing': 1,
                     'missing_w': 1,
+                    'report': 0,
+                    'obsolete': 0,
+                    'changed': 0,
+                    'changed_w': 0,
                     'unchanged': 2,
-                    'unchanged_w': 2
+                    'unchanged_w': 2,
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.dtd': [
@@ -567,9 +648,17 @@ class TestDTD(unittest.TestCase, ContentMixin):
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 0,
                     'warnings': 1,
+                    'missing': 0,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
+                    'changed': 0,
+                    'changed_w': 0,
                     'unchanged': 2,
-                    'unchanged_w': 2
+                    'unchanged_w': 2,
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.dtd': [
@@ -597,11 +686,17 @@ class TestDTD(unittest.TestCase, ContentMixin):
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 0,
                     'warnings': 1,
+                    'missing': 0,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
+                    'changed': 1,
+                    'changed_w': 2,
                     'unchanged': 2,
                     'unchanged_w': 2,
-                    'changed': 1,
-                    'changed_w': 2
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.dtd': [
@@ -658,8 +753,17 @@ bar = lBar
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 0,
+                    'warnings': 0,
+                    'missing': 0,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
                     'changed': 3,
-                    'changed_w': 3
+                    'changed_w': 3,
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }},
              'details': {}
              }
@@ -697,10 +801,17 @@ eff = lEff
                 },
                 'summary': {
                     None: {
-                        'changed': 2,
-                        'changed_w': 2,
+                        'errors': 0,
+                        'warnings': 0,
                         'missing': 2,
                         'missing_w': 2,
+                        'report': 0,
+                        'obsolete': 0,
+                        'changed': 2,
+                        'changed_w': 2,
+                        'unchanged': 0,
+                        'unchanged_w': 0,
+                        'keys': 0,
                     }
                 }
             }
@@ -748,11 +859,17 @@ eff = lEff {
                 },
                 'summary': {
                     None: {
-                        'changed': 1,
-                        'changed_w': 1,
+                        'errors': 3,
+                        'warnings': 0,
                         'missing': 2,
                         'missing_w': 2,
-                        'errors': 3
+                        'report': 0,
+                        'obsolete': 0,
+                        'changed': 1,
+                        'changed_w': 1,
+                        'unchanged': 0,
+                        'unchanged_w': 0,
+                        'keys': 0,
                     }
                 }
             }
@@ -793,8 +910,17 @@ foo = Localized { bar }
                 'details': {},
                 'summary': {
                     None: {
+                        'errors': 0,
+                        'warnings': 0,
+                        'missing': 0,
+                        'missing_w': 0,
+                        'report': 0,
+                        'obsolete': 0,
                         'changed': 1,
-                        'changed_w': 1
+                        'changed_w': 1,
+                        'unchanged': 0,
+                        'unchanged_w': 0,
+                        'keys': 0,
                     }
                 }
             }
@@ -850,9 +976,17 @@ baz = Localized { qux }
                 },
                 'summary': {
                     None: {
+                        'errors': 0,
+                        'warnings': 4,
+                        'missing': 0,
+                        'missing_w': 0,
+                        'report': 0,
+                        'obsolete': 0,
                         'changed': 3,
                         'changed_w': 3,
-                        'warnings': 4
+                        'unchanged': 0,
+                        'unchanged_w': 0,
+                        'keys': 0,
                     }
                 }
             }
@@ -899,7 +1033,19 @@ eff = lEff
                     ],
                 },
                 'summary': {
-                    None: {'changed': 3, 'changed_w': 5, 'errors': 2}
+                    None: {
+                        'errors': 2,
+                        'warnings': 0,
+                        'missing': 0,
+                        'missing_w': 0,
+                        'report': 0,
+                        'obsolete': 0,
+                        'changed': 3,
+                        'changed_w': 5,
+                        'unchanged': 0,
+                        'unchanged_w': 0,
+                        'keys': 0,
+                    }
                 }
             }
         )
@@ -955,10 +1101,17 @@ eff = lEff
                 },
                 'summary': {
                     None: {
-                        'changed': 4,
-                        'changed_w': 4,
+                        'errors': 0,
+                        'warnings': 0,
                         'missing': 1,
                         'missing_w': 1,
+                        'report': 0,
+                        'obsolete': 0,
+                        'changed': 4,
+                        'changed_w': 4,
+                        'unchanged': 0,
+                        'unchanged_w': 0,
+                        'keys': 0,
                     }
                 }
             }
@@ -1003,7 +1156,19 @@ bar = lBar
                     ]
                 },
                 'summary': {
-                    None: {'changed': 2, 'changed_w': 4, 'errors': 2}
+                    None: {
+                        'errors': 2,
+                        'warnings': 0,
+                        'missing': 0,
+                        'missing_w': 0,
+                        'report': 0,
+                        'obsolete': 0,
+                        'changed': 2,
+                        'changed_w': 4,
+                        'unchanged': 0,
+                        'unchanged_w': 0,
+                        'keys': 0,
+                    }
                 }
             }
         )
@@ -1040,8 +1205,17 @@ bar = lBar
                 'details': {},
                 'summary': {
                     None: {
+                        'errors': 0,
+                        'warnings': 0,
+                        'missing': 0,
+                        'missing_w': 0,
+                        'report': 0,
+                        'obsolete': 0,
                         'changed': 2,
                         'changed_w': 2,
+                        'unchanged': 0,
+                        'unchanged_w': 0,
+                        'keys': 0,
                     }
                 }
             }
@@ -1074,10 +1248,17 @@ bar = barVal
                 'details': {},
                 'summary': {
                     None: {
+                        'errors': 0,
+                        'warnings': 0,
+                        'missing': 0,
+                        'missing_w': 0,
+                        'report': 0,
+                        'obsolete': 0,
                         'changed': 1,
                         'changed_w': 1,
                         'unchanged': 1,
                         'unchanged_w': 1,
+                        'keys': 0,
                     }
                 }
             }
@@ -1111,8 +1292,17 @@ bar = lBar
                 'details': {},
                 'summary': {
                     None: {
+                        'errors': 0,
+                        'warnings': 0,
+                        'missing': 0,
+                        'missing_w': 0,
+                        'report': 0,
+                        'obsolete': 0,
                         'changed': 2,
                         'changed_w': 2,
+                        'unchanged': 0,
+                        'unchanged_w': 0,
+                        'keys': 0,
                     }
                 }
             }
@@ -1144,8 +1334,15 @@ bar = duplicated bar
                 {None: {
                     'errors': 1,
                     'warnings': 1,
+                    'missing': 0,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
                     'changed': 3,
-                    'changed_w': 6
+                    'changed_w': 6,
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.ftl': [
@@ -1174,9 +1371,17 @@ bar = duplicated bar
             cc.observers.toJSON(),
             {'summary':
                 {None: {
+                    'errors': 0,
                     'warnings': 3,
+                    'missing': 0,
+                    'missing_w': 0,
+                    'report': 0,
+                    'obsolete': 0,
                     'changed': 1,
-                    'changed_w': 2
+                    'changed_w': 2,
+                    'unchanged': 0,
+                    'unchanged_w': 0,
+                    'keys': 0,
                 }},
              'details': {
                  'l10n.ftl': [

third_party/python/compare-locales/compare_locales/tests/test_parser.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
+import pkg_resources
 import shutil
 import tempfile
 import textwrap
@@ -87,3 +88,31 @@ class TestUniversalNewlines(unittest.TestCase):
         self.assertEqual(
             self.parser.ctx.contents,
             'one\ntwo\nthree\n')
+
+
+class TestPlugins(unittest.TestCase):
+    def setUp(self):
+        self.old_working_set_state = pkg_resources.working_set.__getstate__()
+        distribution = pkg_resources.Distribution(__file__)
+        entry_point = pkg_resources.EntryPoint.parse(
+            'test_parser = compare_locales.tests.test_parser:DummyParser',
+            dist=distribution
+        )
+        distribution._ep_map = {
+            'compare_locales.parsers': {
+                'test_parser': entry_point
+            }
+        }
+        pkg_resources.working_set.add(distribution)
+
+    def tearDown(self):
+        pkg_resources.working_set.__setstate__(self.old_working_set_state)
+
+    def test_dummy_parser(self):
+        p = parser.getParser('some/weird/file.ext')
+        self.assertIsInstance(p, DummyParser)
+
+
+class DummyParser(parser.Parser):
+    def use(self, path):
+        return path.endswith('weird/file.ext')

third_party/python/compare-locales/setup.py

@@ -52,8 +52,11 @@ setup(name="compare-locales",
           'compare_locales.tests': ['data/*.properties', 'data/*.dtd']
       },
       install_requires=[
-          'fluent.syntax >=0.14.0, <0.16',
+          'fluent.syntax >=0.17.0, <0.18',
           'pytoml',
           'six',
       ],
+      tests_require=[
+          'mock<4.0',
+      ],
       test_suite='compare_locales.tests')

third_party/python/fluent.migrate/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.1
+Name: fluent.migrate
+Version: 0.9
+Summary: Toolchain to migrate legacy translation to Fluent.
+Home-page: https://hg.mozilla.org/l10n/fluent-migration/
+Author: Mozilla
+Author-email: l10n-drivers@mozilla.org
+License: APL 2
+Description: Fluent Migration Tools
+        ======================
+        
+        Programmatically create Fluent files from existing content in both legacy
+        and Fluent formats. Use recipes written in Python to migrate content for each
+        of your localizations.
+        
+        `migrate-l10n` is a CLI script which uses the `fluent.migrate` module under
+        the hood to run migrations on existing translations.
+        
+        `validate-l10n-recipe` is a CLI script to test a migration recipe for common
+        errors, without trying to apply it.
+        
+        Installation
+        ------------
+        
+        Install from PyPI:
+        
+            pip install fluent.migrate[hg]
+        
+        If you only want to use the `MigrationContext` API, you can drop the
+        requirement on `python-hglib`:
+        
+            pip install fluent.migrate
+        
+        Usage
+        -----
+        
+        Migrations consist of _recipes_, which are applied to a _localization repository_, based on _template files_.
+        You can find recipes for Firefox in `mozilla-central/python/l10n/fluent_migrations/`,
+        the reference repository is [gecko-strings](https://hg.mozilla.org/l10n/gecko-strings/) or _quarantine_.
+        You apply those migrations to l10n repositories in [l10n-central](https://hg.mozilla.org/l10n-central/), or to `gecko-strings` for testing.
+        
+        The migrations are run as python modules, so you need to have their file location in `PYTHONPATH`.
+        
+        An example would look like
+        
+            $ migrate-l10n --lang it --reference-dir gecko-strings --localization-dir l10n-central/it bug_1451992_preferences_sitedata bug_1451992_preferences_translation
+        
+        Contact
+        -------
+        
+         - mailing list: https://lists.mozilla.org/listinfo/tools-l10n
+         - bugzilla: [Open Bugs](https://bugzilla.mozilla.org/buglist.cgi?component=Fluent%20Migration&product=Localization%20Infrastructure%20and%20Tools&bug_status=__open__) - [New Bug](https://bugzilla.mozilla.org/enter_bug.cgi?product=Localization%20Infrastructure%20and%20Tools&component=Fluent%20Migration)
+        
+Keywords: fluent,localization,l10n
+Platform: UNKNOWN
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.7
+Description-Content-Type: text/markdown
+Provides-Extra: hg

third_party/python/fluent.migrate/README.md

@@ -0,0 +1,44 @@
+Fluent Migration Tools
+======================
+
+Programmatically create Fluent files from existing content in both legacy
+and Fluent formats. Use recipes written in Python to migrate content for each
+of your localizations.
+
+`migrate-l10n` is a CLI script which uses the `fluent.migrate` module under
+the hood to run migrations on existing translations.
+
+`validate-l10n-recipe` is a CLI script to test a migration recipe for common
+errors, without trying to apply it.
+
+Installation
+------------
+
+Install from PyPI:
+
+    pip install fluent.migrate[hg]
+
+If you only want to use the `MigrationContext` API, you can drop the
+requirement on `python-hglib`:
+
+    pip install fluent.migrate
+
+Usage
+-----
+
+Migrations consist of _recipes_, which are applied to a _localization repository_, based on _template files_.
+You can find recipes for Firefox in `mozilla-central/python/l10n/fluent_migrations/`,
+the reference repository is [gecko-strings](https://hg.mozilla.org/l10n/gecko-strings/) or _quarantine_.
+You apply those migrations to l10n repositories in [l10n-central](https://hg.mozilla.org/l10n-central/), or to `gecko-strings` for testing.
+
+The migrations are run as python modules, so you need to have their file location in `PYTHONPATH`.
+
+An example would look like
+
+    $ migrate-l10n --lang it --reference-dir gecko-strings --localization-dir l10n-central/it bug_1451992_preferences_sitedata bug_1451992_preferences_translation
+
+Contact
+-------
+
+ - mailing list: https://lists.mozilla.org/listinfo/tools-l10n
+ - bugzilla: [Open Bugs](https://bugzilla.mozilla.org/buglist.cgi?component=Fluent%20Migration&product=Localization%20Infrastructure%20and%20Tools&bug_status=__open__) - [New Bug](https://bugzilla.mozilla.org/enter_bug.cgi?product=Localization%20Infrastructure%20and%20Tools&component=Fluent%20Migration)

third_party/python/fluent.migrate/fluent/migrate/_context.py

@@ -0,0 +1,325 @@
+# coding=utf8
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import os
+import codecs
+from functools import partial
+import logging
+from six.moves import zip_longest
+
+import fluent.syntax.ast as FTL
+from fluent.syntax.parser import FluentParser
+from fluent.syntax.serializer import FluentSerializer
+from compare_locales.parser import getParser
+from compare_locales.plurals import get_plural
+
+from .merge import merge_resource
+from .errors import (
+    UnreadableReferenceError,
+)
+
+
+class InternalContext(object):
+    """Internal context for merging translation resources.
+
+    For the public interface, see `context.MigrationContext`.
+    """
+
+    def __init__(
+        self, lang, reference_dir, localization_dir, enforce_translated=False
+    ):
+        self.fluent_parser = FluentParser(with_spans=False)
+        self.fluent_serializer = FluentSerializer()
+
+        # An iterable of plural category names relevant to the context's
+        # language.  E.g. ('one', 'other') for English.
+        self.plural_categories = get_plural(lang)
+        if self.plural_categories is None:
+            logger = logging.getLogger('migrate')
+            logger.warning(
+                'Plural rule for "{}" is not defined in '
+                'compare-locales'.format(lang))
+            self.plural_categories = ('one', 'other')
+
+        self.enforce_translated = enforce_translated
+        # Parsed input resources stored by resource path.
+        self.reference_resources = {}
+        self.localization_resources = {}
+        self.target_resources = {}
+
+        # An iterable of `FTL.Message` objects some of whose nodes can be the
+        # transform operations.
+        self.transforms = {}
+
+    def read_ftl_resource(self, path):
+        """Read an FTL resource and parse it into an AST."""
+        f = codecs.open(path, 'r', 'utf8')
+        try:
+            contents = f.read()
+        except UnicodeDecodeError as err:
+            logger = logging.getLogger('migrate')
+            logger.warning('Unable to read file {}: {}'.format(path, err))
+            raise err
+        finally:
+            f.close()
+
+        ast = self.fluent_parser.parse(contents)
+
+        annots = [
+            annot
+            for entry in ast.body
+            if isinstance(entry, FTL.Junk)
+            for annot in entry.annotations
+        ]
+
+        if len(annots):
+            logger = logging.getLogger('migrate')
+            for annot in annots:
+                msg = annot.message
+                logger.warning('Syntax error in {}: {}'.format(path, msg))
+
+        return ast
+
+    def read_legacy_resource(self, path):
+        """Read a legacy resource and parse it into a dict."""
+        parser = getParser(path)
+        parser.readFile(path)
+        # Transform the parsed result which is an iterator into a dict.
+        return {
+            entity.key: entity.val for entity in parser
+            if entity.localized or self.enforce_translated
+        }
+
+    def read_reference_ftl(self, path):
+        """Read and parse a reference FTL file.
+
+        A missing resource file is a fatal error and will raise an
+        UnreadableReferenceError.
+        """
+        fullpath = os.path.join(self.reference_dir, path)
+        try:
+            return self.read_ftl_resource(fullpath)
+        except IOError:
+            error_message = 'Missing reference file: {}'.format(fullpath)
+            logging.getLogger('migrate').error(error_message)
+            raise UnreadableReferenceError(error_message)
+        except UnicodeDecodeError as err:
+            error_message = 'Error reading file {}: {}'.format(fullpath, err)
+            logging.getLogger('migrate').error(error_message)
+            raise UnreadableReferenceError(error_message)
+
+    def read_localization_ftl(self, path):
+        """Read and parse an existing localization FTL file.
+
+        Create a new FTL.Resource if the file doesn't exist or can't be
+        decoded.
+        """
+        fullpath = os.path.join(self.localization_dir, path)
+        try:
+            return self.read_ftl_resource(fullpath)
+        except IOError:
+            logger = logging.getLogger('migrate')
+            logger.info(
+                'Localization file {} does not exist and '
+                'it will be created'.format(path))
+            return FTL.Resource()
+        except UnicodeDecodeError:
+            logger = logging.getLogger('migrate')
+            logger.warning(
+                'Localization file {} has broken encoding. '
+                'It will be re-created and some translations '
+                'may be lost'.format(path))
+            return FTL.Resource()
+
+    def maybe_add_localization(self, path):
+        """Add a localization resource to migrate translations from.
+
+        Uses a compare-locales parser to create a dict of (key, string value)
+        tuples.
+        For Fluent sources, we store the AST.
+        """
+        try:
+            fullpath = os.path.join(self.localization_dir, path)
+            if not fullpath.endswith('.ftl'):
+                collection = self.read_legacy_resource(fullpath)
+            else:
+                collection = self.read_ftl_resource(fullpath)
+        except IOError:
+            logger = logging.getLogger('migrate')
+            logger.warning('Missing localization file: {}'.format(path))
+        else:
+            self.localization_resources[path] = collection
+
+    def get_legacy_source(self, path, key):
+        """Get an entity value from a localized legacy source.
+
+        Used by the `Source` transform.
+        """
+        resource = self.localization_resources[path]
+        return resource.get(key, None)
+
+    def get_fluent_source_pattern(self, path, key):
+        """Get a pattern from a localized Fluent source.
+
+        If the key contains a `.`, does an attribute lookup.
+        Used by the `COPY_PATTERN` transform.
+        """
+        resource = self.localization_resources[path]
+        msg_key, _, attr_key = key.partition('.')
+        found = None
+        for entry in resource.body:
+            if isinstance(entry, (FTL.Message, FTL.Term)):
+                if entry.id.name == msg_key:
+                    found = entry
+                    break
+        if found is None:
+            return None
+        if not attr_key:
+            return found.value
+        for attribute in found.attributes:
+            if attribute.id.name == attr_key:
+                return attribute.value
+        return None
+
+    def messages_equal(self, res1, res2):
+        """Compare messages and terms of two FTL resources.
+
+        Uses FTL.BaseNode.equals to compare all messages/terms
+        in two FTL resources.
+        If the order or number of messages differ, the result is also False.
+        """
+        def message_id(message):
+            "Return the message's identifer name for sorting purposes."
+            return message.id.name
+
+        messages1 = sorted(
+            (entry for entry in res1.body
+             if isinstance(entry, FTL.Message)
+                or isinstance(entry, FTL.Term)),
+            key=message_id)
+        messages2 = sorted(
+            (entry for entry in res2.body
+             if isinstance(entry, FTL.Message)
+                or isinstance(entry, FTL.Term)),
+            key=message_id)
+        for msg1, msg2 in zip_longest(messages1, messages2):
+            if msg1 is None or msg2 is None:
+                return False
+            if not msg1.equals(msg2):
+                return False
+        return True
+
+    def merge_changeset(self, changeset=None, known_translations=None):
+        """Return a generator of FTL ASTs for the changeset.
+
+        The input data must be configured earlier using the `add_*` methods.
+        if given, `changeset` must be a set of (path, key) tuples describing
+        which legacy translations are to be merged. If `changeset` is None,
+        all legacy translations will be allowed to be migrated in a single
+        changeset.
+
+        We use the `in_changeset` method to determine if a message should be
+        migrated for the given changeset.
+
+        Given `changeset`, return a dict whose keys are resource paths and
+        values are `FTL.Resource` instances.  The values will also be used to
+        update this context's existing localization resources.
+        """
+
+        if changeset is None:
+            # Merge all known legacy translations. Used in tests.
+            changeset = {
+                (path, key)
+                for path, strings in self.localization_resources.items()
+                if not path.endswith('.ftl')
+                for key in strings.keys()
+            }
+
+        if known_translations is None:
+            known_translations = changeset
+
+        for path, reference in self.reference_resources.items():
+            current = self.target_resources[path]
+            transforms = self.transforms.get(path, [])
+            in_changeset = partial(
+                self.in_changeset, changeset, known_translations, path)
+
+            # Merge legacy translations with the existing ones using the
+            # reference as a template.
+            snapshot = merge_resource(
+                self, reference, current, transforms, in_changeset
+            )
+
+            # Skip this path if the messages in the merged snapshot are
+            # identical to those in the current state of the localization file.
+            # This may happen when:
+            #
+            #   - none of the transforms is in the changset, or
+            #   - all messages which would be migrated by the context's
+            #     transforms already exist in the current state.
+            if self.messages_equal(current, snapshot):
+                continue
+
+            # Store the merged snapshot on the context so that the next merge
+            # already takes it into account as the existing localization.
+            self.target_resources[path] = snapshot
+
+            # The result for this path is a complete `FTL.Resource`.
+            yield path, snapshot
+
+    def in_changeset(self, changeset, known_translations, path, ident):
+        """Check if a message should be migrated in this changeset.
+
+        The message is identified by path and ident.
+
+
+        A message will be migrated only if all of its dependencies
+        are present in the currently processed changeset.
+
+        If a transform defined for this message points to a missing
+        legacy translation, this message will not be merged. The
+        missing legacy dependency won't be present in the changeset.
+
+        This also means that partially translated messages (e.g.
+        constructed from two legacy strings out of which only one is
+        avaiable) will never be migrated.
+        """
+        message_deps = self.dependencies.get((path, ident), None)
+
+        # Don't merge if we don't have a transform for this message.
+        if message_deps is None:
+            return False
+
+        # As a special case, if a transform exists but has no
+        # dependecies, it's a hardcoded `FTL.Node` which doesn't
+        # migrate any existing translation but rather creates a new
+        # one.  Merge it.
+        if len(message_deps) == 0:
+            return True
+
+        # Make sure all the dependencies are present in the current
+        # changeset. Partial migrations are not currently supported.
+        # See https://bugzilla.mozilla.org/show_bug.cgi?id=1321271
+        # We only return True if our current changeset touches
+        # the transform, and we have all of the dependencies.
+        active_deps = message_deps & changeset
+        available_deps = message_deps & known_translations
+        return active_deps and message_deps == available_deps
+
+    def serialize_changeset(self, changeset, known_translations=None):
+        """Return a dict of serialized FTLs for the changeset.
+
+        Given `changeset`, return a dict whose keys are resource paths and
+        values are serialized FTL snapshots.
+        """
+
+        return {
+            path: self.fluent_serializer.serialize(snapshot)
+            for path, snapshot in self.merge_changeset(
+                changeset, known_translations
+            )
+        }
+
+
+logging.basicConfig()

third_party/python/fluent.migrate/fluent/migrate/changesets.py

@@ -27,7 +27,7 @@ def convert_blame_to_changesets(blame_json):
         }
 
     It will be transformed into a list of changesets which can be fed into
-    `MergeContext.serialize_changeset`:
+    `InternalContext.serialize_changeset`:
 
         [
             {

third_party/python/fluent.migrate/fluent/migrate/context.py

@@ -2,181 +2,72 @@
 from __future__ import unicode_literals
 from __future__ import absolute_import
 
-import os
-import codecs
-from functools import partial
 import logging
-from six.moves import zip_longest
 
 import fluent.syntax.ast as FTL
-from fluent.syntax.parser import FluentParser
-from fluent.syntax.serializer import FluentSerializer
 from fluent.migrate.util import fold
-from compare_locales.parser import getParser
-from compare_locales.plurals import CATEGORIES_BY_LOCALE
 
 from .transforms import Source
-from .merge import merge_resource
-from .util import get_message
+from .util import get_message, skeleton
 from .errors import (
     EmptyLocalizationError,
     UnreadableReferenceError,
 )
+from ._context import InternalContext
 
 
-class MergeContext(object):
+__all__ = [
+    'EmptyLocalizationError',
+    'UnreadableReferenceError',
+    'MigrationContext',
+]
+
+
+class MigrationContext(InternalContext):
     """Stateful context for merging translation resources.
 
-    `MergeContext` must be configured with the target language and the
+    `MigrationContext` must be configured with the target locale and the
     directory locations of the input data.
 
     The transformation takes four types of input data:
 
         - The en-US FTL reference files which will be used as templates for
-          message order, comments and sections.
+          message order, comments and sections. If the reference_dir is None,
+          the migration will create Messages and Terms in the order given by
+          the transforms.
 
-        - The current FTL files for the given language.
-
-        - The legacy (DTD, properties) translation files for the given
-          language.  The translations from these files will be transformed
-          into FTL and merged into the existing FTL files for this language.
+        - The current FTL files for the given locale.
 
         - A list of `FTL.Message` or `FTL.Term` objects some of whose nodes
           are special helper or transform nodes:
 
               helpers: VARIABLE_REFERENCE, MESSAGE_REFERENCE, TERM_REFERENCE
               transforms: COPY, REPLACE_IN_TEXT, REPLACE, PLURALS, CONCAT
+              fluent value helper: COPY_PATTERN
+
+    The legacy (DTD, properties) translation files are deduced by the
+    dependencies in the transforms. The translations from these files will be
+    read from the localization_dir and transformed into FTL and merged
+    into the existing FTL files for the given language.
     """
 
-    def __init__(self, lang, reference_dir, localization_dir):
-        self.fluent_parser = FluentParser(with_spans=False)
-        self.fluent_serializer = FluentSerializer()
-
-        # An iterable of plural category names relevant to the context's
-        # language.  E.g. ('one', 'other') for English.
-        try:
-            self.plural_categories = CATEGORIES_BY_LOCALE[lang]
-        except KeyError as locale_key:
-            logger = logging.getLogger('migrate')
-            logger.warning(
-                'Plural rule for "{}" is not defined in '
-                'compare-locales'.format(locale_key))
-            self.plural_categories = ('one', 'other')
-
+    def __init__(
+        self, locale, reference_dir, localization_dir, enforce_translated=False
+    ):
+        super(MigrationContext, self).__init__(
+            locale, reference_dir, localization_dir,
+            enforce_translated=enforce_translated
+        )
+        self.locale = locale
         # Paths to directories with input data, relative to CWD.
         self.reference_dir = reference_dir
         self.localization_dir = localization_dir
 
-        # Parsed input resources stored by resource path.
-        self.reference_resources = {}
-        self.localization_resources = {}
-        self.target_resources = {}
-
-        # An iterable of `FTL.Message` objects some of whose nodes can be the
-        # transform operations.
-        self.transforms = {}
-
         # A dict whose keys are `(path, key)` tuples corresponding to target
         # FTL translations, and values are sets of `(path, key)` tuples
         # corresponding to localized entities which will be migrated.
         self.dependencies = {}
 
-    def read_ftl_resource(self, path):
-        """Read an FTL resource and parse it into an AST."""
-        f = codecs.open(path, 'r', 'utf8')
-        try:
-            contents = f.read()
-        except UnicodeDecodeError as err:
-            logger = logging.getLogger('migrate')
-            logger.warning('Unable to read file {}: {}'.format(path, err))
-            raise err
-        finally:
-            f.close()
-
-        ast = self.fluent_parser.parse(contents)
-
-        annots = [
-            annot
-            for entry in ast.body
-            if isinstance(entry, FTL.Junk)
-            for annot in entry.annotations
-        ]
-
-        if len(annots):
-            logger = logging.getLogger('migrate')
-            for annot in annots:
-                msg = annot.message
-                logger.warning('Syntax error in {}: {}'.format(path, msg))
-
-        return ast
-
-    def read_legacy_resource(self, path):
-        """Read a legacy resource and parse it into a dict."""
-        parser = getParser(path)
-        parser.readFile(path)
-        # Transform the parsed result which is an iterator into a dict.
-        return {entity.key: entity.val for entity in parser}
-
-    def read_reference_ftl(self, path):
-        """Read and parse a reference FTL file.
-
-        A missing resource file is a fatal error and will raise an
-        UnreadableReferenceError.
-        """
-        fullpath = os.path.join(self.reference_dir, path)
-        try:
-            return self.read_ftl_resource(fullpath)
-        except IOError:
-            error_message = 'Missing reference file: {}'.format(fullpath)
-            logging.getLogger('migrate').error(error_message)
-            raise UnreadableReferenceError(error_message)
-        except UnicodeDecodeError as err:
-            error_message = 'Error reading file {}: {}'.format(fullpath, err)
-            logging.getLogger('migrate').error(error_message)
-            raise UnreadableReferenceError(error_message)
-
-    def read_localization_ftl(self, path):
-        """Read and parse an existing localization FTL file.
-
-        Create a new FTL.Resource if the file doesn't exist or can't be
-        decoded.
-        """
-        fullpath = os.path.join(self.localization_dir, path)
-        try:
-            return self.read_ftl_resource(fullpath)
-        except IOError:
-            logger = logging.getLogger('migrate')
-            logger.info(
-                'Localization file {} does not exist and '
-                'it will be created'.format(path))
-            return FTL.Resource()
-        except UnicodeDecodeError:
-            logger = logging.getLogger('migrate')
-            logger.warning(
-                'Localization file {} has broken encoding. '
-                'It will be re-created and some translations '
-                'may be lost'.format(path))
-            return FTL.Resource()
-
-    def maybe_add_localization(self, path):
-        """Add a localization resource to migrate translations from.
-
-        Uses a compare-locales parser to create a dict of (key, string value)
-        tuples.
-        For Fluent sources, we store the AST.
-        """
-        try:
-            fullpath = os.path.join(self.localization_dir, path)
-            if not fullpath.endswith('.ftl'):
-                collection = self.read_legacy_resource(fullpath)
-            else:
-                collection = self.read_ftl_resource(fullpath)
-        except IOError:
-            logger = logging.getLogger('migrate')
-            logger.warning('Missing localization file: {}'.format(path))
-        else:
-            self.localization_resources[path] = collection
-
     def add_transforms(self, target, reference, transforms):
         """Define transforms for target using reference as template.
 
@@ -191,13 +82,26 @@ class MergeContext(object):
 
         Each transform is scanned for `Source` nodes which will be used to
         build the list of dependencies for the transformed message.
+
+        For transforms that merely copy legacy messages or Fluent patterns,
+        using `fluent.migrate.helpers.transforms_from` is recommended.
         """
         def get_sources(acc, cur):
             if isinstance(cur, Source):
                 acc.add((cur.path, cur.key))
             return acc
 
-        reference_ast = self.read_reference_ftl(reference)
+        if self.reference_dir is None:
+            # Add skeletons to resource body for each transform
+            # if there's no reference.
+            reference_ast = self.reference_resources.get(target)
+            if reference_ast is None:
+                reference_ast = FTL.Resource()
+            reference_ast.body.extend(
+                skeleton(transform) for transform in transforms
+            )
+        else:
+            reference_ast = self.read_reference_ftl(reference)
         self.reference_resources[target] = reference_ast
 
         for node in transforms:
@@ -210,6 +114,9 @@ class MergeContext(object):
 
             # The target Fluent message should exist in the reference file. If
             # it doesn't, it's probably a typo.
+            # Of course, only if we're having a reference.
+            if self.reference_dir is None:
+                continue
             if get_message(reference_ast.body, ident) is None:
                 logger = logging.getLogger('migrate')
                 logger.warning(
@@ -243,176 +150,3 @@ class MergeContext(object):
         if target not in self.target_resources:
             target_ast = self.read_localization_ftl(target)
             self.target_resources[target] = target_ast
-
-    def get_legacy_source(self, path, key):
-        """Get an entity value from a localized legacy source.
-
-        Used by the `Source` transform.
-        """
-        resource = self.localization_resources[path]
-        return resource.get(key, None)
-
-    def get_fluent_source_pattern(self, path, key):
-        """Get a pattern from a localized Fluent source.
-
-        If the key contains a `.`, does an attribute lookup.
-        Used by the `COPY_PATTERN` transform.
-        """
-        resource = self.localization_resources[path]
-        msg_key, _, attr_key = key.partition('.')
-        found = None
-        for entry in resource.body:
-            if isinstance(entry, (FTL.Message, FTL.Term)):
-                if entry.id.name == msg_key:
-                    found = entry
-                    break
-        if found is None:
-            return None
-        if not attr_key:
-            return found.value
-        for attribute in found.attributes:
-            if attribute.id.name == attr_key:
-                return attribute.value
-        return None
-
-    def messages_equal(self, res1, res2):
-        """Compare messages and terms of two FTL resources.
-
-        Uses FTL.BaseNode.equals to compare all messages/terms
-        in two FTL resources.
-        If the order or number of messages differ, the result is also False.
-        """
-        def message_id(message):
-            "Return the message's identifer name for sorting purposes."
-            return message.id.name
-
-        messages1 = sorted(
-            (entry for entry in res1.body
-             if isinstance(entry, FTL.Message)
-                or isinstance(entry, FTL.Term)),
-            key=message_id)
-        messages2 = sorted(
-            (entry for entry in res2.body
-             if isinstance(entry, FTL.Message)
-                or isinstance(entry, FTL.Term)),
-            key=message_id)
-        for msg1, msg2 in zip_longest(messages1, messages2):
-            if msg1 is None or msg2 is None:
-                return False
-            if not msg1.equals(msg2):
-                return False
-        return True
-
-    def merge_changeset(self, changeset=None, known_translations=None):
-        """Return a generator of FTL ASTs for the changeset.
-
-        The input data must be configured earlier using the `add_*` methods.
-        if given, `changeset` must be a set of (path, key) tuples describing
-        which legacy translations are to be merged. If `changeset` is None,
-        all legacy translations will be allowed to be migrated in a single
-        changeset.
-
-        We use the `in_changeset` method to determine if a message should be
-        migrated for the given changeset.
-
-        Given `changeset`, return a dict whose keys are resource paths and
-        values are `FTL.Resource` instances.  The values will also be used to
-        update this context's existing localization resources.
-        """
-
-        if changeset is None:
-            # Merge all known legacy translations. Used in tests.
-            changeset = {
-                (path, key)
-                for path, strings in self.localization_resources.items()
-                if not path.endswith('.ftl')
-                for key in strings.keys()
-            }
-
-        if known_translations is None:
-            known_translations = changeset
-
-        for path, reference in self.reference_resources.items():
-            current = self.target_resources[path]
-            transforms = self.transforms.get(path, [])
-            in_changeset = partial(
-                self.in_changeset, changeset, known_translations, path)
-
-            # Merge legacy translations with the existing ones using the
-            # reference as a template.
-            snapshot = merge_resource(
-                self, reference, current, transforms, in_changeset
-            )
-
-            # Skip this path if the messages in the merged snapshot are
-            # identical to those in the current state of the localization file.
-            # This may happen when:
-            #
-            #   - none of the transforms is in the changset, or
-            #   - all messages which would be migrated by the context's
-            #     transforms already exist in the current state.
-            if self.messages_equal(current, snapshot):
-                continue
-
-            # Store the merged snapshot on the context so that the next merge
-            # already takes it into account as the existing localization.
-            self.target_resources[path] = snapshot
-
-            # The result for this path is a complete `FTL.Resource`.
-            yield path, snapshot
-
-    def in_changeset(self, changeset, known_translations, path, ident):
-        """Check if a message should be migrated in this changeset.
-
-        The message is identified by path and ident.
-
-
-        A message will be migrated only if all of its dependencies
-        are present in the currently processed changeset.
-
-        If a transform defined for this message points to a missing
-        legacy translation, this message will not be merged. The
-        missing legacy dependency won't be present in the changeset.
-
-        This also means that partially translated messages (e.g.
-        constructed from two legacy strings out of which only one is
-        avaiable) will never be migrated.
-        """
-        message_deps = self.dependencies.get((path, ident), None)
-
-        # Don't merge if we don't have a transform for this message.
-        if message_deps is None:
-            return False
-
-        # As a special case, if a transform exists but has no
-        # dependecies, it's a hardcoded `FTL.Node` which doesn't
-        # migrate any existing translation but rather creates a new
-        # one.  Merge it.
-        if len(message_deps) == 0:
-            return True
-
-        # Make sure all the dependencies are present in the current
-        # changeset. Partial migrations are not currently supported.
-        # See https://bugzilla.mozilla.org/show_bug.cgi?id=1321271
-        # We only return True if our current changeset touches
-        # the transform, and we have all of the dependencies.
-        active_deps = message_deps & changeset
-        available_deps = message_deps & known_translations
-        return active_deps and message_deps == available_deps
-
-    def serialize_changeset(self, changeset, known_translations=None):
-        """Return a dict of serialized FTLs for the changeset.
-
-        Given `changeset`, return a dict whose keys are resource paths and
-        values are serialized FTL snapshots.
-        """
-
-        return {
-            path: self.fluent_serializer.serialize(snapshot)
-            for path, snapshot in self.merge_changeset(
-                changeset, known_translations
-            )
-        }
-
-
-logging.basicConfig()

third_party/python/fluent.migrate/fluent/migrate/errors.py

@@ -1,3 +1,7 @@
+class SkipTransform(RuntimeError):
+    pass
+
+
 class MigrationError(ValueError):
     pass
 

third_party/python/fluent.migrate/fluent/migrate/helpers.py

@@ -6,7 +6,7 @@ nodes.
 
 They take a string argument and immediately return a corresponding AST node.
 (As opposed to Transforms which are AST nodes on their own and only return the
-migrated AST nodes when they are evaluated by a MergeContext.) """
+migrated AST nodes when they are evaluated by a MigrationContext.) """
 
 from __future__ import unicode_literals
 from __future__ import absolute_import

third_party/python/fluent.migrate/fluent/migrate/merge.py

@@ -4,6 +4,7 @@ from __future__ import absolute_import
 
 import fluent.syntax.ast as FTL
 
+from .errors import SkipTransform
 from .transforms import evaluate
 from .util import get_message, get_transform
 
@@ -50,7 +51,10 @@ def merge_resource(ctx, reference, current, transforms, in_changeset):
         if transform is not None and in_changeset(ident):
             if transform.comment is None:
                 transform.comment = entry.comment
-            return evaluate(ctx, transform)
+            try:
+                return evaluate(ctx, transform)
+            except SkipTransform:
+                return None
 
     body = merge_body(reference.body)
     return FTL.Resource(body)

third_party/python/fluent.migrate/fluent/migrate/tool.py

@@ -10,7 +10,7 @@ import sys
 import hglib
 import six
 
-from fluent.migrate.context import MergeContext
+from fluent.migrate.context import MigrationContext
 from fluent.migrate.errors import MigrationError
 from fluent.migrate.changesets import convert_blame_to_changesets
 from fluent.migrate.blame import Blame
@@ -25,8 +25,8 @@ def dont_write_bytecode():
 
 
 class Migrator(object):
-    def __init__(self, language, reference_dir, localization_dir, dry_run):
-        self.language = language
+    def __init__(self, locale, reference_dir, localization_dir, dry_run):
+        self.locale = locale
         self.reference_dir = reference_dir
         self.localization_dir = localization_dir
         self.dry_run = dry_run
@@ -45,11 +45,11 @@ class Migrator(object):
 
     def run(self, migration):
         print('\nRunning migration {} for {}'.format(
-            migration.__name__, self.language))
+            migration.__name__, self.locale))
 
         # For each migration create a new context.
-        ctx = MergeContext(
-            self.language, self.reference_dir, self.localization_dir
+        ctx = MigrationContext(
+            self.locale, self.reference_dir, self.localization_dir
         )
 
         try:
@@ -57,7 +57,7 @@ class Migrator(object):
             migration.migrate(ctx)
         except MigrationError as e:
             print('  Skipping migration {} for {}:\n    {}'.format(
-                migration.__name__, self.language, e))
+                migration.__name__, self.locale, e))
             return
 
         # Keep track of how many changesets we're committing.
@@ -125,9 +125,9 @@ class Migrator(object):
             print('    WARNING: hg commit failed ({})'.format(err))
 
 
-def main(lang, reference_dir, localization_dir, migrations, dry_run):
+def main(locale, reference_dir, localization_dir, migrations, dry_run):
     """Run migrations and commit files with the result."""
-    migrator = Migrator(lang, reference_dir, localization_dir, dry_run)
+    migrator = Migrator(locale, reference_dir, localization_dir, dry_run)
 
     for migration in migrations:
         migrator.run(migration)
@@ -144,8 +144,8 @@ def cli():
         help='migrations to run (Python modules)'
     )
     parser.add_argument(
-        '--lang', type=str,
-        help='target language code'
+        '--locale', '--lang', type=str,
+        help='target locale code (--lang is deprecated)'
     )
     parser.add_argument(
         '--reference-dir', type=str,
@@ -172,7 +172,7 @@ def cli():
         migrations = map(importlib.import_module, args.migrations)
 
     main(
-        lang=args.lang,
+        locale=args.locale,
         reference_dir=args.reference_dir,
         localization_dir=args.localization_dir,
         migrations=migrations,

third_party/python/fluent.migrate/fluent/migrate/transforms.py

@@ -3,7 +3,7 @@
 
 Transforms are AST nodes which describe how legacy translations should be
 migrated.  They are created inert and only return the migrated AST nodes when
-they are evaluated by a MergeContext.
+they are evaluated by a MigrationContext.
 
 All Transforms evaluate to Fluent Patterns. This makes them suitable for
 defining migrations of values of message, attributes and variants.  The special
@@ -13,7 +13,7 @@ elements: TextElements and Placeables.
 
 The COPY, REPLACE and PLURALS Transforms inherit from Source which is a special
 AST Node defining the location (the file path and the id) of the legacy
-translation.  During the migration, the current MergeContext scans the
+translation.  During the migration, the current MigrationContext scans the
 migration spec for Source nodes and extracts the information about all legacy
 translations being migrated. For instance,
 
@@ -238,6 +238,35 @@ class COPY_PATTERN(FluentSource):
     pass
 
 
+class TransformPattern(FluentSource, FTL.Transformer):
+    """Base class for modifying a Fluent pattern as part of a migration.
+
+    Implement visit_* methods of the Transformer pattern to do the
+    actual modifications.
+    """
+    def __call__(self, ctx):
+        pattern = super(TransformPattern, self).__call__(ctx)
+        return self.visit(pattern)
+
+    def visit_Pattern(self, node):
+        # Make sure we're creating valid Patterns after restructuring
+        # transforms.
+        node = self.generic_visit(node)
+        pattern = Transform.pattern_of(*node.elements)
+        return pattern
+
+    def visit_Placeable(self, node):
+        # Ensure we have a Placeable with an expression still.
+        # Transforms could have replaced the expression with
+        # a Pattern or PatternElement, in which case we
+        # just pass that through.
+        # Patterns then get flattened by visit_Pattern.
+        node = self.generic_visit(node)
+        if isinstance(node.expression, (FTL.Pattern, FTL.PatternElement)):
+            return node.expression
+        return node
+
+
 class LegacySource(Source):
     """Declare the source translation to be migrated with other transforms.
 
@@ -253,9 +282,12 @@ class LegacySource(Source):
     https://github.com/python/cpython/blob/2.7/Lib/htmlentitydefs.py
     https://github.com/python/cpython/blob/3.6/Lib/html/entities.py
 
+    By default, leading and trailing whitespace on each line as well as
+    leading and trailing empty lines will be stripped from the source
+    translation's content. Set `trim=False` to disable this behavior.
     """
 
-    def __init__(self, path, key, trim=False):
+    def __init__(self, path, key, trim=None):
         if path.endswith('.ftl'):
             raise NotSupportedError(
                 'Please use COPY_PATTERN to migrate from Fluent files '
@@ -269,9 +301,9 @@ class LegacySource(Source):
 
     @staticmethod
     def trim_text(text):
-        # strip leading white-space
+        # strip leading white-space from each line
         text = re.sub('^[ \t]+', '', text, flags=re.M)
-        # strip trailing white-space
+        # strip trailing white-space from each line
         text = re.sub('[ \t]+$', '', text, flags=re.M)
         # strip leading and trailing empty lines
         text = text.strip('\r\n')
@@ -279,7 +311,7 @@ class LegacySource(Source):
 
     def __call__(self, ctx):
         text = self.get_text(ctx)
-        if self.trim:
+        if self.trim is not False:
             text = self.trim_text(text)
         return FTL.TextElement(text)
 
@@ -496,7 +528,38 @@ class PLURALS(LegacySource):
 
 
 class CONCAT(Transform):
-    """Create a new Pattern from Patterns, PatternElements and Expressions."""
+    """Create a new Pattern from Patterns, PatternElements and Expressions.
+
+    When called with at least two elements, `CONCAT` disables the trimming
+    behavior of the elements which are subclasses of `LegacySource` by
+    setting `trim=False`, unless `trim` has already been set explicitly. The
+    following two `CONCAT` calls are equivalent:
+
+       CONCAT(
+           FTL.TextElement("Hello"),
+           COPY("file.properties", "hello")
+       )
+
+       CONCAT(
+           FTL.TextElement("Hello"),
+           COPY("file.properties", "hello", trim=False)
+       )
+
+    Set `trim=True` explicitly to force trimming:
+
+       CONCAT(
+           FTL.TextElement("Hello "),
+           COPY("file.properties", "hello", trim=True)
+       )
+
+    When called with a single element and when the element is a subclass of
+    `LegacySource`, the trimming behavior is not changed. The following two
+    transforms are equivalent:
+
+       CONCAT(COPY("file.properties", "hello"))
+
+       COPY("file.properties", "hello")
+    """
 
     def __init__(self, *elements, **kwargs):
         # We want to support both passing elements as *elements in the
@@ -505,5 +568,14 @@ class CONCAT(Transform):
         # attributes as kwargs.
         self.elements = list(kwargs.get('elements', elements))
 
+        # We want to make CONCAT(COPY()) equivalent to COPY() so that it's
+        # always safe (no-op) to wrap transforms in a CONCAT. This is used by
+        # the implementation of transforms_from.
+        if len(self.elements) > 1:
+            for elem in self.elements:
+                # Only change trim if it hasn't been set explicitly.
+                if isinstance(elem, LegacySource) and elem.trim is None:
+                    elem.trim = False
+
     def __call__(self, ctx):
         return Transform.pattern_of(*self.elements)

third_party/python/fluent.migrate/fluent/migrate/util.py

@@ -62,6 +62,18 @@ def get_transform(body, ident):
             return transform
 
 
+def skeleton(node):
+    """Create a skeleton copy of the given node.
+
+    For localizable entries, the value is None and the attributes are {}.
+    That's not a valid Fluent entry, so it requires further manipulation to
+    set values and/or attributes.
+    """
+    if isinstance(node, LOCALIZABLE_ENTRIES):
+        return type(node)(id=node.id.clone(), value=None)
+    return node.clone()
+
+
 def ftl(code):
     """Nicer indentation for FTL code.
 

third_party/python/fluent.migrate/fluent/migrate/validator.py

@@ -21,6 +21,20 @@ class BadContextAPIException(Exception):
     pass
 
 
+def process_assign(node, context):
+    if isinstance(node.value, ast.Str):
+        val = node.value.s
+    elif isinstance(node.value, ast.Name):
+        val = context.get(node.value.id)
+    elif isinstance(node.value, ast.Call):
+        val = node.value
+    if val is None:
+        return
+    for target in node.targets:
+        if isinstance(target, ast.Name):
+            context[target.id] = val
+
+
 class Validator(object):
     """Validate a migration recipe
 
@@ -55,16 +69,7 @@ class Validator(object):
                 migrate_func = top_level
                 details = self.inspect_migrate(migrate_func, global_assigns)
             if isinstance(top_level, ast.Assign):
-                val = None
-                if isinstance(top_level.value, ast.Str):
-                    val = top_level.value.s
-                elif isinstance(top_level.value, ast.Name):
-                    val = global_assigns.get(top_level.value.id)
-                if val is None:
-                    continue
-                for target in top_level.targets:
-                    if isinstance(target, ast.Name):
-                        global_assigns[target.id] = val
+                process_assign(top_level, global_assigns)
             if isinstance(top_level, (ast.Import, ast.ImportFrom)):
                 if 'module' in top_level._fields:
                     module = top_level.module
@@ -102,8 +107,7 @@ class Validator(object):
         visitor = MigrateAnalyzer(ctx_var, global_assigns)
         visitor.visit(migrate_func)
         return {
-            'sources': visitor.sources,
-            'references': visitor.targets,
+            'references': visitor.references,
             'issues': visitor.issues,
         }
 
@@ -118,6 +122,9 @@ def full_name(node, global_assigns):
     return '.'.join(reversed(leafs))
 
 
+PATH_TYPES = six.string_types + (ast.Call,)
+
+
 class MigrateAnalyzer(ast.NodeVisitor):
     def __init__(self, ctx_var, global_assigns):
         super(MigrateAnalyzer, self).__init__()
@@ -125,19 +132,23 @@ class MigrateAnalyzer(ast.NodeVisitor):
         self.global_assigns = global_assigns
         self.depth = 0
         self.issues = []
-        self.targets = set()
-        self.sources = set()
+        self.references = set()
 
     def generic_visit(self, node):
         self.depth += 1
         super(MigrateAnalyzer, self).generic_visit(node)
         self.depth -= 1
 
+    def visit_Assign(self, node):
+        if self.depth == 1:
+            process_assign(node, self.global_assigns)
+        self.generic_visit(node)
+
     def visit_Attribute(self, node):
         if isinstance(node.value, ast.Name) and node.value.id == self.ctx_var:
             if node.attr not in (
-                'maybe_add_localization',
                 'add_transforms',
+                'locale',
             ):
                 raise BadContextAPIException(
                     'Unexpected attribute access on {}.{}'.format(
@@ -161,8 +172,6 @@ class MigrateAnalyzer(ast.NodeVisitor):
         self.generic_visit(node)
 
     def call_ctx(self, node):
-        if node.func.attr == 'maybe_add_localization':
-            return self.call_maybe_add_localization(node)
         if node.func.attr == 'add_transforms':
             return self.call_add_transforms(node)
         raise BadContextAPIException(
@@ -171,71 +180,39 @@ class MigrateAnalyzer(ast.NodeVisitor):
             )
         )
 
-    def call_maybe_add_localization(self, node):
-        self.issues.append({
-            'msg': (
-                'Calling {}.maybe_add_localization is not required'
-            ).format(self.ctx_var),
-            'line': node.lineno
-        })
-        args_msg = (
-            'Expected arguments to {}.maybe_add_localization: '
-            'str'
-        ).format(self.ctx_var)
-        if not self.check_arguments(node, ((ast.Str, ast.Name),)):
-            raise BadContextAPIException(args_msg)
-        path = node.args[0]
-        if isinstance(path, ast.Str):
-            path = path.s
-        if isinstance(path, ast.Name):
-            path = self.global_assigns.get(path.id)
-        if not isinstance(path, six.string_types):
-            self.issues.append({
-                'msg': args_msg,
-                'line': node.args[0].lineno
-            })
-            return
-        if path != mozpath.normpath(path):
-            self.issues.append({
-                'msg': (
-                    'Argument to {}.maybe_add_localization needs to be a '
-                    'normalized path: "{}"'
-                ).format(self.ctx_var, path),
-                'line': node.args[0].lineno
-            })
-        else:
-            self.sources.add(path)
-
     def call_add_transforms(self, node):
         args_msg = (
             'Expected arguments to {}.add_transforms: '
-            'path, path, list'
+            'target_ftl_path, reference_ftl_path, list_of_transforms'
         ).format(self.ctx_var)
-        if not self.check_arguments(
-            node,
-            ((ast.Str, ast.Name), (ast.Str, ast.Name), (ast.List, ast.Call))
-        ):
+        ref_msg = (
+            'Expected second argument to {}.add_transforms: '
+            'reference should be string or variable with string value'
+        ).format(self.ctx_var)
+        # Just check call signature here, check actual types below
+        if not self.check_arguments(node, (ast.AST, ast.AST, ast.AST)):
             self.issues.append({
                 'msg': args_msg,
                 'line': node.lineno,
             })
             return
-        in_target, in_reference = [
-            n.s if isinstance(n, ast.Str) else self.global_assigns.get(n.id)
-            for n in node.args[:2]
-        ]
-        if (
-            isinstance(in_target, six.string_types) and
-            isinstance(in_reference, six.string_types) and
-            in_target == in_reference
-        ):
-            self.targets.add(in_target)
-        else:
+        in_reference = node.args[1]
+        if isinstance(in_reference, ast.Name):
+            in_reference = self.global_assigns.get(in_reference.id)
+        if isinstance(in_reference, ast.Str):
+            in_reference = in_reference.s
+        if not isinstance(in_reference, six.string_types):
             self.issues.append({
-                'msg': args_msg,
-                'line': node.lineno,
+                'msg': ref_msg,
+                'line': node.args[1].lineno,
             })
-        self.generic_visit(node)
+            return
+        self.references.add(in_reference)
+        # Checked node.args[1].
+        # There's not a lot we can say about our target path,
+        # ignoring that.
+        # For our transforms, we want more checks.
+        self.generic_visit(node.args[2])
 
     def call_transform(self, node, dotted):
         module, called = dotted.rsplit('.', 1)
@@ -259,13 +236,11 @@ class MigrateAnalyzer(ast.NodeVisitor):
             path = path.s
         if isinstance(path, ast.Name):
             path = self.global_assigns.get(path.id)
-        if not isinstance(path, six.string_types):
+        if not isinstance(path, PATH_TYPES):
             self.issues.append({
                 'msg': bad_args,
                 'line': node.lineno
             })
-            return
-        self.sources.add(path)
 
     def call_helpers_transforms_from(self, node):
         args_msg = (
@@ -288,7 +263,9 @@ class MigrateAnalyzer(ast.NodeVisitor):
                 v = v.s
             if isinstance(v, ast.Name):
                 v = self.global_assigns.get(v.id)
-            if not isinstance(v, six.string_types):
+            if isinstance(v, ast.Call):
+                v = 'determined at runtime'
+            if not isinstance(v, PATH_TYPES):
                 msg = 'Bad keyword arg {} to transforms_from'.format(
                     keyword.arg
                 )
@@ -315,7 +292,6 @@ class MigrateAnalyzer(ast.NodeVisitor):
             'msg': issue,
             'line': node.lineno,
         } for issue in set(ti.issues))
-        self.sources.update(ti.sources)
 
     def check_arguments(
         self, node, argspec, check_kwargs=True, allow_more=False
@@ -338,7 +314,6 @@ class MigrateAnalyzer(ast.NodeVisitor):
 class TransformsInspector(FTL.Visitor):
     def __init__(self):
         super(TransformsInspector, self).__init__()
-        self.sources = set()
         self.issues = []
 
     def generic_visit(self, node):
@@ -350,8 +325,6 @@ class TransformsInspector(FTL.Visitor):
                 self.issues.append(
                     'Source "{}" needs to be a normalized path'.format(src)
                 )
-            else:
-                self.sources.add(src)
         super(TransformsInspector, self).generic_visit(node)
 
 

third_party/python/fluent.migrate/setup.cfg

@@ -0,0 +1,16 @@
+[options.entry_points]
+console_scripts = 
+	migrate-l10n=fluent.migrate.tool:cli
+	validate-l10n-recipe=fluent.migrate.validator:cli
+
+[metadata]
+long_description = file: README.md
+long_description_content_type = text/markdown
+
+[bdist_wheel]
+universal = 1
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

third_party/python/fluent.migrate/setup.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+
+setup(
+    name='fluent.migrate',
+    version='0.9',
+    description='Toolchain to migrate legacy translation to Fluent.',
+    author='Mozilla',
+    author_email='l10n-drivers@mozilla.org',
+    license='APL 2',
+    url='https://hg.mozilla.org/l10n/fluent-migration/',
+    keywords=['fluent', 'localization', 'l10n'],
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: Apache Software License',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.7',
+    ],
+    packages=['fluent', 'fluent.migrate'],
+    install_requires=[
+        'compare-locales >=7.6, <8.1',
+        'fluent.syntax >=0.17.0, <0.18',
+        'six',
+    ],
+    extras_require={
+        'hg': ['python-hglib',],
+    },
+    tests_require=[
+        'mock',
+    ],
+    test_suite='tests.migrate'
+)

third_party/python/fluent.syntax/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: fluent.syntax
-Version: 0.15.1
+Version: 0.17.0
 Summary: Localization library for expressive translations.
 Home-page: https://github.com/projectfluent/python-fluent
 Author: Mozilla

third_party/python/fluent.syntax/fluent/syntax/errors.py

@@ -65,4 +65,6 @@ def get_error_message(code, args):
         return 'Unbalanced closing brace in TextElement.'
     if code == 'E0028':
         return 'Expected an inline expression'
+    if code == 'E0029':
+        return 'Expected simple expression as selector'
     return code

third_party/python/fluent.syntax/fluent/syntax/parser.py

@@ -508,11 +508,20 @@ class FluentParser(object):
                 else:
                     raise ParseError('E0018')
 
-            if (
+            elif (
                 isinstance(selector, ast.TermReference)
-                and selector.attribute is None
             ):
-                raise ParseError('E0017')
+                if selector.attribute is None:
+                    raise ParseError('E0017')
+            elif not (
+                isinstance(selector, (
+                    ast.StringLiteral,
+                    ast.NumberLiteral,
+                    ast.VariableReference,
+                    ast.FunctionReference,
+                ))
+            ):
+                raise ParseError('E0029')
 
             ps.next()
             ps.next()
@@ -555,17 +564,21 @@ class FluentParser(object):
                 ps.next()
                 attribute = self.get_identifier(ps)
             arguments = None
-            if ps.current_char == '(':
+            ps.peek_blank()
+            if ps.current_peek == '(':
+                ps.skip_to_peek()
                 arguments = self.get_call_arguments(ps)
             return ast.TermReference(id, attribute, arguments)
 
         if ps.is_identifier_start():
             id = self.get_identifier(ps)
+            ps.peek_blank()
 
-            if ps.current_char == '(':
+            if ps.current_peek == '(':
                 # It's a Function. Ensure it's all upper-case.
                 if not re.match('^[A-Z][A-Z0-9_-]*$', id.name):
                     raise ParseError('E0008')
+                ps.skip_to_peek()
                 args = self.get_call_arguments(ps)
                 return ast.FunctionReference(id, args)
 

third_party/python/fluent.syntax/setup.py

@@ -2,7 +2,7 @@
 from setuptools import setup
 
 setup(name='fluent.syntax',
-      version='0.15.1',
+      version='0.17.0',
       description='Localization library for expressive translations.',
       long_description='See https://github.com/projectfluent/python-fluent/ for more info.',
       author='Mozilla',

third_party/python/requirements.in

@@ -19,11 +19,12 @@
 attrs==18.1.0
 biplist==1.0.3
 blessings==1.7
-compare-locales==7.2.5
+compare-locales==8.0.0
 cookies==2.2.1
 distro==1.4.0
 ecdsa==0.15
 esprima==4.0.1
+fluent.migrate==0.9
 jsmin==2.1.0
 json-e==2.7.0
 mozilla-version==0.3.0

third_party/python/requirements.txt

@@ -22,10 +22,10 @@ click==7.0 \
     --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
     --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
     # via pip-tools
-compare-locales==7.2.5 \
-    --hash=sha256:9d332ac3cb389724b02d725f724b54393b39499cf70fefcd8267b15b1e5ec281 \
-    --hash=sha256:af035b6c2e53689a815e8a0d104a51359b2d45cf5109595348eda03e68ff9bec \
-    # via -r requirements-mach-vendor-python.in
+compare-locales==8.0.0 \
+    --hash=sha256:077b007bd2c025284f73994970e7fada7fbdcbb4199ff010e378b305dee6d469 \
+    --hash=sha256:ee02bdad012cdc9f6c6df24d7518ba2c5084f6bac0d176b4826156accc8d48d6 \
+    # via -r requirements-mach-vendor-python.in, fluent.migrate
 cookies==2.2.1 \
     --hash=sha256:15bee753002dff684987b8df8c235288eb8d45f8191ae056254812dfd42c81d3 \
     --hash=sha256:d6b698788cae4cfa4e62ef8643a9ca332b79bd96cb314294b864ae8d7eb3ee8e \
@@ -41,10 +41,14 @@ ecdsa==0.15 \
 esprima==4.0.1 \
     --hash=sha256:08db1a876d3c2910db9cfaeb83108193af5411fc3a3a66ebefacd390d21323ee \
     # via -r requirements-mach-vendor-python.in
-fluent.syntax==0.15.1 \
-    --hash=sha256:3d3c95b9de82df498172d9447576e787e809b753c6f7f5acf64a9ef8d7782c81 \
-    --hash=sha256:e95bb3abfe7cf51b2c6d1e92d57335f07c737ba57bbbba18e57618bd15f78d4b \
-    # via compare-locales
+fluent.migrate==0.9 \
+    --hash=sha256:735c86816ef7b7b03b32ff9985685f2d99cb0ed135351e4760a85236538f0beb \
+    --hash=sha256:d42a001bd7292cef400e63f3d77c0c813a6a6162e7bd2dfa14eb01172d21e788 \
+    # via -r requirements-mach-vendor-python.in
+fluent.syntax==0.17.0 \
+    --hash=sha256:ac3db2f77d62b032fdf1f17ef5c390b7801a9e9fb58d41eca3825c0d47b88d79 \
+    --hash=sha256:e26be470aeebe4badd84f7bb0b648414e0f2ef95d26e5336d634af99e402ea61 \
+    # via compare-locales, fluent.migrate
 jsmin==2.1.0 \
     --hash=sha256:5d07bf0251a4128e5e8e8eef603849b6b5741c337bff087731a248f9cc774f56 \
     # via -r requirements-mach-vendor-python.in