From 0b69680daaed2e2a5a06c2ff882a25f13377a381 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Thu, 14 May 2020 08:13:21 +0000 Subject: [PATCH] Bug 1637254, vendor l10n python libraries updates, r=stas This updates compare-locales to 8.0. The major version bump is due to the changes to the json output. This also updates fluent.syntax to 0.17. Vendor in fluent.migrate 0.9. This is the first formal vendoring, the version we had in-tree wasn't released on pypi before. Differential Revision: https://phabricator.services.mozilla.com/D75127 --- third_party/python/compare-locales/PKG-INFO | 2 +- .../compare_locales/__init__.py | 2 +- .../compare_locales/checks/base.py | 71 ++++ .../compare_locales/checks/dtd.py | 34 +- .../compare_locales/checks/fluent.py | 49 ++- .../compare_locales/checks/properties.py | 5 +- .../compare_locales/compare/content.py | 26 +- .../compare_locales/compare/observer.py | 31 +- .../integration_tests/test_plurals.py | 50 +-- .../compare_locales/parser/__init__.py | 8 + .../compare_locales/parser/po.py | 8 +- .../compare_locales/paths/configparser.py | 19 +- .../compare_locales/paths/files.py | 41 +- .../compare_locales/paths/matcher.py | 27 +- .../compare_locales/paths/project.py | 24 ++ .../compare_locales/plurals.py | 304 +++++++-------- .../tests/fluent/test_checks.py | 174 +++++++++ .../compare_locales/tests/paths/__init__.py | 88 +++-- .../tests/paths/test_configparser.py | 58 ++- .../compare_locales/tests/paths/test_files.py | 283 +++++++++++++- .../tests/paths/test_matcher.py | 60 +++ .../compare_locales/tests/po/test_parser.py | 2 +- .../compare_locales/tests/test_checks.py | 89 +++++ .../compare_locales/tests/test_compare.py | 24 +- .../compare_locales/tests/test_merge.py | 273 +++++++++++-- .../compare_locales/tests/test_parser.py | 29 ++ third_party/python/compare-locales/setup.py | 5 +- third_party/python/fluent.migrate/PKG-INFO | 62 +++ third_party/python/fluent.migrate/README.md | 44 +++ .../fluent.migrate/fluent/migrate/_context.py | 325 ++++++++++++++++ .../fluent/migrate/changesets.py | 2 +- .../fluent.migrate/fluent/migrate/context.py | 358 +++--------------- .../fluent.migrate/fluent/migrate/errors.py | 4 + .../fluent.migrate/fluent/migrate/helpers.py | 2 +- .../fluent.migrate/fluent/migrate/merge.py | 6 +- .../fluent.migrate/fluent/migrate/tool.py | 24 +- .../fluent/migrate/transforms.py | 86 ++++- .../fluent.migrate/fluent/migrate/util.py | 12 + .../fluent/migrate/validator.py | 131 +++---- third_party/python/fluent.migrate/setup.cfg | 16 + third_party/python/fluent.migrate/setup.py | 34 ++ third_party/python/fluent.syntax/PKG-INFO | 2 +- .../fluent.syntax/fluent/syntax/errors.py | 2 + .../fluent.syntax/fluent/syntax/parser.py | 23 +- third_party/python/fluent.syntax/setup.py | 2 +- third_party/python/requirements.in | 3 +- third_party/python/requirements.txt | 20 +- 47 files changed, 2180 insertions(+), 764 deletions(-) create mode 100644 third_party/python/compare-locales/compare_locales/tests/test_checks.py create mode 100644 third_party/python/fluent.migrate/PKG-INFO create mode 100644 third_party/python/fluent.migrate/README.md create mode 100644 third_party/python/fluent.migrate/fluent/migrate/_context.py mode change 100644 => 100755 third_party/python/fluent.migrate/fluent/migrate/tool.py create mode 100644 third_party/python/fluent.migrate/setup.cfg create mode 100644 third_party/python/fluent.migrate/setup.py diff --git a/third_party/python/compare-locales/PKG-INFO b/third_party/python/compare-locales/PKG-INFO index 3a8ea6db883e..5aab8982b238 100644 --- a/third_party/python/compare-locales/PKG-INFO +++ b/third_party/python/compare-locales/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: compare-locales -Version: 7.2.5 +Version: 8.0.0 Summary: Lint Mozilla localizations Home-page: UNKNOWN Author: Axel Hecht diff --git a/third_party/python/compare-locales/compare_locales/__init__.py b/third_party/python/compare-locales/compare_locales/__init__.py index d72e34593ee5..890e24462334 100644 --- a/third_party/python/compare-locales/compare_locales/__init__.py +++ b/third_party/python/compare-locales/compare_locales/__init__.py @@ -1 +1 @@ -version = "7.2.5" +version = "8.0.0" diff --git a/third_party/python/compare-locales/compare_locales/checks/base.py b/third_party/python/compare-locales/compare_locales/checks/base.py index 7ec01ed87ab5..3b04caa7a90c 100644 --- a/third_party/python/compare-locales/compare_locales/checks/base.py +++ b/third_party/python/compare-locales/compare_locales/checks/base.py @@ -6,6 +6,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import re +import six class EntityPos(int): @@ -54,3 +55,73 @@ class Checker(object): Only do this if self.needs_reference is True. ''' self.reference = reference + + +class CSSCheckMixin(object): + def maybe_style(self, ref_value, l10n_value): + ref_map, _ = self.parse_css_spec(ref_value) + if not ref_map: + return + l10n_map, errors = self.parse_css_spec(l10n_value) + for t in self.check_style(ref_map, l10n_map, errors): + yield t + + def check_style(self, ref_map, l10n_map, errors): + if not l10n_map: + yield ('error', 0, 'reference is a CSS spec', 'css') + return + if errors: + yield ('error', 0, 'reference is a CSS spec', 'css') + return + msgs = [] + for prop, unit in l10n_map.items(): + if prop not in ref_map: + msgs.insert(0, '%s only in l10n' % prop) + continue + else: + ref_unit = ref_map.pop(prop) + if unit != ref_unit: + msgs.append("units for %s don't match " + "(%s != %s)" % (prop, unit, ref_unit)) + for prop in six.iterkeys(ref_map): + msgs.insert(0, '%s only in reference' % prop) + if msgs: + yield ('warning', 0, ', '.join(msgs), 'css') + + def parse_css_spec(self, val): + if not hasattr(self, '_css_spec'): + self._css_spec = re.compile( + r'(?:' + r'(?P(?:min\-|max\-)?(?:width|height))' + r'[ \t\r\n]*:[ \t\r\n]*' + r'(?P[0-9]+|[0-9]*\.[0-9]+)' + r'(?Pch|em|ex|rem|px|cm|mm|in|pc|pt)' + r')' + r'|\Z' + ) + self._css_sep = re.compile(r'[ \t\r\n]*(?P;)?[ \t\r\n]*$') + refMap = errors = None + end = 0 + for m in self._css_spec.finditer(val): + if end == 0 and m.start() == m.end(): + # no CSS spec found, just immediately end of string + return None, None + if m.start() > end: + split = self._css_sep.match(val, end, m.start()) + if split is None: + errors = errors or [] + errors.append({ + 'pos': end, + 'code': 'css-bad-content', + }) + elif end > 0 and split.group('semi') is None: + errors = errors or [] + errors.append({ + 'pos': end, + 'code': 'css-missing-semicolon', + }) + if m.group('prop'): + refMap = refMap or {} + refMap[m.group('prop')] = m.group('unit') + end = m.end() + return refMap, errors diff --git a/third_party/python/compare-locales/compare_locales/checks/dtd.py b/third_party/python/compare-locales/compare_locales/checks/dtd.py index 702f0b34617a..37d3c7846d2a 100644 --- a/third_party/python/compare-locales/compare_locales/checks/dtd.py +++ b/third_party/python/compare-locales/compare_locales/checks/dtd.py @@ -9,10 +9,10 @@ from xml import sax import six from compare_locales.parser import DTDParser -from .base import Checker +from .base import Checker, CSSCheckMixin -class DTDChecker(Checker): +class DTDChecker(Checker, CSSCheckMixin): """Tests to run on DTD files. Uses xml.sax for the heavy lifting of xml parsing. @@ -68,11 +68,6 @@ class DTDChecker(Checker): num = re.compile('^%s$' % numPattern) lengthPattern = '%s(em|px|ch|cm|in)' % numPattern length = re.compile('^%s$' % lengthPattern) - spec = re.compile(r'((?:min\-)?(?:width|height))[ \t\r\n]*:[ \t\r\n]*%s' % - lengthPattern) - style = re.compile( - r'^%(spec)s[ \t\r\n]*(;[ \t\r\n]*%(spec)s[ \t\r\n]*)*;?$' % - {'spec': spec.pattern}) def check(self, refEnt, l10nEnt): """Try to parse the refvalue inside a dummy element, and keep @@ -177,28 +172,9 @@ class DTDChecker(Checker): # just a length, width="100em" if self.length.match(refValue) and not self.length.match(l10nValue): yield ('error', 0, 'reference is a CSS length', 'css') - # real CSS spec, style="width:100px;" - if self.style.match(refValue): - if not self.style.match(l10nValue): - yield ('error', 0, 'reference is a CSS spec', 'css') - else: - # warn if different properties or units - refMap = dict((s, u) for s, _, u in - self.spec.findall(refValue)) - msgs = [] - for s, _, u in self.spec.findall(l10nValue): - if s not in refMap: - msgs.insert(0, '%s only in l10n' % s) - continue - else: - ru = refMap.pop(s) - if u != ru: - msgs.append("units for %s don't match " - "(%s != %s)" % (s, u, ru)) - for s in six.iterkeys(refMap): - msgs.insert(0, '%s only in reference' % s) - if msgs: - yield ('warning', 0, ', '.join(msgs), 'css') + # Check for actual CSS style attribute values + for t in self.maybe_style(refValue, l10nValue): + yield t if self.extra_tests is not None and 'android-dtd' in self.extra_tests: for t in self.processAndroidContent(self.texthandler.textcontent): diff --git a/third_party/python/compare-locales/compare_locales/checks/fluent.py b/third_party/python/compare-locales/compare_locales/checks/fluent.py index 9081c9831194..1dbe7ca57302 100644 --- a/third_party/python/compare-locales/compare_locales/checks/fluent.py +++ b/third_party/python/compare-locales/compare_locales/checks/fluent.py @@ -10,7 +10,7 @@ from collections import defaultdict from fluent.syntax import ast as ftl from fluent.syntax.serializer import serialize_variant_key -from .base import Checker +from .base import Checker, CSSCheckMixin from compare_locales import plurals @@ -26,10 +26,26 @@ MSGS = { 'obsolete-attribute': 'Obsolete attribute: {name}', 'duplicate-variant': 'Variant key "{name}" is duplicated', 'missing-plural': 'Plural categories missing: {categories}', + 'plain-message': '{message}', } -class ReferenceMessageVisitor(ftl.Visitor): +def pattern_variants(pattern): + """Get variants of plain text of a pattern. + + For now, just return simple text patterns. + This can be improved to allow for SelectExpressions + of simple text patterns, or even nested expressions, and Literals. + Variants with Variable-, Message-, or TermReferences should be ignored. + """ + elements = pattern.elements + if len(elements) == 1: + if isinstance(elements[0], ftl.TextElement): + return [elements[0].value] + return [] + + +class ReferenceMessageVisitor(ftl.Visitor, CSSCheckMixin): def __init__(self): # References to Messages, their Attributes, and Terms # Store reference name and type @@ -42,6 +58,9 @@ class ReferenceMessageVisitor(ftl.Visitor): self.message_has_value = False # Map attribute names to positions self.attribute_positions = {} + # Map of CSS style attribute properties and units + self.css_styles = None + self.css_errors = None def generic_visit(self, node): if isinstance( @@ -62,6 +81,14 @@ class ReferenceMessageVisitor(ftl.Visitor): self.refs = self.entry_refs[node.id.name] super(ReferenceMessageVisitor, self).generic_visit(node) self.refs = old_refs + if node.id.name != 'style': + return + text_values = pattern_variants(node.value) + if not text_values: + self.css_styles = 'skip' + return + # right now, there's just one possible text value + self.css_styles, self.css_errors = self.parse_css_spec(text_values[0]) def visit_SelectExpression(self, node): # optimize select expressions to only go through the variants @@ -142,8 +169,9 @@ class GenericL10nChecks(object): ) ) # Check for plural categories - if self.locale in plurals.CATEGORIES_BY_LOCALE: - known_plurals = set(plurals.CATEGORIES_BY_LOCALE[self.locale]) + known_plurals = plurals.get_plural(self.locale) + if known_plurals: + known_plurals = set(known_plurals) # Ask for known plurals, but check for plurals w/out `other`. # `other` is used for all kinds of things. check_plurals = known_plurals.copy() @@ -214,6 +242,19 @@ class L10nMessageVisitor(GenericL10nChecks, ReferenceMessageVisitor): self.reference_refs = self.reference.entry_refs[node.id.name] super(L10nMessageVisitor, self).visit_Attribute(node) self.reference_refs = old_reference_refs + if node.id.name != 'style' or self.css_styles == 'skip': + return + ref_styles = self.reference.css_styles + if ref_styles in ('skip', None): + # Reference is complex, l10n isn't. + # Let's still validate the css spec. + ref_styles = {} + for cat, msg, pos, _ in self.check_style( + ref_styles, + self.css_styles, + self.css_errors + ): + self.messages.append((cat, msg, pos)) def visit_SelectExpression(self, node): super(L10nMessageVisitor, self).visit_SelectExpression(node) diff --git a/third_party/python/compare-locales/compare_locales/checks/properties.py b/third_party/python/compare-locales/compare_locales/checks/properties.py index 37b4691f19f7..9ff2e4cdae16 100644 --- a/third_party/python/compare-locales/compare_locales/checks/properties.py +++ b/third_party/python/compare-locales/compare_locales/checks/properties.py @@ -71,8 +71,9 @@ class PropertiesChecker(Checker): '''Check for the stringbundle plurals logic. The common variable pattern is #1. ''' - if self.locale in plurals.CATEGORIES_BY_LOCALE: - expected_forms = len(plurals.CATEGORIES_BY_LOCALE[self.locale]) + known_plurals = plurals.get_plural(self.locale) + if known_plurals: + expected_forms = len(known_plurals) found_forms = l10nValue.count(';') + 1 msg = 'expecting {} plurals, found {}'.format( expected_forms, diff --git a/third_party/python/compare-locales/compare_locales/compare/content.py b/third_party/python/compare-locales/compare_locales/compare/content.py index 186f80783883..03ba222d8ec5 100644 --- a/third_party/python/compare-locales/compare_locales/compare/content.py +++ b/third_party/python/compare-locales/compare_locales/compare/content.py @@ -244,19 +244,17 @@ class ContentComparer: l10n, merge_file, missings, skips, l10n_ctx, p.capabilities, p.encoding) - stats = {} - for cat, value in ( - ('missing', missing), - ('missing_w', missing_w), - ('report', report), - ('obsolete', obsolete), - ('changed', changed), - ('changed_w', changed_w), - ('unchanged', unchanged), - ('unchanged_w', unchanged_w), - ('keys', keys)): - if value: - stats[cat] = value + stats = { + 'missing': missing, + 'missing_w': missing_w, + 'report': report, + 'obsolete': obsolete, + 'changed': changed, + 'changed_w': changed_w, + 'unchanged': unchanged, + 'unchanged_w': unchanged_w, + 'keys': keys, + } self.observers.updateStats(l10n, stats) pass @@ -294,7 +292,7 @@ class ContentComparer: return # strip parse errors entities = [e for e in entities if not isinstance(e, parser.Junk)] - self.observers.updateStats(missing, {'missingInFiles': len(entities)}) + self.observers.updateStats(missing, {'missing': len(entities)}) missing_w = 0 for e in entities: missing_w += e.count_words() diff --git a/third_party/python/compare-locales/compare_locales/compare/observer.py b/third_party/python/compare-locales/compare_locales/compare/observer.py index 0a691d5a7cc6..7301d9a35690 100644 --- a/third_party/python/compare-locales/compare_locales/compare/observer.py +++ b/third_party/python/compare-locales/compare_locales/compare/observer.py @@ -20,7 +20,19 @@ class Observer(object): for quiet=2, skip missing and obsolete files. For quiet=3, skip warnings and errors. ''' - self.summary = defaultdict(lambda: defaultdict(int)) + self.summary = defaultdict(lambda: { + "errors": 0, + "warnings": 0, + "missing": 0, + "missing_w": 0, + "report": 0, + "obsolete": 0, + "changed": 0, + "changed_w": 0, + "unchanged": 0, + "unchanged_w": 0, + "keys": 0, + }) self.details = Tree(list) self.quiet = quiet self.filter = filter @@ -164,20 +176,12 @@ class ObserverList(Observer): if len(self.observers) > 1: # add ourselves if there's more than one project for loc, lst in summaries.items(): - lst.append(self.summary.get(loc, {})) - # normalize missing and missingInFiles -> missing - for summarylist in summaries.values(): - for summary in summarylist: - if 'missingInFiles' in summary: - summary['missing'] = ( - summary.get('missing', 0) - + summary.pop('missingInFiles') - ) + lst.append(self.summary[loc]) keys = ( 'errors', 'warnings', 'missing', 'missing_w', - 'obsolete', 'obsolete_w', + 'obsolete', 'changed', 'changed_w', 'unchanged', 'unchanged_w', 'keys', @@ -192,7 +196,7 @@ class ObserverList(Observer): segment = [''] * len(keys) for summary in summaries: for row, key in enumerate(keys): - segment[row] += ' {:6}'.format(summary.get(key, '')) + segment[row] += ' {:6}'.format(summary.get(key) or '') out += [ lead + row @@ -201,8 +205,7 @@ class ObserverList(Observer): ] total = sum([summaries[-1].get(k, 0) - for k in ['changed', 'unchanged', 'report', 'missing', - 'missingInFiles'] + for k in ['changed', 'unchanged', 'report', 'missing'] ]) rate = 0 if total: diff --git a/third_party/python/compare-locales/compare_locales/integration_tests/test_plurals.py b/third_party/python/compare-locales/compare_locales/integration_tests/test_plurals.py index 72b69c06d1a2..b36c41222be9 100644 --- a/third_party/python/compare-locales/compare_locales/integration_tests/test_plurals.py +++ b/third_party/python/compare-locales/compare_locales/integration_tests/test_plurals.py @@ -3,14 +3,13 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -import ast import json -import os import unittest -import six from six.moves.urllib.error import URLError from six.moves.urllib.request import urlopen +from compare_locales import plurals + TRANSVISION_URL = ( 'https://transvision.mozfr.org/' @@ -30,24 +29,16 @@ class TestPlural(unittest.TestCase): def test_valid_forms(self): reference_form_map = self._load_transvision() - compare_locales_map = self._parse_plurals_py() - # Notify locales in compare-locales but not in Transvision - # Might be incubator locales - extra_locales = set() - extra_locales.update(compare_locales_map) - extra_locales.difference_update(reference_form_map) - for locale in sorted(extra_locales): - print("{} not in Transvision, OK".format(locale)) - compare_locales_map.pop(locale) # Strip matches from dicts, to make diff for test small - locales = set() - locales.update(compare_locales_map) - locales.intersection_update(reference_form_map) + locales = list(reference_form_map) + cl_form_map = {} for locale in locales: - if compare_locales_map[locale] == reference_form_map[locale]: - compare_locales_map.pop(locale) + cl_form = str(plurals.get_plural_rule(locale)) + if cl_form == reference_form_map[locale]: reference_form_map.pop(locale) - self.assertDictEqual(reference_form_map, compare_locales_map) + else: + cl_form_map[locale] = cl_form + self.assertDictEqual(reference_form_map, cl_form_map) def _load_transvision(self): '''Use the Transvision API to load all values of pluralRule @@ -59,26 +50,3 @@ class TestPlural(unittest.TestCase): except URLError: raise unittest.SkipTest("Couldn't load Transvision API.") return json.loads(data) - - def _parse_plurals_py(self): - '''Load compare_locales.plurals, parse the AST, and inspect - the dictionary assigned to CATEGORIES_BY_LOCALE to find - the actual plural number. - Convert both number and locale code to unicode for comparing - to json. - ''' - path = os.path.join(os.path.dirname(__file__), '..', 'plurals.py') - with open(path) as source_file: - plurals_ast = ast.parse(source_file.read()) - assign_cats_statement = [ - s for s in plurals_ast.body - if isinstance(s, ast.Assign) - and any(t.id == 'CATEGORIES_BY_LOCALE' for t in s.targets) - ][0] - return dict( - (six.text_type(k.s), six.text_type(v.slice.value.n)) - for k, v in zip( - assign_cats_statement.value.keys, - assign_cats_statement.value.values - ) - ) diff --git a/third_party/python/compare-locales/compare_locales/parser/__init__.py b/third_party/python/compare-locales/compare_locales/parser/__init__.py index 64081e5bc26c..8ab36cb0827d 100644 --- a/third_party/python/compare-locales/compare_locales/parser/__init__.py +++ b/third_party/python/compare-locales/compare_locales/parser/__init__.py @@ -54,6 +54,14 @@ def getParser(path): for item in __constructors: if re.search(item[0], path): return item[1] + try: + from pkg_resources import iter_entry_points + for entry_point in iter_entry_points('compare_locales.parsers'): + p = entry_point.resolve()() + if p.use(path): + return p + except (ImportError, IOError): + pass raise UserWarning("Cannot find Parser") diff --git a/third_party/python/compare-locales/compare_locales/parser/po.py b/third_party/python/compare-locales/compare_locales/parser/po.py index 305f0ece7ac8..5880cf7c71c9 100644 --- a/third_party/python/compare-locales/compare_locales/parser/po.py +++ b/third_party/python/compare-locales/compare_locales/parser/po.py @@ -24,7 +24,11 @@ class PoEntityMixin(object): @property def val(self): - return self.stringlist_val + return ( + self.stringlist_val + if self.stringlist_val + else self.stringlist_key[0] + ) @property def key(self): @@ -33,7 +37,7 @@ class PoEntityMixin(object): @property def localized(self): # gettext denotes a non-localized string by an empty value - return bool(self.val) + return bool(self.stringlist_val) def __repr__(self): return self.key[0] diff --git a/third_party/python/compare-locales/compare_locales/paths/configparser.py b/third_party/python/compare-locales/compare_locales/paths/configparser.py index 6ade581ab056..ce56df10b736 100644 --- a/third_party/python/compare-locales/compare_locales/paths/configparser.py +++ b/third_party/python/compare-locales/compare_locales/paths/configparser.py @@ -40,6 +40,7 @@ class TOMLParser(object): self.processPaths(ctx) self.processFilters(ctx) self.processIncludes(ctx) + self.processExcludes(ctx) self.processLocales(ctx) return self.asConfig(ctx) @@ -101,15 +102,23 @@ class TOMLParser(object): ctx.pc.add_rules(rule) def processIncludes(self, ctx): + for child in self._processChild(ctx, 'includes'): + ctx.pc.add_child(child) + + def processExcludes(self, ctx): + for child in self._processChild(ctx, 'excludes'): + ctx.pc.exclude(child) + + def _processChild(self, ctx, field): assert ctx.data is not None - if 'includes' not in ctx.data: + if field not in ctx.data: return - for include in ctx.data['includes']: - # resolve include['path'] against our root and env + for child_config in ctx.data[field]: + # resolve child_config['path'] against our root and env p = mozpath.normpath( expand( ctx.pc.root, - include['path'], + child_config['path'], ctx.pc.environ ) ) @@ -125,7 +134,7 @@ class TOMLParser(object): .getLogger('compare-locales.io') .error('%s: %s', e.strerror, e.filename)) continue - ctx.pc.add_child(child) + yield child def asConfig(self, ctx): return ctx.pc diff --git a/third_party/python/compare-locales/compare_locales/paths/files.py b/third_party/python/compare-locales/compare_locales/paths/files.py index 87d255872c2a..b7ec21b9f530 100644 --- a/third_party/python/compare-locales/compare_locales/paths/files.py +++ b/third_party/python/compare-locales/compare_locales/paths/files.py @@ -10,6 +10,16 @@ from compare_locales import mozpath REFERENCE_LOCALE = 'en-x-moz-reference' +class ConfigList(list): + def maybe_extend(self, other): + '''Add configs from other list if this list doesn't have this path yet. + ''' + for config in other: + if any(mine.path == config.path for mine in self): + continue + self.append(config) + + class ProjectFiles(object): '''Iterable object to get all files and tests for a locale and a list of ProjectConfigs. @@ -20,14 +30,26 @@ class ProjectFiles(object): def __init__(self, locale, projects, mergebase=None): self.locale = locale self.matchers = [] + self.exclude = None self.mergebase = mergebase - configs = [] + configs = ConfigList() + excludes = ConfigList() for project in projects: # Only add this project if we're not in validation mode, # and the given locale is enabled for the project. if locale is not None and locale not in project.all_locales: continue - configs.extend(project.configs) + configs.maybe_extend(project.configs) + excludes.maybe_extend(project.excludes) + # If an excluded config is explicitly included, drop if from the + # excludes. + excludes = [ + exclude + for exclude in excludes + if not any(c.path == exclude.path for c in configs) + ] + if excludes: + self.exclude = ProjectFiles(locale, excludes) for pc in configs: if locale and pc.locales is not None and locale not in pc.locales: continue @@ -123,6 +145,9 @@ class ProjectFiles(object): def iter_reference(self): '''Iterate over reference files.''' + # unset self.exclude, as we don't want that for our reference files + exclude = self.exclude + self.exclude = None known = {} for matchers in self.matchers: if 'reference' not in matchers: @@ -137,6 +162,7 @@ class ProjectFiles(object): } for path, d in sorted(known.items()): yield (path, d.get('reference'), None, d['test']) + self.exclude = exclude def _files(self, matcher): '''Base implementation of getting all files in a hierarchy @@ -146,12 +172,16 @@ class ProjectFiles(object): ''' base = matcher.prefix if self._isfile(base): + if self.exclude and self.exclude.match(base) is not None: + return if matcher.match(base) is not None: yield base return for d, dirs, files in self._walk(base): for f in files: p = mozpath.join(d, f) + if self.exclude and self.exclude.match(p) is not None: + continue if matcher.match(p) is not None: yield p @@ -168,9 +198,14 @@ class ProjectFiles(object): This routine doesn't check that the files actually exist. ''' + if ( + self.locale is not None and + self.exclude and self.exclude.match(path) is not None + ): + return for matchers in self.matchers: matcher = matchers['l10n'] - if matcher.match(path) is not None: + if self.locale is not None and matcher.match(path) is not None: ref = merge = None if 'reference' in matchers: ref = matcher.sub(matchers['reference'], path) diff --git a/third_party/python/compare-locales/compare_locales/paths/matcher.py b/third_party/python/compare-locales/compare_locales/paths/matcher.py index 0077eea8586b..554d167686cf 100644 --- a/third_party/python/compare-locales/compare_locales/paths/matcher.py +++ b/third_party/python/compare-locales/compare_locales/paths/matcher.py @@ -33,7 +33,7 @@ class Matcher(object): uses that to transform l10n and en-US paths back and forth. ''' - def __init__(self, pattern_or_other, env={}, root=None): + def __init__(self, pattern_or_other, env={}, root=None, encoding=None): '''Create regular expression similar to mozpath.match(). ''' parser = PatternParser() @@ -50,12 +50,14 @@ class Matcher(object): self.env.update(real_env) if root is not None: self.pattern.root = root + self.encoding = other.encoding return self.env = real_env pattern = pattern_or_other self.pattern = parser.parse(pattern) if root is not None: self.pattern.root = root + self.encoding = encoding def with_env(self, environ): return Matcher(self, environ) @@ -64,7 +66,10 @@ class Matcher(object): def prefix(self): subpattern = Pattern(self.pattern[:self.pattern.prefix_length]) subpattern.root = self.pattern.root - return subpattern.expand(self.env) + prefix = subpattern.expand(self.env) + if self.encoding is not None: + prefix = prefix.encode(self.encoding) + return prefix def match(self, path): '''Test the given path against this matcher and its environment. @@ -77,6 +82,8 @@ class Matcher(object): if m is None: return None d = m.groupdict() + if self.encoding is not None: + d = {key: value.decode(self.encoding) for key, value in d.items()} if 'android_locale' in d and 'locale' not in d: # map android_locale to locale code locale = d['android_locale'] @@ -94,9 +101,10 @@ class Matcher(object): def _cache_regex(self): if self._cached_re is not None: return - self._cached_re = re.compile( - self.pattern.regex_pattern(self.env) + '$' - ) + pattern = self.pattern.regex_pattern(self.env) + '$' + if self.encoding is not None: + pattern = pattern.encode(self.encoding) + self._cached_re = re.compile(pattern) def sub(self, other, path): ''' @@ -108,11 +116,14 @@ class Matcher(object): return None env = {} env.update( - (key, Literal(value)) + (key, Literal(value if value is not None else '')) for key, value in m.items() ) env.update(other.env) - return other.pattern.expand(env) + path = other.pattern.expand(env) + if self.encoding is not None: + path = path.encode(self.encoding) + return path def concat(self, other): '''Concat two Matcher objects. @@ -164,6 +175,8 @@ class Matcher(object): continue if self.env[k] != other.env[k]: return False + if self.encoding != other.encoding: + return False return True diff --git a/third_party/python/compare-locales/compare_locales/paths/project.py b/third_party/python/compare-locales/compare_locales/paths/project.py index 0eab0f1bb6bd..269b6fed9dc4 100644 --- a/third_party/python/compare-locales/compare_locales/paths/project.py +++ b/third_party/python/compare-locales/compare_locales/paths/project.py @@ -9,6 +9,10 @@ from .matcher import Matcher import six +class ExcludeError(ValueError): + pass + + class ProjectConfig(object): '''Abstraction of l10n project configuration data. ''' @@ -30,6 +34,7 @@ class ProjectConfig(object): self._all_locales = None self.environ = {} self.children = [] + self.excludes = [] self._cache = None def same(self, other): @@ -112,8 +117,20 @@ class ProjectConfig(object): def add_child(self, child): self._all_locales = None # clear cache + if child.excludes: + raise ExcludeError( + 'Included configs cannot declare their own excludes.' + ) self.children.append(child) + def exclude(self, child): + for config in child.configs: + if config.excludes: + raise ExcludeError( + 'Excluded configs cannot declare their own excludes.' + ) + self.excludes.append(child) + def set_locales(self, locales, deep=False): self._all_locales = None # clear cache self.locales = locales @@ -168,6 +185,8 @@ class ProjectConfig(object): return self._cache self._cache = self.FilterCache(locale) for paths in self.paths: + if 'locales' in paths and locale not in paths['locales']: + continue self._cache.l10n_paths.append(paths['l10n'].with_env({ "locale": locale })) @@ -180,6 +199,11 @@ class ProjectConfig(object): return self._cache def _filter(self, l10n_file, entity=None): + if any( + exclude.filter(l10n_file) == 'error' + for exclude in self.excludes + ): + return actions = set( child._filter(l10n_file, entity=entity) for child in self.children) diff --git a/third_party/python/compare-locales/compare_locales/plurals.py b/third_party/python/compare-locales/compare_locales/plurals.py index 65bc015d0a2b..f54d1dafac8f 100644 --- a/third_party/python/compare-locales/compare_locales/plurals.py +++ b/third_party/python/compare-locales/compare_locales/plurals.py @@ -58,152 +58,160 @@ CATEGORIES_EXCEPTIONS = { } CATEGORIES_BY_LOCALE = { - 'ace': CATEGORIES_BY_INDEX[0], - 'ach': CATEGORIES_BY_INDEX[1], - 'af': CATEGORIES_BY_INDEX[1], - 'ak': CATEGORIES_BY_INDEX[2], - 'an': CATEGORIES_BY_INDEX[1], - 'ar': CATEGORIES_BY_INDEX[12], - 'arn': CATEGORIES_BY_INDEX[1], - 'as': CATEGORIES_BY_INDEX[1], - 'ast': CATEGORIES_BY_INDEX[1], - 'az': CATEGORIES_BY_INDEX[1], - 'be': CATEGORIES_BY_INDEX[7], - 'bg': CATEGORIES_BY_INDEX[1], - 'bn': CATEGORIES_BY_INDEX[2], - 'bn-BD': CATEGORIES_BY_INDEX[2], - 'bn-IN': CATEGORIES_BY_INDEX[2], - 'br': CATEGORIES_BY_INDEX[16], - 'brx': CATEGORIES_BY_INDEX[1], - 'bs': CATEGORIES_BY_INDEX[19], - 'ca': CATEGORIES_BY_INDEX[1], - 'cak': CATEGORIES_BY_INDEX[1], - 'crh': CATEGORIES_BY_INDEX[1], - 'cs': CATEGORIES_BY_INDEX[8], - 'csb': CATEGORIES_BY_INDEX[9], - 'cv': CATEGORIES_BY_INDEX[1], - 'cy': CATEGORIES_BY_INDEX[18], - 'da': CATEGORIES_BY_INDEX[1], - 'de': CATEGORIES_BY_INDEX[1], - 'dsb': CATEGORIES_BY_INDEX[10], - 'el': CATEGORIES_BY_INDEX[1], - 'en-CA': CATEGORIES_BY_INDEX[1], - 'en-GB': CATEGORIES_BY_INDEX[1], - 'en-US': CATEGORIES_BY_INDEX[1], - 'en-ZA': CATEGORIES_BY_INDEX[1], - 'en-x-moz-reference': CATEGORIES_BY_INDEX[1], # for reference validation - 'eo': CATEGORIES_BY_INDEX[1], - 'es-AR': CATEGORIES_BY_INDEX[1], - 'es-CL': CATEGORIES_BY_INDEX[1], - 'es-ES': CATEGORIES_BY_INDEX[1], - 'es-MX': CATEGORIES_BY_INDEX[1], - 'et': CATEGORIES_BY_INDEX[1], - 'eu': CATEGORIES_BY_INDEX[1], - 'fa': CATEGORIES_BY_INDEX[2], - 'ff': CATEGORIES_BY_INDEX[1], - 'fi': CATEGORIES_BY_INDEX[1], - 'fr': CATEGORIES_BY_INDEX[2], - 'frp': CATEGORIES_BY_INDEX[2], - 'fur': CATEGORIES_BY_INDEX[1], - 'fy-NL': CATEGORIES_BY_INDEX[1], - 'ga-IE': CATEGORIES_BY_INDEX[11], - 'gd': CATEGORIES_BY_INDEX[4], - 'gl': CATEGORIES_BY_INDEX[1], - 'gn': CATEGORIES_BY_INDEX[1], - 'gu-IN': CATEGORIES_BY_INDEX[2], - 'he': CATEGORIES_BY_INDEX[1], - 'hi-IN': CATEGORIES_BY_INDEX[2], - 'hr': CATEGORIES_BY_INDEX[19], - 'hsb': CATEGORIES_BY_INDEX[10], - 'hto': CATEGORIES_BY_INDEX[1], - 'hu': CATEGORIES_BY_INDEX[1], - 'hy-AM': CATEGORIES_BY_INDEX[1], - 'ia': CATEGORIES_BY_INDEX[1], - 'id': CATEGORIES_BY_INDEX[0], - 'ilo': CATEGORIES_BY_INDEX[0], - 'is': CATEGORIES_BY_INDEX[15], - 'it': CATEGORIES_BY_INDEX[1], - 'ja': CATEGORIES_BY_INDEX[0], - 'ja-JP-mac': CATEGORIES_BY_INDEX[0], - 'jiv': CATEGORIES_BY_INDEX[17], - 'ka': CATEGORIES_BY_INDEX[1], - 'kab': CATEGORIES_BY_INDEX[1], - 'kk': CATEGORIES_BY_INDEX[1], - 'km': CATEGORIES_BY_INDEX[0], - 'kn': CATEGORIES_BY_INDEX[1], - 'ko': CATEGORIES_BY_INDEX[0], - 'ks': CATEGORIES_BY_INDEX[1], - 'ku': CATEGORIES_BY_INDEX[1], - 'lb': CATEGORIES_BY_INDEX[1], - 'lg': CATEGORIES_BY_INDEX[1], - 'lij': CATEGORIES_BY_INDEX[1], - 'lo': CATEGORIES_BY_INDEX[0], - 'lt': CATEGORIES_BY_INDEX[6], - 'ltg': CATEGORIES_BY_INDEX[3], - 'lv': CATEGORIES_BY_INDEX[3], - 'lus': CATEGORIES_BY_INDEX[0], - 'mai': CATEGORIES_BY_INDEX[1], - 'meh': CATEGORIES_BY_INDEX[0], - 'mix': CATEGORIES_BY_INDEX[0], - 'mk': CATEGORIES_BY_INDEX[15], - 'ml': CATEGORIES_BY_INDEX[1], - 'mn': CATEGORIES_BY_INDEX[1], - 'mr': CATEGORIES_BY_INDEX[1], - 'ms': CATEGORIES_BY_INDEX[0], - 'my': CATEGORIES_BY_INDEX[0], - 'nb-NO': CATEGORIES_BY_INDEX[1], - 'ne-NP': CATEGORIES_BY_INDEX[1], - 'nl': CATEGORIES_BY_INDEX[1], - 'nn-NO': CATEGORIES_BY_INDEX[1], - 'nr': CATEGORIES_BY_INDEX[1], - 'nso': CATEGORIES_BY_INDEX[2], - 'ny': CATEGORIES_BY_INDEX[1], - 'oc': CATEGORIES_BY_INDEX[2], - 'or': CATEGORIES_BY_INDEX[1], - 'pa-IN': CATEGORIES_BY_INDEX[2], - 'pai': CATEGORIES_BY_INDEX[0], - 'pl': CATEGORIES_BY_INDEX[9], - 'pt-BR': CATEGORIES_BY_INDEX[1], - 'pt-PT': CATEGORIES_BY_INDEX[1], - 'quy': CATEGORIES_BY_INDEX[1], - 'qvi': CATEGORIES_BY_INDEX[1], - 'rm': CATEGORIES_BY_INDEX[1], - 'ro': CATEGORIES_BY_INDEX[5], - 'ru': CATEGORIES_BY_INDEX[7], - 'rw': CATEGORIES_BY_INDEX[1], - 'sah': CATEGORIES_BY_INDEX[0], - 'sat': CATEGORIES_BY_INDEX[1], - 'sc': CATEGORIES_BY_INDEX[1], - 'scn': CATEGORIES_BY_INDEX[1], - 'si': CATEGORIES_BY_INDEX[1], - 'sk': CATEGORIES_BY_INDEX[8], - 'sl': CATEGORIES_BY_INDEX[10], - 'son': CATEGORIES_BY_INDEX[1], - 'sq': CATEGORIES_BY_INDEX[1], - 'sr': CATEGORIES_BY_INDEX[19], - 'ss': CATEGORIES_BY_INDEX[1], - 'st': CATEGORIES_BY_INDEX[1], - 'sv-SE': CATEGORIES_BY_INDEX[1], - 'sw': CATEGORIES_BY_INDEX[1], - 'ta-LK': CATEGORIES_BY_INDEX[1], - 'ta': CATEGORIES_BY_INDEX[1], - 'te': CATEGORIES_BY_INDEX[1], - 'th': CATEGORIES_BY_INDEX[0], - 'tl': CATEGORIES_BY_INDEX[1], - 'tn': CATEGORIES_BY_INDEX[1], - 'tr': CATEGORIES_BY_INDEX[1], - 'trs': CATEGORIES_BY_INDEX[1], - 'ts': CATEGORIES_BY_INDEX[1], - 'tsz': CATEGORIES_BY_INDEX[1], - 'uk': CATEGORIES_BY_INDEX[7], - 'ur': CATEGORIES_BY_INDEX[1], - 'uz': CATEGORIES_BY_INDEX[1], - 've': CATEGORIES_BY_INDEX[1], - 'vi': CATEGORIES_BY_INDEX[0], - 'wo': CATEGORIES_BY_INDEX[0], - 'xh': CATEGORIES_BY_INDEX[1], - 'zam': CATEGORIES_BY_INDEX[1], - 'zh-CN': CATEGORIES_BY_INDEX[0], - 'zh-TW': CATEGORIES_BY_INDEX[0], - 'zu': CATEGORIES_BY_INDEX[2], + 'ace': 0, + 'ach': 1, + 'af': 1, + 'ak': 2, + 'an': 1, + 'ar': 12, + 'arn': 1, + 'as': 1, + 'ast': 1, + 'az': 1, + 'be': 7, + 'bg': 1, + 'bn': 2, + 'bo': 0, + 'br': 16, + 'brx': 1, + 'bs': 19, + 'ca': 1, + 'cak': 1, + 'ckb': 1, + 'crh': 1, + 'cs': 8, + 'csb': 9, + 'cv': 1, + 'cy': 18, + 'da': 1, + 'de': 1, + 'dsb': 10, + 'el': 1, + 'en': 1, + 'eo': 1, + 'es': 1, + 'et': 1, + 'eu': 1, + 'fa': 2, + 'ff': 1, + 'fi': 1, + 'fr': 2, + 'frp': 2, + 'fur': 1, + 'fy': 1, + 'ga': 11, + 'gd': 4, + 'gl': 1, + 'gn': 1, + 'gu': 2, + 'he': 1, + 'hi': 2, + 'hr': 19, + 'hsb': 10, + 'hto': 1, + 'hu': 1, + 'hy': 1, + 'hye': 1, + 'ia': 1, + 'id': 0, + 'ilo': 0, + 'is': 15, + 'it': 1, + 'ja': 0, + 'jiv': 17, + 'ka': 1, + 'kab': 1, + 'kk': 1, + 'km': 0, + 'kn': 1, + 'ko': 0, + 'ks': 1, + 'ku': 1, + 'lb': 1, + 'lg': 1, + 'lij': 1, + 'lo': 0, + 'lt': 6, + 'ltg': 3, + 'lv': 3, + 'lus': 0, + 'mai': 1, + 'meh': 0, + 'mix': 0, + 'mk': 15, + 'ml': 1, + 'mn': 1, + 'mr': 1, + 'ms': 0, + 'my': 0, + 'nb': 1, + 'ne': 1, + 'nl': 1, + 'nn': 1, + 'nr': 1, + 'nso': 2, + 'ny': 1, + 'oc': 2, + 'or': 1, + 'pa': 2, + 'pai': 0, + 'pl': 9, + 'pt': 1, + 'quy': 1, + 'qvi': 1, + 'rm': 1, + 'ro': 5, + 'ru': 7, + 'rw': 1, + 'sah': 0, + 'sat': 1, + 'sc': 1, + 'scn': 1, + 'si': 1, + 'sk': 8, + 'sl': 10, + 'son': 1, + 'sq': 1, + 'sr': 19, + 'ss': 1, + 'st': 1, + 'sv': 1, + 'sw': 1, + 'ta': 1, + 'ta': 1, + 'te': 1, + 'th': 0, + 'tl': 1, + 'tn': 1, + 'tr': 1, + 'trs': 1, + 'ts': 1, + 'tsz': 1, + 'uk': 7, + 'ur': 1, + 'uz': 1, + 've': 1, + 'vi': 0, + 'wo': 0, + 'xh': 1, + 'zam': 1, + 'zh-CN': 0, + 'zh-TW': 0, + 'zu': 2, } + + +def get_plural(locale): + plural_form = get_plural_rule(locale) + if plural_form is None: + return None + return CATEGORIES_BY_INDEX[plural_form] + + +def get_plural_rule(locale): + if locale is None: + return None + if locale in CATEGORIES_BY_LOCALE: + return CATEGORIES_BY_LOCALE[locale] + locale = locale.split('-', 1)[0] + return CATEGORIES_BY_LOCALE.get(locale) diff --git a/third_party/python/compare-locales/compare_locales/tests/fluent/test_checks.py b/third_party/python/compare-locales/compare_locales/tests/fluent/test_checks.py index 23a1efe80abb..5a906d2a8d2a 100644 --- a/third_party/python/compare-locales/compare_locales/tests/fluent/test_checks.py +++ b/third_party/python/compare-locales/compare_locales/tests/fluent/test_checks.py @@ -403,5 +403,179 @@ msg = { $val -> ) +class CSSStyleTest(BaseHelper): + file = File('foo.ftl', 'foo.ftl') + refContent = b'''\ +simple = + .style = width:1px +select = + .style = {PLATFORM() -> + [windows] width:1px + *[unix] max-width:1px + } +ref = + .style = {simple.style} +broken = + .style = 28em +''' + + def test_simple(self): + self._test(dedent_ftl( + '''\ + simple = + .style = width:2px + '''), + tuple()) + self._test(dedent_ftl( + '''\ + simple = + .style = max-width:2px + '''), + ( + ( + 'warning', 0, + 'width only in reference, max-width only in l10n', 'fluent' + ), + )) + self._test(dedent_ftl( + '''\ + simple = + .style = stuff + '''), + ( + ( + 'error', 0, + 'reference is a CSS spec', 'fluent' + ), + )) + # Cover the current limitations of only plain strings + self._test(dedent_ftl( + '''\ + simple = + .style = {"width:3px"} + '''), + tuple()) + + def test_select(self): + self._test(dedent_ftl( + '''\ + select = + .style = width:2px + '''), + ( + ( + 'warning', 0, + 'width only in l10n', 'fluent' + ), + )) + self._test(dedent_ftl( + '''\ + select = + .style = max-width:2px + '''), + ( + ( + 'warning', 0, + 'max-width only in l10n', 'fluent' + ), + )) + self._test(dedent_ftl( + '''\ + select = + .style = stuff + '''), + ( + ( + 'error', 0, + 'reference is a CSS spec', 'fluent' + ), + )) + # Cover the current limitations of only plain strings + self._test(dedent_ftl( + '''\ + select = + .style = {"width:1px"} + '''), + tuple()) + + def test_ref(self): + self._test(dedent_ftl( + '''\ + ref = + .style = width:2px + '''), + ( + ( + 'warning', 0, + 'width only in l10n', 'fluent' + ), + ( + 'warning', 0, + 'Missing message reference: simple.style', 'fluent' + ), + )) + self._test(dedent_ftl( + '''\ + ref = + .style = max-width:2px + '''), + ( + ( + 'warning', 0, + 'max-width only in l10n', 'fluent' + ), + ( + 'warning', 0, + 'Missing message reference: simple.style', 'fluent' + ), + )) + self._test(dedent_ftl( + '''\ + ref = + .style = stuff + '''), + ( + ( + 'error', 0, + 'reference is a CSS spec', 'fluent' + ), + ( + 'warning', 0, + 'Missing message reference: simple.style', 'fluent' + ), + )) + # Cover the current limitations of only plain strings + self._test(dedent_ftl( + '''\ + ref = + .style = {"width:1px"} + '''), + ( + ( + 'warning', 0, + 'Missing message reference: simple.style', 'fluent' + ), + )) + + def test_broken(self): + self._test(dedent_ftl( + '''\ + broken = + .style = 27em + '''), + (('error', 0, 'reference is a CSS spec', 'fluent'),)) + self._test(dedent_ftl( + '''\ + broken = + .style = width: 27em + '''), + ( + ( + 'warning', 0, + 'width only in l10n', 'fluent' + ), + )) + + if __name__ == '__main__': unittest.main() diff --git a/third_party/python/compare-locales/compare_locales/tests/paths/__init__.py b/third_party/python/compare-locales/compare_locales/tests/paths/__init__.py index 7418970e1ded..1a99c53e2f2e 100644 --- a/third_party/python/compare-locales/compare_locales/tests/paths/__init__.py +++ b/third_party/python/compare-locales/compare_locales/tests/paths/__init__.py @@ -5,6 +5,8 @@ from __future__ import absolute_import +from collections import defaultdict +import six import tempfile from compare_locales.paths import ( ProjectConfig, File, ProjectFiles, TOMLParser @@ -21,6 +23,9 @@ class Rooted(object): def path(self, leaf=''): return self.root + leaf + def leaf(self, path): + return mozpath.relpath(path, self.root) + class SetupMixin(object): def setUp(self): @@ -36,18 +41,64 @@ class SetupMixin(object): self.cfg.set_locales(['de']) -class MockNode(object): - def __init__(self, name): - self.name = name +class MockOS(object): + '''Mock `os.path.isfile` and `os.walk` based on a list of files. + ''' + def __init__(self, root, paths): + self.root = root self.files = [] self.dirs = {} + if not paths: + return + if isinstance(paths[0], six.string_types): + paths = [ + mozpath.split(path) + for path in sorted(paths) + ] + child_paths = defaultdict(list) + for segs in paths: + if len(segs) == 1: + self.files.append(segs[0]) + else: + child_paths[segs[0]].append(segs[1:]) + for root, leafs in child_paths.items(): + self.dirs[root] = MockOS(mozpath.join(self.root, root), leafs) - def walk(self): - subdirs = sorted(self.dirs) - if self.name is not None: - yield self.name, subdirs, self.files + def find(self, dir_path): + relpath = mozpath.relpath(dir_path, self.root) + if relpath.startswith('..'): + return None + if relpath in ('', '.'): + return self + segs = mozpath.split(relpath) + node = self + while segs: + seg = segs.pop(0) + if seg not in node.dirs: + return None + node = node.dirs[seg] + return node + + def isfile(self, path): + dirname = mozpath.dirname(path) + if dirname: + node = self.find(dirname) + else: + node = self + return node and mozpath.basename(path) in node.files + + def walk(self, path=None): + if path is None: + node = self + else: + node = self.find(path) + if node is None: + return + subdirs = sorted(node.dirs) + if node.root is not None: + yield node.root, subdirs, node.files for subdir in subdirs: - child = self.dirs[subdir] + child = node.dirs[subdir] for tpl in child.walk(): yield tpl @@ -56,25 +107,18 @@ class MockProjectFiles(ProjectFiles): def __init__(self, mocks, locale, projects, mergebase=None): (super(MockProjectFiles, self) .__init__(locale, projects, mergebase=mergebase)) - self.mocks = mocks + root = mozpath.commonprefix(mocks) + files = [mozpath.relpath(f, root) for f in mocks] + self.mocks = MockOS(root, files) def _isfile(self, path): - return path in self.mocks + return self.mocks.isfile(path) def _walk(self, base): base = mozpath.normpath(base) - local_files = [ - mozpath.split(mozpath.relpath(f, base)) - for f in self.mocks if f.startswith(base) - ] - root = MockNode(base) - for segs in local_files: - node = root - for n, seg in enumerate(segs[:-1]): - if seg not in node.dirs: - node.dirs[seg] = MockNode('/'.join([base] + segs[:n+1])) - node = node.dirs[seg] - node.files.append(segs[-1]) + root = self.mocks.find(base) + if not root: + return for tpl in root.walk(): yield tpl diff --git a/third_party/python/compare-locales/compare_locales/tests/paths/test_configparser.py b/third_party/python/compare-locales/compare_locales/tests/paths/test_configparser.py index a503e50653bd..fe9d7dcf6eca 100644 --- a/third_party/python/compare-locales/compare_locales/tests/paths/test_configparser.py +++ b/third_party/python/compare-locales/compare_locales/tests/paths/test_configparser.py @@ -5,15 +5,16 @@ from __future__ import absolute_import, unicode_literals import unittest +import six from . import MockTOMLParser from compare_locales.paths.matcher import Matcher -from compare_locales.paths.project import ProjectConfig +from compare_locales.paths.project import ProjectConfig, ExcludeError from compare_locales import mozpath class TestConfigParser(unittest.TestCase): - def test_imports(self): + def test_includes(self): parser = MockTOMLParser({ "root.toml": """ basepath = "." @@ -21,9 +22,14 @@ basepath = "." o = "toolkit" [[includes]] path = "{o}/other.toml" +[[includes]] + path = "dom/more.toml" """, "other.toml": """ basepath = "." +""", + "more.toml": """ +basepath = "." """ }) config = parser.parse("root.toml") @@ -32,9 +38,55 @@ basepath = "." self.assertEqual(configs[0], config) self.assertListEqual( [c.path for c in configs], - ["root.toml", mozpath.abspath("toolkit/other.toml")] + [ + "root.toml", + mozpath.abspath("toolkit/other.toml"), + mozpath.abspath("dom/more.toml"), + ] ) + def test_excludes(self): + parser = MockTOMLParser({ + "root.toml": """ +basepath = "." +[[excludes]] + path = "exclude.toml" +[[excludes]] + path = "other-exclude.toml" + """, + "exclude.toml": """ +basepath = "." +""", + "other-exclude.toml": """ +basepath = "." +""", + "grandparent.toml": """ +basepath = "." +[[includes]] + path = "root.toml" +""", + "wrapped.toml": """ +basepath = "." +[[excludes]] + path = "root.toml" + """ + }) + config = parser.parse("root.toml") + self.assertIsInstance(config, ProjectConfig) + configs = list(config.configs) + self.assertListEqual(configs, [config]) + self.assertEqual( + [c.path for c in config.excludes], + [ + mozpath.abspath("exclude.toml"), + mozpath.abspath("other-exclude.toml"), + ] + ) + with six.assertRaisesRegex(self, ExcludeError, 'Included configs'): + parser.parse("grandparent.toml") + with six.assertRaisesRegex(self, ExcludeError, 'Excluded configs'): + parser.parse("wrapped.toml") + def test_paths(self): parser = MockTOMLParser({ "l10n.toml": """ diff --git a/third_party/python/compare-locales/compare_locales/tests/paths/test_files.py b/third_party/python/compare-locales/compare_locales/tests/paths/test_files.py index 3bd0692d396d..997d7d2ffc20 100644 --- a/third_party/python/compare-locales/compare_locales/tests/paths/test_files.py +++ b/third_party/python/compare-locales/compare_locales/tests/paths/test_files.py @@ -5,17 +5,64 @@ from __future__ import absolute_import import unittest +import mock from compare_locales.paths import ( - ProjectConfig + File, + ProjectConfig, + ProjectFiles, ) from . import ( + MockOS, MockProjectFiles, MockTOMLParser, Rooted, ) +class TestMockOS(Rooted, unittest.TestCase): + def setUp(self): + self.node = MockOS('jazz', [ + 'one/bit', + 'two/deep/in/directories/with/file1', + 'two/deep/in/directories/with/file2', + 'three/feet', + ]) + + def test_isfile(self): + self.assertTrue(self.node.isfile('jazz/one/bit')) + self.assertFalse(self.node.isfile('jazz/one')) + self.assertFalse(self.node.isfile('foo')) + + def test_walk(self): + self.assertListEqual( + list(self.node.walk()), + [ + ('jazz', ['one', 'three', 'two'], []), + ('jazz/one', [], ['bit']), + ('jazz/three', [], ['feet']), + ('jazz/two', ['deep'], []), + ('jazz/two/deep', ['in'], []), + ('jazz/two/deep/in', ['directories'], []), + ('jazz/two/deep/in/directories', ['with'], []), + ('jazz/two/deep/in/directories/with', [], [ + 'file1', + 'file2', + ]), + ] + ) + + def test_find(self): + self.assertIsNone(self.node.find('foo')) + self.assertIsNone(self.node.find('jazz/one/bit')) + self.assertIsNone(self.node.find('jazz/one/bit/too/much')) + self.assertIsNotNone(self.node.find('jazz/one')) + self.assertListEqual(list(self.node.find('jazz/one').walk()), [ + ('jazz/one', [], ['bit']), + ]) + self.assertEqual(self.node.find('jazz'), self.node) + + class TestProjectPaths(Rooted, unittest.TestCase): def test_l10n_path(self): cfg = ProjectConfig(None) @@ -244,6 +291,240 @@ class TestProjectPaths(Rooted, unittest.TestCase): ]) +@mock.patch('os.path.isfile') +@mock.patch('os.walk') +class TestExcludes(Rooted, unittest.TestCase): + def _list(self, locale, _walk, _isfile): + parser = MockTOMLParser({ + "pontoon.toml": + '''\ +basepath = "." + +[[includes]] + path = "configs-pontoon.toml" + +[[excludes]] + path = "configs-vendor.toml" +[[excludes]] + path = "configs-special-templates.toml" +''', + "vendor.toml": + '''\ +basepath = "." + +[[includes]] + path = "configs-vendor.toml" + +[[excludes]] + path = "configs-special-templates.toml" +''', + "configs-pontoon.toml": + '''\ +basepath = "." + +locales = [ + "de", + "gd", + "it", +] + +[[paths]] + reference = "en/**/*.ftl" + l10n = "{locale}/**/*.ftl" +''', + "configs-vendor.toml": + '''\ +basepath = "." + +locales = [ + "de", + "it", +] + +[[paths]] + reference = "en/firefox/*.ftl" + l10n = "{locale}/firefox/*.ftl" +''', + "configs-special-templates.toml": + '''\ +basepath = "." + +[[paths]] + reference = "en/firefox/home.ftl" + l10n = "{locale}/firefox/home.ftl" + locales = [ + "de", + "fr", + ] +[[paths]] + reference = "en/firefox/pagina.ftl" + l10n = "{locale}/firefox/pagina.ftl" + locales = [ + "gd", + ] +''', + }) + pontoon = parser.parse(self.path('/pontoon.toml')) + vendor = parser.parse(self.path('/vendor.toml')) + pc = ProjectFiles(locale, [pontoon, vendor]) + mock_files = [ + '{}/{}/{}'.format(locale, dir, f) + for locale in ('de', 'en', 'gd', 'it') + for dir, files in ( + ('firefox', ('home.ftl', 'feature.ftl')), + ('mozorg', ('mission.ftl',)), + ) + for f in files + ] + os_ = MockOS(self.root, mock_files) + _isfile.side_effect = os_.isfile + _walk.side_effect = os_.walk + local_files = [self.leaf(p).split('/', 1)[1] for p, _, _, _ in pc] + return pontoon, vendor, local_files + + def test_reference(self, _walk, _isfile): + pontoon_config, vendor_config, files = self._list(None, _walk, _isfile) + pontoon_files = ProjectFiles(None, [pontoon_config]) + vendor_files = ProjectFiles(None, [vendor_config]) + self.assertListEqual( + files, + [ + 'firefox/feature.ftl', + 'firefox/home.ftl', + 'mozorg/mission.ftl', + ] + ) + ref_path = self.path('/en/firefox/feature.ftl') + self.assertIsNotNone(pontoon_files.match(ref_path)) + self.assertIsNotNone(vendor_files.match(ref_path)) + ref_path = self.path('/en/firefox/home.ftl') + self.assertIsNotNone(pontoon_files.match(ref_path)) + self.assertIsNotNone(vendor_files.match(ref_path)) + ref_path = self.path('/en/mozorg/mission.ftl') + self.assertIsNotNone(pontoon_files.match(ref_path)) + self.assertIsNone(vendor_files.match(ref_path)) + + def test_de(self, _walk, _isfile): + # home.ftl excluded completely by configs-special-templates.toml + # firefox/* only in vendor + pontoon_config, vendor_config, files = self._list('de', _walk, _isfile) + pontoon_files = ProjectFiles('de', [pontoon_config]) + vendor_files = ProjectFiles('de', [vendor_config]) + self.assertListEqual( + files, + [ + 'firefox/feature.ftl', + # 'firefox/home.ftl', + 'mozorg/mission.ftl', + ] + ) + l10n_path = self.path('/de/firefox/feature.ftl') + ref_path = self.path('/en/firefox/feature.ftl') + self.assertEqual( + pontoon_config.filter( + File( + l10n_path, + 'de/firefox/feature.ftl', + locale='de' + ) + ), + 'ignore' + ) + self.assertIsNone(pontoon_files.match(l10n_path)) + self.assertIsNone(pontoon_files.match(ref_path)) + self.assertIsNotNone(vendor_files.match(l10n_path)) + self.assertIsNotNone(vendor_files.match(ref_path)) + l10n_path = self.path('/de/firefox/home.ftl') + ref_path = self.path('/en/firefox/home.ftl') + self.assertEqual( + pontoon_config.filter( + File( + l10n_path, + 'de/firefox/home.ftl', + locale='de' + ) + ), + 'ignore' + ) + self.assertIsNone(pontoon_files.match(l10n_path)) + self.assertIsNone(pontoon_files.match(ref_path)) + self.assertIsNone(vendor_files.match(l10n_path)) + self.assertIsNone(vendor_files.match(ref_path)) + l10n_path = self.path('/de/mozorg/mission.ftl') + ref_path = self.path('/en/mozorg/mission.ftl') + self.assertEqual( + pontoon_config.filter( + File( + l10n_path, + 'de/mozorg/mission.ftl', + locale='de' + ) + ), + 'error' + ) + self.assertIsNotNone(pontoon_files.match(l10n_path)) + self.assertIsNotNone(pontoon_files.match(ref_path)) + self.assertIsNone(vendor_files.match(l10n_path)) + self.assertIsNone(vendor_files.match(ref_path)) + + def test_gd(self, _walk, _isfile): + # only community localization + pontoon_config, vendor_config, files = self._list('gd', _walk, _isfile) + pontoon_files = ProjectFiles('gd', [pontoon_config]) + vendor_files = ProjectFiles('gd', [vendor_config]) + self.assertListEqual( + files, + [ + 'firefox/feature.ftl', + 'firefox/home.ftl', + 'mozorg/mission.ftl', + ] + ) + l10n_path = self.path('/gd/firefox/home.ftl') + ref_path = self.path('/en/firefox/home.ftl') + self.assertEqual( + pontoon_config.filter( + File( + l10n_path, + 'gd/firefox/home.ftl', + locale='gd' + ) + ), + 'error' + ) + self.assertIsNotNone(pontoon_files.match(l10n_path)) + self.assertIsNotNone(pontoon_files.match(ref_path)) + self.assertIsNone(vendor_files.match(l10n_path)) + self.assertIsNone(vendor_files.match(ref_path)) + + def test_it(self, _walk, _isfile): + # all pages translated, but split between vendor and community + pontoon_config, vendor_config, files = self._list('it', _walk, _isfile) + pontoon_files = ProjectFiles('it', [pontoon_config]) + vendor_files = ProjectFiles('it', [vendor_config]) + self.assertListEqual( + files, + [ + 'firefox/feature.ftl', + 'firefox/home.ftl', + 'mozorg/mission.ftl', + ] + ) + l10n_path = self.path('/it/firefox/home.ftl') + ref_path = self.path('/en/firefox/home.ftl') + file = File( + l10n_path, + 'it/firefox/home.ftl', + locale='it' + ) + self.assertEqual(pontoon_config.filter(file), 'ignore') + self.assertEqual(vendor_config.filter(file), 'error') + self.assertIsNone(pontoon_files.match(l10n_path)) + self.assertIsNone(pontoon_files.match(ref_path)) + self.assertIsNotNone(vendor_files.match(l10n_path)) + self.assertIsNotNone(vendor_files.match(ref_path)) + + class TestL10nMerge(Rooted, unittest.TestCase): # need to go through TOMLParser, as that's handling most of the # environment diff --git a/third_party/python/compare-locales/compare_locales/tests/paths/test_matcher.py b/third_party/python/compare-locales/compare_locales/tests/paths/test_matcher.py index ac705967011f..74a20a84ce4b 100644 --- a/third_party/python/compare-locales/compare_locales/tests/paths/test_matcher.py +++ b/third_party/python/compare-locales/compare_locales/tests/paths/test_matcher.py @@ -46,6 +46,18 @@ class TestMatcher(unittest.TestCase): self.assertTrue(one.match('foo/tender/bar/baz.qux')) self.assertFalse(one.match('foo/nobar/baz.qux')) self.assertFalse(one.match('foo/tender/bar')) + other = Matcher('baz/**/qux/**') + self.assertEqual(one.sub(other, 'foo/bar/baz.qux'), 'baz/qux/baz.qux') + self.assertEqual( + one.sub(other, 'foo/tender/bar/baz.qux'), + 'baz/tender/qux/baz.qux' + ) + + def test_encoded_matcher(self): + one = Matcher('foo/*', encoding='utf-8') + self.assertTrue(one.match(b'foo/bar')) + other = Matcher('bar/*', encoding='utf-8') + self.assertEqual(one.sub(other, b'foo/baz'), b'bar/baz') def test_prefix(self): self.assertEqual( @@ -87,6 +99,22 @@ class TestMatcher(unittest.TestCase): 'foo/' ) + def test_encoded_prefix(self): + self.assertEqual( + Matcher('foo/bar.file', encoding='utf-8').prefix, b'foo/bar.file' + ) + self.assertEqual( + Matcher('foo/*', encoding='utf-8').prefix, b'foo/' + ) + self.assertEqual( + Matcher('foo/{v}/bar', encoding='utf-8').prefix, + b'foo/' + ) + self.assertEqual( + Matcher('foo/{v}/bar', {'v': 'expanded'}, encoding='utf-8').prefix, + b'foo/expanded/bar' + ) + def test_variables(self): self.assertDictEqual( Matcher('foo/bar.file').match('foo/bar.file'), @@ -157,6 +185,30 @@ class TestMatcher(unittest.TestCase): } ) + def test_encoded_variables(self): + self.assertDictEqual( + Matcher('foo/bar.file', encoding='utf-8').match(b'foo/bar.file'), + {} + ) + self.assertDictEqual( + Matcher( + '{path}/bar.file', encoding='utf-8' + ).match(b'foo/bar.file'), + { + 'path': 'foo' + } + ) + self.assertDictEqual( + Matcher('{l}*', { + 'l': 'foo/{locale}/' + }, encoding='utf-8').match(b'foo/it/path'), + { + 'l': 'foo/it/', + 'locale': 'it', + 's1': 'path', + } + ) + def test_variables_sub(self): one = Matcher('{base}/{loc}/*', {'base': 'ONE_BASE'}) other = Matcher('{base}/somewhere/*', {'base': 'OTHER_BASE'}) @@ -164,6 +216,14 @@ class TestMatcher(unittest.TestCase): one.sub(other, 'ONE_BASE/ab-CD/special'), 'OTHER_BASE/somewhere/special' ) + one = Matcher('{base}/{loc}/*', {'base': 'ONE_BASE'}, encoding='utf-8') + other = Matcher( + '{base}/somewhere/*', {'base': 'OTHER_BASE'}, encoding='utf-8' + ) + self.assertEqual( + one.sub(other, b'ONE_BASE/ab-CD/special'), + b'OTHER_BASE/somewhere/special' + ) def test_copy(self): one = Matcher('{base}/{loc}/*', { diff --git a/third_party/python/compare-locales/compare_locales/tests/po/test_parser.py b/third_party/python/compare-locales/compare_locales/tests/po/test_parser.py index dfa68695d9ba..e02fe6628319 100644 --- a/third_party/python/compare-locales/compare_locales/tests/po/test_parser.py +++ b/third_party/python/compare-locales/compare_locales/tests/po/test_parser.py @@ -128,7 +128,7 @@ msgstr "" (Whitespace, '\n'), (('reference 1', None), 'translated string'), (Whitespace, '\n'), - (('reference 2', None), ''), + (('reference 2', None), 'reference 2'), (Whitespace, '\n'), ) ) diff --git a/third_party/python/compare-locales/compare_locales/tests/test_checks.py b/third_party/python/compare-locales/compare_locales/tests/test_checks.py new file mode 100644 index 000000000000..193ac60c6b31 --- /dev/null +++ b/third_party/python/compare-locales/compare_locales/tests/test_checks.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +import unittest + +from compare_locales.checks.base import CSSCheckMixin + + +class CSSParserTest(unittest.TestCase): + def setUp(self): + self.mixin = CSSCheckMixin() + + def test_other(self): + refMap, errors = self.mixin.parse_css_spec('foo') + self.assertIsNone(refMap) + self.assertIsNone(errors) + + def test_css_specs(self): + for prop in ( + 'min-width', 'width', 'max-width', + 'min-height', 'height', 'max-height', + ): + refMap, errors = self.mixin.parse_css_spec('{}:1px;'.format(prop)) + self.assertDictEqual( + refMap, {prop: 'px'} + ) + self.assertIsNone(errors) + + def test_single_whitespace(self): + refMap, errors = self.mixin.parse_css_spec('width:15px;') + self.assertDictEqual( + refMap, {'width': 'px'} + ) + self.assertIsNone(errors) + refMap, errors = self.mixin.parse_css_spec('width : \t 15px ; ') + self.assertDictEqual( + refMap, {'width': 'px'} + ) + self.assertIsNone(errors) + refMap, errors = self.mixin.parse_css_spec('width: 15px') + self.assertDictEqual( + refMap, {'width': 'px'} + ) + self.assertIsNone(errors) + + def test_multiple(self): + refMap, errors = self.mixin.parse_css_spec('width:15px;height:20.2em;') + self.assertDictEqual( + refMap, {'height': 'em', 'width': 'px'} + ) + self.assertIsNone(errors) + refMap, errors = self.mixin.parse_css_spec( + 'width:15px \t\t; height:20em' + ) + self.assertDictEqual( + refMap, {'height': 'em', 'width': 'px'} + ) + self.assertIsNone(errors) + + def test_errors(self): + refMap, errors = self.mixin.parse_css_spec('width:15pxfoo') + self.assertDictEqual( + refMap, {'width': 'px'} + ) + self.assertListEqual( + errors, [{'pos': 10, 'code': 'css-bad-content'}] + ) + refMap, errors = self.mixin.parse_css_spec('width:15px height:20em') + self.assertDictEqual( + refMap, {'height': 'em', 'width': 'px'} + ) + self.assertListEqual( + errors, [{'pos': 10, 'code': 'css-missing-semicolon'}] + ) + refMap, errors = self.mixin.parse_css_spec('witdth:15px') + self.assertIsNone(refMap) + self.assertIsNone(errors) + refMap, errors = self.mixin.parse_css_spec('width:1,5px') + self.assertIsNone(refMap) + self.assertIsNone(errors) + refMap, errors = self.mixin.parse_css_spec('width:1.5.1px') + self.assertIsNone(refMap) + self.assertIsNone(errors) + refMap, errors = self.mixin.parse_css_spec('width:1.px') + self.assertIsNone(refMap) + self.assertIsNone(errors) diff --git a/third_party/python/compare-locales/compare_locales/tests/test_compare.py b/third_party/python/compare-locales/compare_locales/tests/test_compare.py index fd0127ee8ef9..acc47cff6808 100644 --- a/third_party/python/compare-locales/compare_locales/tests/test_compare.py +++ b/third_party/python/compare-locales/compare_locales/tests/test_compare.py @@ -91,7 +91,17 @@ class TestObserver(unittest.TestCase): self.assertDictEqual(obs.toJSON(), { 'summary': { 'de': { - 'missing': 15 + 'errors': 0, + 'warnings': 0, + 'missing': 15, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, + 'changed': 0, + 'changed_w': 0, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, } }, 'details': { @@ -112,7 +122,17 @@ class TestObserver(unittest.TestCase): self.assertDictEqual(obs.toJSON(), { 'summary': { 'de': { - 'missing': 15 + 'errors': 0, + 'warnings': 0, + 'missing': 15, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, + 'changed': 0, + 'changed_w': 0, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, } }, 'details': { diff --git a/third_party/python/compare-locales/compare_locales/tests/test_merge.py b/third_party/python/compare-locales/compare_locales/tests/test_merge.py index 7c8039acceda..a10a04ca16a1 100644 --- a/third_party/python/compare-locales/compare_locales/tests/test_merge.py +++ b/third_party/python/compare-locales/compare_locales/tests/test_merge.py @@ -150,10 +150,17 @@ class TestDefines(unittest.TestCase, ContentMixin): cc.observers.toJSON(), {'summary': {None: { + 'errors': 0, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 1, 'changed_w': 2, 'unchanged': 1, - 'unchanged_w': 1 + 'unchanged_w': 1, + 'keys': 0, }}, 'details': {} } @@ -189,10 +196,17 @@ class TestDefines(unittest.TestCase, ContentMixin): { 'summary': {None: { + 'errors': 0, + 'warnings': 0, 'missing': 1, 'missing_w': 2, + 'report': 0, + 'obsolete': 0, + 'changed': 0, + 'changed_w': 0, 'unchanged': 1, - 'unchanged_w': 1 + 'unchanged_w': 1, + 'keys': 0, }}, 'details': { @@ -238,8 +252,18 @@ eff = lEff word cc.observers.toJSON(), {'summary': {None: { + 'errors': 0, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 3, - 'changed_w': 5 + 'changed_w': 5, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, + }}, 'details': {} } @@ -265,10 +289,17 @@ eff = effVal""") cc.observers.toJSON(), {'summary': {None: { + 'errors': 0, + 'warnings': 0, + 'missing': 2, + 'missing_w': 2, + 'report': 0, + 'obsolete': 0, 'changed': 1, 'changed_w': 1, - 'missing': 2, - 'missing_w': 2 + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, }}, 'details': { 'l10n.properties': [ @@ -297,8 +328,17 @@ eff = effVal""") cc.observers.toJSON(), {'summary': {None: { - 'missingInFiles': 3, - 'missing_w': 3 + 'errors': 0, + 'warnings': 0, + 'missing': 3, + 'missing_w': 3, + 'report': 0, + 'obsolete': 0, + 'changed': 0, + 'changed_w': 0, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, }}, 'details': { 'l10n.properties': [ @@ -326,11 +366,17 @@ eff = leffVal cc.observers.toJSON(), {'summary': {None: { + 'errors': 1, + 'warnings': 0, + 'missing': 1, + 'missing_w': 1, + 'report': 0, + 'obsolete': 0, 'changed': 2, 'changed_w': 3, - 'errors': 1, - 'missing': 1, - 'missing_w': 1 + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, }}, 'details': { 'l10n.properties': [ @@ -364,11 +410,17 @@ eff = leffVal cc.observers.toJSON(), {'summary': {None: { + 'errors': 0, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 1, 'changed': 1, 'changed_w': 1, - 'obsolete': 1, 'unchanged': 1, - 'unchanged_w': 1 + 'unchanged_w': 1, + 'keys': 0, }}, 'details': { 'l10n.properties': [ @@ -422,8 +474,15 @@ bar = duplicated bar {None: { 'errors': 1, 'warnings': 1, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 3, - 'changed_w': 6 + 'changed_w': 6, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, }}, 'details': { 'l10n.properties': [ @@ -465,8 +524,17 @@ class TestDTD(unittest.TestCase, ContentMixin): cc.observers.toJSON(), {'summary': {None: { + 'errors': 0, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 3, - 'changed_w': 3 + 'changed_w': 3, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, }}, 'details': {} } @@ -492,10 +560,17 @@ class TestDTD(unittest.TestCase, ContentMixin): cc.observers.toJSON(), {'summary': {None: { + 'errors': 0, + 'warnings': 0, + 'missing': 2, + 'missing_w': 2, + 'report': 0, + 'obsolete': 0, 'changed': 1, 'changed_w': 1, - 'missing': 2, - 'missing_w': 2 + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, }}, 'details': { 'l10n.dtd': [ @@ -529,10 +604,16 @@ class TestDTD(unittest.TestCase, ContentMixin): {'summary': {None: { 'errors': 1, + 'warnings': 0, 'missing': 1, 'missing_w': 1, + 'report': 0, + 'obsolete': 0, + 'changed': 0, + 'changed_w': 0, 'unchanged': 2, - 'unchanged_w': 2 + 'unchanged_w': 2, + 'keys': 0, }}, 'details': { 'l10n.dtd': [ @@ -567,9 +648,17 @@ class TestDTD(unittest.TestCase, ContentMixin): cc.observers.toJSON(), {'summary': {None: { + 'errors': 0, 'warnings': 1, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, + 'changed': 0, + 'changed_w': 0, 'unchanged': 2, - 'unchanged_w': 2 + 'unchanged_w': 2, + 'keys': 0, }}, 'details': { 'l10n.dtd': [ @@ -597,11 +686,17 @@ class TestDTD(unittest.TestCase, ContentMixin): cc.observers.toJSON(), {'summary': {None: { + 'errors': 0, 'warnings': 1, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, + 'changed': 1, + 'changed_w': 2, 'unchanged': 2, 'unchanged_w': 2, - 'changed': 1, - 'changed_w': 2 + 'keys': 0, }}, 'details': { 'l10n.dtd': [ @@ -658,8 +753,17 @@ bar = lBar cc.observers.toJSON(), {'summary': {None: { + 'errors': 0, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 3, - 'changed_w': 3 + 'changed_w': 3, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, }}, 'details': {} } @@ -697,10 +801,17 @@ eff = lEff }, 'summary': { None: { - 'changed': 2, - 'changed_w': 2, + 'errors': 0, + 'warnings': 0, 'missing': 2, 'missing_w': 2, + 'report': 0, + 'obsolete': 0, + 'changed': 2, + 'changed_w': 2, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, } } } @@ -748,11 +859,17 @@ eff = lEff { }, 'summary': { None: { - 'changed': 1, - 'changed_w': 1, + 'errors': 3, + 'warnings': 0, 'missing': 2, 'missing_w': 2, - 'errors': 3 + 'report': 0, + 'obsolete': 0, + 'changed': 1, + 'changed_w': 1, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, } } } @@ -793,8 +910,17 @@ foo = Localized { bar } 'details': {}, 'summary': { None: { + 'errors': 0, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 1, - 'changed_w': 1 + 'changed_w': 1, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, } } } @@ -850,9 +976,17 @@ baz = Localized { qux } }, 'summary': { None: { + 'errors': 0, + 'warnings': 4, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 3, 'changed_w': 3, - 'warnings': 4 + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, } } } @@ -899,7 +1033,19 @@ eff = lEff ], }, 'summary': { - None: {'changed': 3, 'changed_w': 5, 'errors': 2} + None: { + 'errors': 2, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, + 'changed': 3, + 'changed_w': 5, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, + } } } ) @@ -955,10 +1101,17 @@ eff = lEff }, 'summary': { None: { - 'changed': 4, - 'changed_w': 4, + 'errors': 0, + 'warnings': 0, 'missing': 1, 'missing_w': 1, + 'report': 0, + 'obsolete': 0, + 'changed': 4, + 'changed_w': 4, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, } } } @@ -1003,7 +1156,19 @@ bar = lBar ] }, 'summary': { - None: {'changed': 2, 'changed_w': 4, 'errors': 2} + None: { + 'errors': 2, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, + 'changed': 2, + 'changed_w': 4, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, + } } } ) @@ -1040,8 +1205,17 @@ bar = lBar 'details': {}, 'summary': { None: { + 'errors': 0, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 2, 'changed_w': 2, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, } } } @@ -1074,10 +1248,17 @@ bar = barVal 'details': {}, 'summary': { None: { + 'errors': 0, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 1, 'changed_w': 1, 'unchanged': 1, 'unchanged_w': 1, + 'keys': 0, } } } @@ -1111,8 +1292,17 @@ bar = lBar 'details': {}, 'summary': { None: { + 'errors': 0, + 'warnings': 0, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 2, 'changed_w': 2, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, } } } @@ -1144,8 +1334,15 @@ bar = duplicated bar {None: { 'errors': 1, 'warnings': 1, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 3, - 'changed_w': 6 + 'changed_w': 6, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, }}, 'details': { 'l10n.ftl': [ @@ -1174,9 +1371,17 @@ bar = duplicated bar cc.observers.toJSON(), {'summary': {None: { + 'errors': 0, 'warnings': 3, + 'missing': 0, + 'missing_w': 0, + 'report': 0, + 'obsolete': 0, 'changed': 1, - 'changed_w': 2 + 'changed_w': 2, + 'unchanged': 0, + 'unchanged_w': 0, + 'keys': 0, }}, 'details': { 'l10n.ftl': [ diff --git a/third_party/python/compare-locales/compare_locales/tests/test_parser.py b/third_party/python/compare-locales/compare_locales/tests/test_parser.py index 05e57242916c..38fe642ddfeb 100644 --- a/third_party/python/compare-locales/compare_locales/tests/test_parser.py +++ b/third_party/python/compare-locales/compare_locales/tests/test_parser.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import +import pkg_resources import shutil import tempfile import textwrap @@ -87,3 +88,31 @@ class TestUniversalNewlines(unittest.TestCase): self.assertEqual( self.parser.ctx.contents, 'one\ntwo\nthree\n') + + +class TestPlugins(unittest.TestCase): + def setUp(self): + self.old_working_set_state = pkg_resources.working_set.__getstate__() + distribution = pkg_resources.Distribution(__file__) + entry_point = pkg_resources.EntryPoint.parse( + 'test_parser = compare_locales.tests.test_parser:DummyParser', + dist=distribution + ) + distribution._ep_map = { + 'compare_locales.parsers': { + 'test_parser': entry_point + } + } + pkg_resources.working_set.add(distribution) + + def tearDown(self): + pkg_resources.working_set.__setstate__(self.old_working_set_state) + + def test_dummy_parser(self): + p = parser.getParser('some/weird/file.ext') + self.assertIsInstance(p, DummyParser) + + +class DummyParser(parser.Parser): + def use(self, path): + return path.endswith('weird/file.ext') diff --git a/third_party/python/compare-locales/setup.py b/third_party/python/compare-locales/setup.py index 8fe29100ab1c..e9a70be5d552 100755 --- a/third_party/python/compare-locales/setup.py +++ b/third_party/python/compare-locales/setup.py @@ -52,8 +52,11 @@ setup(name="compare-locales", 'compare_locales.tests': ['data/*.properties', 'data/*.dtd'] }, install_requires=[ - 'fluent.syntax >=0.14.0, <0.16', + 'fluent.syntax >=0.17.0, <0.18', 'pytoml', 'six', ], + tests_require=[ + 'mock<4.0', + ], test_suite='compare_locales.tests') diff --git a/third_party/python/fluent.migrate/PKG-INFO b/third_party/python/fluent.migrate/PKG-INFO new file mode 100644 index 000000000000..aad939b339d1 --- /dev/null +++ b/third_party/python/fluent.migrate/PKG-INFO @@ -0,0 +1,62 @@ +Metadata-Version: 2.1 +Name: fluent.migrate +Version: 0.9 +Summary: Toolchain to migrate legacy translation to Fluent. +Home-page: https://hg.mozilla.org/l10n/fluent-migration/ +Author: Mozilla +Author-email: l10n-drivers@mozilla.org +License: APL 2 +Description: Fluent Migration Tools + ====================== + + Programmatically create Fluent files from existing content in both legacy + and Fluent formats. Use recipes written in Python to migrate content for each + of your localizations. + + `migrate-l10n` is a CLI script which uses the `fluent.migrate` module under + the hood to run migrations on existing translations. + + `validate-l10n-recipe` is a CLI script to test a migration recipe for common + errors, without trying to apply it. + + Installation + ------------ + + Install from PyPI: + + pip install fluent.migrate[hg] + + If you only want to use the `MigrationContext` API, you can drop the + requirement on `python-hglib`: + + pip install fluent.migrate + + Usage + ----- + + Migrations consist of _recipes_, which are applied to a _localization repository_, based on _template files_. + You can find recipes for Firefox in `mozilla-central/python/l10n/fluent_migrations/`, + the reference repository is [gecko-strings](https://hg.mozilla.org/l10n/gecko-strings/) or _quarantine_. + You apply those migrations to l10n repositories in [l10n-central](https://hg.mozilla.org/l10n-central/), or to `gecko-strings` for testing. + + The migrations are run as python modules, so you need to have their file location in `PYTHONPATH`. + + An example would look like + + $ migrate-l10n --lang it --reference-dir gecko-strings --localization-dir l10n-central/it bug_1451992_preferences_sitedata bug_1451992_preferences_translation + + Contact + ------- + + - mailing list: https://lists.mozilla.org/listinfo/tools-l10n + - bugzilla: [Open Bugs](https://bugzilla.mozilla.org/buglist.cgi?component=Fluent%20Migration&product=Localization%20Infrastructure%20and%20Tools&bug_status=__open__) - [New Bug](https://bugzilla.mozilla.org/enter_bug.cgi?product=Localization%20Infrastructure%20and%20Tools&component=Fluent%20Migration) + +Keywords: fluent,localization,l10n +Platform: UNKNOWN +Classifier: Development Status :: 3 - Alpha +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3.7 +Description-Content-Type: text/markdown +Provides-Extra: hg diff --git a/third_party/python/fluent.migrate/README.md b/third_party/python/fluent.migrate/README.md new file mode 100644 index 000000000000..5d925eece963 --- /dev/null +++ b/third_party/python/fluent.migrate/README.md @@ -0,0 +1,44 @@ +Fluent Migration Tools +====================== + +Programmatically create Fluent files from existing content in both legacy +and Fluent formats. Use recipes written in Python to migrate content for each +of your localizations. + +`migrate-l10n` is a CLI script which uses the `fluent.migrate` module under +the hood to run migrations on existing translations. + +`validate-l10n-recipe` is a CLI script to test a migration recipe for common +errors, without trying to apply it. + +Installation +------------ + +Install from PyPI: + + pip install fluent.migrate[hg] + +If you only want to use the `MigrationContext` API, you can drop the +requirement on `python-hglib`: + + pip install fluent.migrate + +Usage +----- + +Migrations consist of _recipes_, which are applied to a _localization repository_, based on _template files_. +You can find recipes for Firefox in `mozilla-central/python/l10n/fluent_migrations/`, +the reference repository is [gecko-strings](https://hg.mozilla.org/l10n/gecko-strings/) or _quarantine_. +You apply those migrations to l10n repositories in [l10n-central](https://hg.mozilla.org/l10n-central/), or to `gecko-strings` for testing. + +The migrations are run as python modules, so you need to have their file location in `PYTHONPATH`. + +An example would look like + + $ migrate-l10n --lang it --reference-dir gecko-strings --localization-dir l10n-central/it bug_1451992_preferences_sitedata bug_1451992_preferences_translation + +Contact +------- + + - mailing list: https://lists.mozilla.org/listinfo/tools-l10n + - bugzilla: [Open Bugs](https://bugzilla.mozilla.org/buglist.cgi?component=Fluent%20Migration&product=Localization%20Infrastructure%20and%20Tools&bug_status=__open__) - [New Bug](https://bugzilla.mozilla.org/enter_bug.cgi?product=Localization%20Infrastructure%20and%20Tools&component=Fluent%20Migration) diff --git a/third_party/python/fluent.migrate/fluent/migrate/_context.py b/third_party/python/fluent.migrate/fluent/migrate/_context.py new file mode 100644 index 000000000000..206f0162fda3 --- /dev/null +++ b/third_party/python/fluent.migrate/fluent/migrate/_context.py @@ -0,0 +1,325 @@ +# coding=utf8 +from __future__ import unicode_literals +from __future__ import absolute_import + +import os +import codecs +from functools import partial +import logging +from six.moves import zip_longest + +import fluent.syntax.ast as FTL +from fluent.syntax.parser import FluentParser +from fluent.syntax.serializer import FluentSerializer +from compare_locales.parser import getParser +from compare_locales.plurals import get_plural + +from .merge import merge_resource +from .errors import ( + UnreadableReferenceError, +) + + +class InternalContext(object): + """Internal context for merging translation resources. + + For the public interface, see `context.MigrationContext`. + """ + + def __init__( + self, lang, reference_dir, localization_dir, enforce_translated=False + ): + self.fluent_parser = FluentParser(with_spans=False) + self.fluent_serializer = FluentSerializer() + + # An iterable of plural category names relevant to the context's + # language. E.g. ('one', 'other') for English. + self.plural_categories = get_plural(lang) + if self.plural_categories is None: + logger = logging.getLogger('migrate') + logger.warning( + 'Plural rule for "{}" is not defined in ' + 'compare-locales'.format(lang)) + self.plural_categories = ('one', 'other') + + self.enforce_translated = enforce_translated + # Parsed input resources stored by resource path. + self.reference_resources = {} + self.localization_resources = {} + self.target_resources = {} + + # An iterable of `FTL.Message` objects some of whose nodes can be the + # transform operations. + self.transforms = {} + + def read_ftl_resource(self, path): + """Read an FTL resource and parse it into an AST.""" + f = codecs.open(path, 'r', 'utf8') + try: + contents = f.read() + except UnicodeDecodeError as err: + logger = logging.getLogger('migrate') + logger.warning('Unable to read file {}: {}'.format(path, err)) + raise err + finally: + f.close() + + ast = self.fluent_parser.parse(contents) + + annots = [ + annot + for entry in ast.body + if isinstance(entry, FTL.Junk) + for annot in entry.annotations + ] + + if len(annots): + logger = logging.getLogger('migrate') + for annot in annots: + msg = annot.message + logger.warning('Syntax error in {}: {}'.format(path, msg)) + + return ast + + def read_legacy_resource(self, path): + """Read a legacy resource and parse it into a dict.""" + parser = getParser(path) + parser.readFile(path) + # Transform the parsed result which is an iterator into a dict. + return { + entity.key: entity.val for entity in parser + if entity.localized or self.enforce_translated + } + + def read_reference_ftl(self, path): + """Read and parse a reference FTL file. + + A missing resource file is a fatal error and will raise an + UnreadableReferenceError. + """ + fullpath = os.path.join(self.reference_dir, path) + try: + return self.read_ftl_resource(fullpath) + except IOError: + error_message = 'Missing reference file: {}'.format(fullpath) + logging.getLogger('migrate').error(error_message) + raise UnreadableReferenceError(error_message) + except UnicodeDecodeError as err: + error_message = 'Error reading file {}: {}'.format(fullpath, err) + logging.getLogger('migrate').error(error_message) + raise UnreadableReferenceError(error_message) + + def read_localization_ftl(self, path): + """Read and parse an existing localization FTL file. + + Create a new FTL.Resource if the file doesn't exist or can't be + decoded. + """ + fullpath = os.path.join(self.localization_dir, path) + try: + return self.read_ftl_resource(fullpath) + except IOError: + logger = logging.getLogger('migrate') + logger.info( + 'Localization file {} does not exist and ' + 'it will be created'.format(path)) + return FTL.Resource() + except UnicodeDecodeError: + logger = logging.getLogger('migrate') + logger.warning( + 'Localization file {} has broken encoding. ' + 'It will be re-created and some translations ' + 'may be lost'.format(path)) + return FTL.Resource() + + def maybe_add_localization(self, path): + """Add a localization resource to migrate translations from. + + Uses a compare-locales parser to create a dict of (key, string value) + tuples. + For Fluent sources, we store the AST. + """ + try: + fullpath = os.path.join(self.localization_dir, path) + if not fullpath.endswith('.ftl'): + collection = self.read_legacy_resource(fullpath) + else: + collection = self.read_ftl_resource(fullpath) + except IOError: + logger = logging.getLogger('migrate') + logger.warning('Missing localization file: {}'.format(path)) + else: + self.localization_resources[path] = collection + + def get_legacy_source(self, path, key): + """Get an entity value from a localized legacy source. + + Used by the `Source` transform. + """ + resource = self.localization_resources[path] + return resource.get(key, None) + + def get_fluent_source_pattern(self, path, key): + """Get a pattern from a localized Fluent source. + + If the key contains a `.`, does an attribute lookup. + Used by the `COPY_PATTERN` transform. + """ + resource = self.localization_resources[path] + msg_key, _, attr_key = key.partition('.') + found = None + for entry in resource.body: + if isinstance(entry, (FTL.Message, FTL.Term)): + if entry.id.name == msg_key: + found = entry + break + if found is None: + return None + if not attr_key: + return found.value + for attribute in found.attributes: + if attribute.id.name == attr_key: + return attribute.value + return None + + def messages_equal(self, res1, res2): + """Compare messages and terms of two FTL resources. + + Uses FTL.BaseNode.equals to compare all messages/terms + in two FTL resources. + If the order or number of messages differ, the result is also False. + """ + def message_id(message): + "Return the message's identifer name for sorting purposes." + return message.id.name + + messages1 = sorted( + (entry for entry in res1.body + if isinstance(entry, FTL.Message) + or isinstance(entry, FTL.Term)), + key=message_id) + messages2 = sorted( + (entry for entry in res2.body + if isinstance(entry, FTL.Message) + or isinstance(entry, FTL.Term)), + key=message_id) + for msg1, msg2 in zip_longest(messages1, messages2): + if msg1 is None or msg2 is None: + return False + if not msg1.equals(msg2): + return False + return True + + def merge_changeset(self, changeset=None, known_translations=None): + """Return a generator of FTL ASTs for the changeset. + + The input data must be configured earlier using the `add_*` methods. + if given, `changeset` must be a set of (path, key) tuples describing + which legacy translations are to be merged. If `changeset` is None, + all legacy translations will be allowed to be migrated in a single + changeset. + + We use the `in_changeset` method to determine if a message should be + migrated for the given changeset. + + Given `changeset`, return a dict whose keys are resource paths and + values are `FTL.Resource` instances. The values will also be used to + update this context's existing localization resources. + """ + + if changeset is None: + # Merge all known legacy translations. Used in tests. + changeset = { + (path, key) + for path, strings in self.localization_resources.items() + if not path.endswith('.ftl') + for key in strings.keys() + } + + if known_translations is None: + known_translations = changeset + + for path, reference in self.reference_resources.items(): + current = self.target_resources[path] + transforms = self.transforms.get(path, []) + in_changeset = partial( + self.in_changeset, changeset, known_translations, path) + + # Merge legacy translations with the existing ones using the + # reference as a template. + snapshot = merge_resource( + self, reference, current, transforms, in_changeset + ) + + # Skip this path if the messages in the merged snapshot are + # identical to those in the current state of the localization file. + # This may happen when: + # + # - none of the transforms is in the changset, or + # - all messages which would be migrated by the context's + # transforms already exist in the current state. + if self.messages_equal(current, snapshot): + continue + + # Store the merged snapshot on the context so that the next merge + # already takes it into account as the existing localization. + self.target_resources[path] = snapshot + + # The result for this path is a complete `FTL.Resource`. + yield path, snapshot + + def in_changeset(self, changeset, known_translations, path, ident): + """Check if a message should be migrated in this changeset. + + The message is identified by path and ident. + + + A message will be migrated only if all of its dependencies + are present in the currently processed changeset. + + If a transform defined for this message points to a missing + legacy translation, this message will not be merged. The + missing legacy dependency won't be present in the changeset. + + This also means that partially translated messages (e.g. + constructed from two legacy strings out of which only one is + avaiable) will never be migrated. + """ + message_deps = self.dependencies.get((path, ident), None) + + # Don't merge if we don't have a transform for this message. + if message_deps is None: + return False + + # As a special case, if a transform exists but has no + # dependecies, it's a hardcoded `FTL.Node` which doesn't + # migrate any existing translation but rather creates a new + # one. Merge it. + if len(message_deps) == 0: + return True + + # Make sure all the dependencies are present in the current + # changeset. Partial migrations are not currently supported. + # See https://bugzilla.mozilla.org/show_bug.cgi?id=1321271 + # We only return True if our current changeset touches + # the transform, and we have all of the dependencies. + active_deps = message_deps & changeset + available_deps = message_deps & known_translations + return active_deps and message_deps == available_deps + + def serialize_changeset(self, changeset, known_translations=None): + """Return a dict of serialized FTLs for the changeset. + + Given `changeset`, return a dict whose keys are resource paths and + values are serialized FTL snapshots. + """ + + return { + path: self.fluent_serializer.serialize(snapshot) + for path, snapshot in self.merge_changeset( + changeset, known_translations + ) + } + + +logging.basicConfig() diff --git a/third_party/python/fluent.migrate/fluent/migrate/changesets.py b/third_party/python/fluent.migrate/fluent/migrate/changesets.py index 6acb93295d5c..e4ad95f2d140 100644 --- a/third_party/python/fluent.migrate/fluent/migrate/changesets.py +++ b/third_party/python/fluent.migrate/fluent/migrate/changesets.py @@ -27,7 +27,7 @@ def convert_blame_to_changesets(blame_json): } It will be transformed into a list of changesets which can be fed into - `MergeContext.serialize_changeset`: + `InternalContext.serialize_changeset`: [ { diff --git a/third_party/python/fluent.migrate/fluent/migrate/context.py b/third_party/python/fluent.migrate/fluent/migrate/context.py index 33460c44d01c..251a0ca20695 100644 --- a/third_party/python/fluent.migrate/fluent/migrate/context.py +++ b/third_party/python/fluent.migrate/fluent/migrate/context.py @@ -2,181 +2,72 @@ from __future__ import unicode_literals from __future__ import absolute_import -import os -import codecs -from functools import partial import logging -from six.moves import zip_longest import fluent.syntax.ast as FTL -from fluent.syntax.parser import FluentParser -from fluent.syntax.serializer import FluentSerializer from fluent.migrate.util import fold -from compare_locales.parser import getParser -from compare_locales.plurals import CATEGORIES_BY_LOCALE from .transforms import Source -from .merge import merge_resource -from .util import get_message +from .util import get_message, skeleton from .errors import ( EmptyLocalizationError, UnreadableReferenceError, ) +from ._context import InternalContext -class MergeContext(object): +__all__ = [ + 'EmptyLocalizationError', + 'UnreadableReferenceError', + 'MigrationContext', +] + + +class MigrationContext(InternalContext): """Stateful context for merging translation resources. - `MergeContext` must be configured with the target language and the + `MigrationContext` must be configured with the target locale and the directory locations of the input data. The transformation takes four types of input data: - The en-US FTL reference files which will be used as templates for - message order, comments and sections. + message order, comments and sections. If the reference_dir is None, + the migration will create Messages and Terms in the order given by + the transforms. - - The current FTL files for the given language. - - - The legacy (DTD, properties) translation files for the given - language. The translations from these files will be transformed - into FTL and merged into the existing FTL files for this language. + - The current FTL files for the given locale. - A list of `FTL.Message` or `FTL.Term` objects some of whose nodes are special helper or transform nodes: helpers: VARIABLE_REFERENCE, MESSAGE_REFERENCE, TERM_REFERENCE transforms: COPY, REPLACE_IN_TEXT, REPLACE, PLURALS, CONCAT + fluent value helper: COPY_PATTERN + + The legacy (DTD, properties) translation files are deduced by the + dependencies in the transforms. The translations from these files will be + read from the localization_dir and transformed into FTL and merged + into the existing FTL files for the given language. """ - def __init__(self, lang, reference_dir, localization_dir): - self.fluent_parser = FluentParser(with_spans=False) - self.fluent_serializer = FluentSerializer() - - # An iterable of plural category names relevant to the context's - # language. E.g. ('one', 'other') for English. - try: - self.plural_categories = CATEGORIES_BY_LOCALE[lang] - except KeyError as locale_key: - logger = logging.getLogger('migrate') - logger.warning( - 'Plural rule for "{}" is not defined in ' - 'compare-locales'.format(locale_key)) - self.plural_categories = ('one', 'other') - + def __init__( + self, locale, reference_dir, localization_dir, enforce_translated=False + ): + super(MigrationContext, self).__init__( + locale, reference_dir, localization_dir, + enforce_translated=enforce_translated + ) + self.locale = locale # Paths to directories with input data, relative to CWD. self.reference_dir = reference_dir self.localization_dir = localization_dir - # Parsed input resources stored by resource path. - self.reference_resources = {} - self.localization_resources = {} - self.target_resources = {} - - # An iterable of `FTL.Message` objects some of whose nodes can be the - # transform operations. - self.transforms = {} - # A dict whose keys are `(path, key)` tuples corresponding to target # FTL translations, and values are sets of `(path, key)` tuples # corresponding to localized entities which will be migrated. self.dependencies = {} - def read_ftl_resource(self, path): - """Read an FTL resource and parse it into an AST.""" - f = codecs.open(path, 'r', 'utf8') - try: - contents = f.read() - except UnicodeDecodeError as err: - logger = logging.getLogger('migrate') - logger.warning('Unable to read file {}: {}'.format(path, err)) - raise err - finally: - f.close() - - ast = self.fluent_parser.parse(contents) - - annots = [ - annot - for entry in ast.body - if isinstance(entry, FTL.Junk) - for annot in entry.annotations - ] - - if len(annots): - logger = logging.getLogger('migrate') - for annot in annots: - msg = annot.message - logger.warning('Syntax error in {}: {}'.format(path, msg)) - - return ast - - def read_legacy_resource(self, path): - """Read a legacy resource and parse it into a dict.""" - parser = getParser(path) - parser.readFile(path) - # Transform the parsed result which is an iterator into a dict. - return {entity.key: entity.val for entity in parser} - - def read_reference_ftl(self, path): - """Read and parse a reference FTL file. - - A missing resource file is a fatal error and will raise an - UnreadableReferenceError. - """ - fullpath = os.path.join(self.reference_dir, path) - try: - return self.read_ftl_resource(fullpath) - except IOError: - error_message = 'Missing reference file: {}'.format(fullpath) - logging.getLogger('migrate').error(error_message) - raise UnreadableReferenceError(error_message) - except UnicodeDecodeError as err: - error_message = 'Error reading file {}: {}'.format(fullpath, err) - logging.getLogger('migrate').error(error_message) - raise UnreadableReferenceError(error_message) - - def read_localization_ftl(self, path): - """Read and parse an existing localization FTL file. - - Create a new FTL.Resource if the file doesn't exist or can't be - decoded. - """ - fullpath = os.path.join(self.localization_dir, path) - try: - return self.read_ftl_resource(fullpath) - except IOError: - logger = logging.getLogger('migrate') - logger.info( - 'Localization file {} does not exist and ' - 'it will be created'.format(path)) - return FTL.Resource() - except UnicodeDecodeError: - logger = logging.getLogger('migrate') - logger.warning( - 'Localization file {} has broken encoding. ' - 'It will be re-created and some translations ' - 'may be lost'.format(path)) - return FTL.Resource() - - def maybe_add_localization(self, path): - """Add a localization resource to migrate translations from. - - Uses a compare-locales parser to create a dict of (key, string value) - tuples. - For Fluent sources, we store the AST. - """ - try: - fullpath = os.path.join(self.localization_dir, path) - if not fullpath.endswith('.ftl'): - collection = self.read_legacy_resource(fullpath) - else: - collection = self.read_ftl_resource(fullpath) - except IOError: - logger = logging.getLogger('migrate') - logger.warning('Missing localization file: {}'.format(path)) - else: - self.localization_resources[path] = collection - def add_transforms(self, target, reference, transforms): """Define transforms for target using reference as template. @@ -191,13 +82,26 @@ class MergeContext(object): Each transform is scanned for `Source` nodes which will be used to build the list of dependencies for the transformed message. + + For transforms that merely copy legacy messages or Fluent patterns, + using `fluent.migrate.helpers.transforms_from` is recommended. """ def get_sources(acc, cur): if isinstance(cur, Source): acc.add((cur.path, cur.key)) return acc - reference_ast = self.read_reference_ftl(reference) + if self.reference_dir is None: + # Add skeletons to resource body for each transform + # if there's no reference. + reference_ast = self.reference_resources.get(target) + if reference_ast is None: + reference_ast = FTL.Resource() + reference_ast.body.extend( + skeleton(transform) for transform in transforms + ) + else: + reference_ast = self.read_reference_ftl(reference) self.reference_resources[target] = reference_ast for node in transforms: @@ -210,6 +114,9 @@ class MergeContext(object): # The target Fluent message should exist in the reference file. If # it doesn't, it's probably a typo. + # Of course, only if we're having a reference. + if self.reference_dir is None: + continue if get_message(reference_ast.body, ident) is None: logger = logging.getLogger('migrate') logger.warning( @@ -243,176 +150,3 @@ class MergeContext(object): if target not in self.target_resources: target_ast = self.read_localization_ftl(target) self.target_resources[target] = target_ast - - def get_legacy_source(self, path, key): - """Get an entity value from a localized legacy source. - - Used by the `Source` transform. - """ - resource = self.localization_resources[path] - return resource.get(key, None) - - def get_fluent_source_pattern(self, path, key): - """Get a pattern from a localized Fluent source. - - If the key contains a `.`, does an attribute lookup. - Used by the `COPY_PATTERN` transform. - """ - resource = self.localization_resources[path] - msg_key, _, attr_key = key.partition('.') - found = None - for entry in resource.body: - if isinstance(entry, (FTL.Message, FTL.Term)): - if entry.id.name == msg_key: - found = entry - break - if found is None: - return None - if not attr_key: - return found.value - for attribute in found.attributes: - if attribute.id.name == attr_key: - return attribute.value - return None - - def messages_equal(self, res1, res2): - """Compare messages and terms of two FTL resources. - - Uses FTL.BaseNode.equals to compare all messages/terms - in two FTL resources. - If the order or number of messages differ, the result is also False. - """ - def message_id(message): - "Return the message's identifer name for sorting purposes." - return message.id.name - - messages1 = sorted( - (entry for entry in res1.body - if isinstance(entry, FTL.Message) - or isinstance(entry, FTL.Term)), - key=message_id) - messages2 = sorted( - (entry for entry in res2.body - if isinstance(entry, FTL.Message) - or isinstance(entry, FTL.Term)), - key=message_id) - for msg1, msg2 in zip_longest(messages1, messages2): - if msg1 is None or msg2 is None: - return False - if not msg1.equals(msg2): - return False - return True - - def merge_changeset(self, changeset=None, known_translations=None): - """Return a generator of FTL ASTs for the changeset. - - The input data must be configured earlier using the `add_*` methods. - if given, `changeset` must be a set of (path, key) tuples describing - which legacy translations are to be merged. If `changeset` is None, - all legacy translations will be allowed to be migrated in a single - changeset. - - We use the `in_changeset` method to determine if a message should be - migrated for the given changeset. - - Given `changeset`, return a dict whose keys are resource paths and - values are `FTL.Resource` instances. The values will also be used to - update this context's existing localization resources. - """ - - if changeset is None: - # Merge all known legacy translations. Used in tests. - changeset = { - (path, key) - for path, strings in self.localization_resources.items() - if not path.endswith('.ftl') - for key in strings.keys() - } - - if known_translations is None: - known_translations = changeset - - for path, reference in self.reference_resources.items(): - current = self.target_resources[path] - transforms = self.transforms.get(path, []) - in_changeset = partial( - self.in_changeset, changeset, known_translations, path) - - # Merge legacy translations with the existing ones using the - # reference as a template. - snapshot = merge_resource( - self, reference, current, transforms, in_changeset - ) - - # Skip this path if the messages in the merged snapshot are - # identical to those in the current state of the localization file. - # This may happen when: - # - # - none of the transforms is in the changset, or - # - all messages which would be migrated by the context's - # transforms already exist in the current state. - if self.messages_equal(current, snapshot): - continue - - # Store the merged snapshot on the context so that the next merge - # already takes it into account as the existing localization. - self.target_resources[path] = snapshot - - # The result for this path is a complete `FTL.Resource`. - yield path, snapshot - - def in_changeset(self, changeset, known_translations, path, ident): - """Check if a message should be migrated in this changeset. - - The message is identified by path and ident. - - - A message will be migrated only if all of its dependencies - are present in the currently processed changeset. - - If a transform defined for this message points to a missing - legacy translation, this message will not be merged. The - missing legacy dependency won't be present in the changeset. - - This also means that partially translated messages (e.g. - constructed from two legacy strings out of which only one is - avaiable) will never be migrated. - """ - message_deps = self.dependencies.get((path, ident), None) - - # Don't merge if we don't have a transform for this message. - if message_deps is None: - return False - - # As a special case, if a transform exists but has no - # dependecies, it's a hardcoded `FTL.Node` which doesn't - # migrate any existing translation but rather creates a new - # one. Merge it. - if len(message_deps) == 0: - return True - - # Make sure all the dependencies are present in the current - # changeset. Partial migrations are not currently supported. - # See https://bugzilla.mozilla.org/show_bug.cgi?id=1321271 - # We only return True if our current changeset touches - # the transform, and we have all of the dependencies. - active_deps = message_deps & changeset - available_deps = message_deps & known_translations - return active_deps and message_deps == available_deps - - def serialize_changeset(self, changeset, known_translations=None): - """Return a dict of serialized FTLs for the changeset. - - Given `changeset`, return a dict whose keys are resource paths and - values are serialized FTL snapshots. - """ - - return { - path: self.fluent_serializer.serialize(snapshot) - for path, snapshot in self.merge_changeset( - changeset, known_translations - ) - } - - -logging.basicConfig() diff --git a/third_party/python/fluent.migrate/fluent/migrate/errors.py b/third_party/python/fluent.migrate/fluent/migrate/errors.py index de3262e587cb..dcc3025377af 100644 --- a/third_party/python/fluent.migrate/fluent/migrate/errors.py +++ b/third_party/python/fluent.migrate/fluent/migrate/errors.py @@ -1,3 +1,7 @@ +class SkipTransform(RuntimeError): + pass + + class MigrationError(ValueError): pass diff --git a/third_party/python/fluent.migrate/fluent/migrate/helpers.py b/third_party/python/fluent.migrate/fluent/migrate/helpers.py index 3d22ead32aa7..12d7d4a64e01 100644 --- a/third_party/python/fluent.migrate/fluent/migrate/helpers.py +++ b/third_party/python/fluent.migrate/fluent/migrate/helpers.py @@ -6,7 +6,7 @@ nodes. They take a string argument and immediately return a corresponding AST node. (As opposed to Transforms which are AST nodes on their own and only return the -migrated AST nodes when they are evaluated by a MergeContext.) """ +migrated AST nodes when they are evaluated by a MigrationContext.) """ from __future__ import unicode_literals from __future__ import absolute_import diff --git a/third_party/python/fluent.migrate/fluent/migrate/merge.py b/third_party/python/fluent.migrate/fluent/migrate/merge.py index 65f19e5cb15e..af6c962e98ab 100644 --- a/third_party/python/fluent.migrate/fluent/migrate/merge.py +++ b/third_party/python/fluent.migrate/fluent/migrate/merge.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import fluent.syntax.ast as FTL +from .errors import SkipTransform from .transforms import evaluate from .util import get_message, get_transform @@ -50,7 +51,10 @@ def merge_resource(ctx, reference, current, transforms, in_changeset): if transform is not None and in_changeset(ident): if transform.comment is None: transform.comment = entry.comment - return evaluate(ctx, transform) + try: + return evaluate(ctx, transform) + except SkipTransform: + return None body = merge_body(reference.body) return FTL.Resource(body) diff --git a/third_party/python/fluent.migrate/fluent/migrate/tool.py b/third_party/python/fluent.migrate/fluent/migrate/tool.py old mode 100644 new mode 100755 index 367cb91ec903..555a44f024ad --- a/third_party/python/fluent.migrate/fluent/migrate/tool.py +++ b/third_party/python/fluent.migrate/fluent/migrate/tool.py @@ -10,7 +10,7 @@ import sys import hglib import six -from fluent.migrate.context import MergeContext +from fluent.migrate.context import MigrationContext from fluent.migrate.errors import MigrationError from fluent.migrate.changesets import convert_blame_to_changesets from fluent.migrate.blame import Blame @@ -25,8 +25,8 @@ def dont_write_bytecode(): class Migrator(object): - def __init__(self, language, reference_dir, localization_dir, dry_run): - self.language = language + def __init__(self, locale, reference_dir, localization_dir, dry_run): + self.locale = locale self.reference_dir = reference_dir self.localization_dir = localization_dir self.dry_run = dry_run @@ -45,11 +45,11 @@ class Migrator(object): def run(self, migration): print('\nRunning migration {} for {}'.format( - migration.__name__, self.language)) + migration.__name__, self.locale)) # For each migration create a new context. - ctx = MergeContext( - self.language, self.reference_dir, self.localization_dir + ctx = MigrationContext( + self.locale, self.reference_dir, self.localization_dir ) try: @@ -57,7 +57,7 @@ class Migrator(object): migration.migrate(ctx) except MigrationError as e: print(' Skipping migration {} for {}:\n {}'.format( - migration.__name__, self.language, e)) + migration.__name__, self.locale, e)) return # Keep track of how many changesets we're committing. @@ -125,9 +125,9 @@ class Migrator(object): print(' WARNING: hg commit failed ({})'.format(err)) -def main(lang, reference_dir, localization_dir, migrations, dry_run): +def main(locale, reference_dir, localization_dir, migrations, dry_run): """Run migrations and commit files with the result.""" - migrator = Migrator(lang, reference_dir, localization_dir, dry_run) + migrator = Migrator(locale, reference_dir, localization_dir, dry_run) for migration in migrations: migrator.run(migration) @@ -144,8 +144,8 @@ def cli(): help='migrations to run (Python modules)' ) parser.add_argument( - '--lang', type=str, - help='target language code' + '--locale', '--lang', type=str, + help='target locale code (--lang is deprecated)' ) parser.add_argument( '--reference-dir', type=str, @@ -172,7 +172,7 @@ def cli(): migrations = map(importlib.import_module, args.migrations) main( - lang=args.lang, + locale=args.locale, reference_dir=args.reference_dir, localization_dir=args.localization_dir, migrations=migrations, diff --git a/third_party/python/fluent.migrate/fluent/migrate/transforms.py b/third_party/python/fluent.migrate/fluent/migrate/transforms.py index f9acb2fa3ba6..5d3ff1d64f66 100644 --- a/third_party/python/fluent.migrate/fluent/migrate/transforms.py +++ b/third_party/python/fluent.migrate/fluent/migrate/transforms.py @@ -3,7 +3,7 @@ Transforms are AST nodes which describe how legacy translations should be migrated. They are created inert and only return the migrated AST nodes when -they are evaluated by a MergeContext. +they are evaluated by a MigrationContext. All Transforms evaluate to Fluent Patterns. This makes them suitable for defining migrations of values of message, attributes and variants. The special @@ -13,7 +13,7 @@ elements: TextElements and Placeables. The COPY, REPLACE and PLURALS Transforms inherit from Source which is a special AST Node defining the location (the file path and the id) of the legacy -translation. During the migration, the current MergeContext scans the +translation. During the migration, the current MigrationContext scans the migration spec for Source nodes and extracts the information about all legacy translations being migrated. For instance, @@ -238,6 +238,35 @@ class COPY_PATTERN(FluentSource): pass +class TransformPattern(FluentSource, FTL.Transformer): + """Base class for modifying a Fluent pattern as part of a migration. + + Implement visit_* methods of the Transformer pattern to do the + actual modifications. + """ + def __call__(self, ctx): + pattern = super(TransformPattern, self).__call__(ctx) + return self.visit(pattern) + + def visit_Pattern(self, node): + # Make sure we're creating valid Patterns after restructuring + # transforms. + node = self.generic_visit(node) + pattern = Transform.pattern_of(*node.elements) + return pattern + + def visit_Placeable(self, node): + # Ensure we have a Placeable with an expression still. + # Transforms could have replaced the expression with + # a Pattern or PatternElement, in which case we + # just pass that through. + # Patterns then get flattened by visit_Pattern. + node = self.generic_visit(node) + if isinstance(node.expression, (FTL.Pattern, FTL.PatternElement)): + return node.expression + return node + + class LegacySource(Source): """Declare the source translation to be migrated with other transforms. @@ -253,9 +282,12 @@ class LegacySource(Source): https://github.com/python/cpython/blob/2.7/Lib/htmlentitydefs.py https://github.com/python/cpython/blob/3.6/Lib/html/entities.py + By default, leading and trailing whitespace on each line as well as + leading and trailing empty lines will be stripped from the source + translation's content. Set `trim=False` to disable this behavior. """ - def __init__(self, path, key, trim=False): + def __init__(self, path, key, trim=None): if path.endswith('.ftl'): raise NotSupportedError( 'Please use COPY_PATTERN to migrate from Fluent files ' @@ -269,9 +301,9 @@ class LegacySource(Source): @staticmethod def trim_text(text): - # strip leading white-space + # strip leading white-space from each line text = re.sub('^[ \t]+', '', text, flags=re.M) - # strip trailing white-space + # strip trailing white-space from each line text = re.sub('[ \t]+$', '', text, flags=re.M) # strip leading and trailing empty lines text = text.strip('\r\n') @@ -279,7 +311,7 @@ class LegacySource(Source): def __call__(self, ctx): text = self.get_text(ctx) - if self.trim: + if self.trim is not False: text = self.trim_text(text) return FTL.TextElement(text) @@ -496,7 +528,38 @@ class PLURALS(LegacySource): class CONCAT(Transform): - """Create a new Pattern from Patterns, PatternElements and Expressions.""" + """Create a new Pattern from Patterns, PatternElements and Expressions. + + When called with at least two elements, `CONCAT` disables the trimming + behavior of the elements which are subclasses of `LegacySource` by + setting `trim=False`, unless `trim` has already been set explicitly. The + following two `CONCAT` calls are equivalent: + + CONCAT( + FTL.TextElement("Hello"), + COPY("file.properties", "hello") + ) + + CONCAT( + FTL.TextElement("Hello"), + COPY("file.properties", "hello", trim=False) + ) + + Set `trim=True` explicitly to force trimming: + + CONCAT( + FTL.TextElement("Hello "), + COPY("file.properties", "hello", trim=True) + ) + + When called with a single element and when the element is a subclass of + `LegacySource`, the trimming behavior is not changed. The following two + transforms are equivalent: + + CONCAT(COPY("file.properties", "hello")) + + COPY("file.properties", "hello") + """ def __init__(self, *elements, **kwargs): # We want to support both passing elements as *elements in the @@ -505,5 +568,14 @@ class CONCAT(Transform): # attributes as kwargs. self.elements = list(kwargs.get('elements', elements)) + # We want to make CONCAT(COPY()) equivalent to COPY() so that it's + # always safe (no-op) to wrap transforms in a CONCAT. This is used by + # the implementation of transforms_from. + if len(self.elements) > 1: + for elem in self.elements: + # Only change trim if it hasn't been set explicitly. + if isinstance(elem, LegacySource) and elem.trim is None: + elem.trim = False + def __call__(self, ctx): return Transform.pattern_of(*self.elements) diff --git a/third_party/python/fluent.migrate/fluent/migrate/util.py b/third_party/python/fluent.migrate/fluent/migrate/util.py index 81e2bc26d295..7fcd1c1b5c98 100644 --- a/third_party/python/fluent.migrate/fluent/migrate/util.py +++ b/third_party/python/fluent.migrate/fluent/migrate/util.py @@ -62,6 +62,18 @@ def get_transform(body, ident): return transform +def skeleton(node): + """Create a skeleton copy of the given node. + + For localizable entries, the value is None and the attributes are {}. + That's not a valid Fluent entry, so it requires further manipulation to + set values and/or attributes. + """ + if isinstance(node, LOCALIZABLE_ENTRIES): + return type(node)(id=node.id.clone(), value=None) + return node.clone() + + def ftl(code): """Nicer indentation for FTL code. diff --git a/third_party/python/fluent.migrate/fluent/migrate/validator.py b/third_party/python/fluent.migrate/fluent/migrate/validator.py index 5a3605e8a8ee..87a87990573c 100644 --- a/third_party/python/fluent.migrate/fluent/migrate/validator.py +++ b/third_party/python/fluent.migrate/fluent/migrate/validator.py @@ -21,6 +21,20 @@ class BadContextAPIException(Exception): pass +def process_assign(node, context): + if isinstance(node.value, ast.Str): + val = node.value.s + elif isinstance(node.value, ast.Name): + val = context.get(node.value.id) + elif isinstance(node.value, ast.Call): + val = node.value + if val is None: + return + for target in node.targets: + if isinstance(target, ast.Name): + context[target.id] = val + + class Validator(object): """Validate a migration recipe @@ -55,16 +69,7 @@ class Validator(object): migrate_func = top_level details = self.inspect_migrate(migrate_func, global_assigns) if isinstance(top_level, ast.Assign): - val = None - if isinstance(top_level.value, ast.Str): - val = top_level.value.s - elif isinstance(top_level.value, ast.Name): - val = global_assigns.get(top_level.value.id) - if val is None: - continue - for target in top_level.targets: - if isinstance(target, ast.Name): - global_assigns[target.id] = val + process_assign(top_level, global_assigns) if isinstance(top_level, (ast.Import, ast.ImportFrom)): if 'module' in top_level._fields: module = top_level.module @@ -102,8 +107,7 @@ class Validator(object): visitor = MigrateAnalyzer(ctx_var, global_assigns) visitor.visit(migrate_func) return { - 'sources': visitor.sources, - 'references': visitor.targets, + 'references': visitor.references, 'issues': visitor.issues, } @@ -118,6 +122,9 @@ def full_name(node, global_assigns): return '.'.join(reversed(leafs)) +PATH_TYPES = six.string_types + (ast.Call,) + + class MigrateAnalyzer(ast.NodeVisitor): def __init__(self, ctx_var, global_assigns): super(MigrateAnalyzer, self).__init__() @@ -125,19 +132,23 @@ class MigrateAnalyzer(ast.NodeVisitor): self.global_assigns = global_assigns self.depth = 0 self.issues = [] - self.targets = set() - self.sources = set() + self.references = set() def generic_visit(self, node): self.depth += 1 super(MigrateAnalyzer, self).generic_visit(node) self.depth -= 1 + def visit_Assign(self, node): + if self.depth == 1: + process_assign(node, self.global_assigns) + self.generic_visit(node) + def visit_Attribute(self, node): if isinstance(node.value, ast.Name) and node.value.id == self.ctx_var: if node.attr not in ( - 'maybe_add_localization', 'add_transforms', + 'locale', ): raise BadContextAPIException( 'Unexpected attribute access on {}.{}'.format( @@ -161,8 +172,6 @@ class MigrateAnalyzer(ast.NodeVisitor): self.generic_visit(node) def call_ctx(self, node): - if node.func.attr == 'maybe_add_localization': - return self.call_maybe_add_localization(node) if node.func.attr == 'add_transforms': return self.call_add_transforms(node) raise BadContextAPIException( @@ -171,71 +180,39 @@ class MigrateAnalyzer(ast.NodeVisitor): ) ) - def call_maybe_add_localization(self, node): - self.issues.append({ - 'msg': ( - 'Calling {}.maybe_add_localization is not required' - ).format(self.ctx_var), - 'line': node.lineno - }) - args_msg = ( - 'Expected arguments to {}.maybe_add_localization: ' - 'str' - ).format(self.ctx_var) - if not self.check_arguments(node, ((ast.Str, ast.Name),)): - raise BadContextAPIException(args_msg) - path = node.args[0] - if isinstance(path, ast.Str): - path = path.s - if isinstance(path, ast.Name): - path = self.global_assigns.get(path.id) - if not isinstance(path, six.string_types): - self.issues.append({ - 'msg': args_msg, - 'line': node.args[0].lineno - }) - return - if path != mozpath.normpath(path): - self.issues.append({ - 'msg': ( - 'Argument to {}.maybe_add_localization needs to be a ' - 'normalized path: "{}"' - ).format(self.ctx_var, path), - 'line': node.args[0].lineno - }) - else: - self.sources.add(path) - def call_add_transforms(self, node): args_msg = ( 'Expected arguments to {}.add_transforms: ' - 'path, path, list' + 'target_ftl_path, reference_ftl_path, list_of_transforms' ).format(self.ctx_var) - if not self.check_arguments( - node, - ((ast.Str, ast.Name), (ast.Str, ast.Name), (ast.List, ast.Call)) - ): + ref_msg = ( + 'Expected second argument to {}.add_transforms: ' + 'reference should be string or variable with string value' + ).format(self.ctx_var) + # Just check call signature here, check actual types below + if not self.check_arguments(node, (ast.AST, ast.AST, ast.AST)): self.issues.append({ 'msg': args_msg, 'line': node.lineno, }) return - in_target, in_reference = [ - n.s if isinstance(n, ast.Str) else self.global_assigns.get(n.id) - for n in node.args[:2] - ] - if ( - isinstance(in_target, six.string_types) and - isinstance(in_reference, six.string_types) and - in_target == in_reference - ): - self.targets.add(in_target) - else: + in_reference = node.args[1] + if isinstance(in_reference, ast.Name): + in_reference = self.global_assigns.get(in_reference.id) + if isinstance(in_reference, ast.Str): + in_reference = in_reference.s + if not isinstance(in_reference, six.string_types): self.issues.append({ - 'msg': args_msg, - 'line': node.lineno, + 'msg': ref_msg, + 'line': node.args[1].lineno, }) - self.generic_visit(node) + return + self.references.add(in_reference) + # Checked node.args[1]. + # There's not a lot we can say about our target path, + # ignoring that. + # For our transforms, we want more checks. + self.generic_visit(node.args[2]) def call_transform(self, node, dotted): module, called = dotted.rsplit('.', 1) @@ -259,13 +236,11 @@ class MigrateAnalyzer(ast.NodeVisitor): path = path.s if isinstance(path, ast.Name): path = self.global_assigns.get(path.id) - if not isinstance(path, six.string_types): + if not isinstance(path, PATH_TYPES): self.issues.append({ 'msg': bad_args, 'line': node.lineno }) - return - self.sources.add(path) def call_helpers_transforms_from(self, node): args_msg = ( @@ -288,7 +263,9 @@ class MigrateAnalyzer(ast.NodeVisitor): v = v.s if isinstance(v, ast.Name): v = self.global_assigns.get(v.id) - if not isinstance(v, six.string_types): + if isinstance(v, ast.Call): + v = 'determined at runtime' + if not isinstance(v, PATH_TYPES): msg = 'Bad keyword arg {} to transforms_from'.format( keyword.arg ) @@ -315,7 +292,6 @@ class MigrateAnalyzer(ast.NodeVisitor): 'msg': issue, 'line': node.lineno, } for issue in set(ti.issues)) - self.sources.update(ti.sources) def check_arguments( self, node, argspec, check_kwargs=True, allow_more=False @@ -338,7 +314,6 @@ class MigrateAnalyzer(ast.NodeVisitor): class TransformsInspector(FTL.Visitor): def __init__(self): super(TransformsInspector, self).__init__() - self.sources = set() self.issues = [] def generic_visit(self, node): @@ -350,8 +325,6 @@ class TransformsInspector(FTL.Visitor): self.issues.append( 'Source "{}" needs to be a normalized path'.format(src) ) - else: - self.sources.add(src) super(TransformsInspector, self).generic_visit(node) diff --git a/third_party/python/fluent.migrate/setup.cfg b/third_party/python/fluent.migrate/setup.cfg new file mode 100644 index 000000000000..957dd1650b35 --- /dev/null +++ b/third_party/python/fluent.migrate/setup.cfg @@ -0,0 +1,16 @@ +[options.entry_points] +console_scripts = + migrate-l10n=fluent.migrate.tool:cli + validate-l10n-recipe=fluent.migrate.validator:cli + +[metadata] +long_description = file: README.md +long_description_content_type = text/markdown + +[bdist_wheel] +universal = 1 + +[egg_info] +tag_build = +tag_date = 0 + diff --git a/third_party/python/fluent.migrate/setup.py b/third_party/python/fluent.migrate/setup.py new file mode 100644 index 000000000000..95f8de9cd7a5 --- /dev/null +++ b/third_party/python/fluent.migrate/setup.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python + +from setuptools import setup + +setup( + name='fluent.migrate', + version='0.9', + description='Toolchain to migrate legacy translation to Fluent.', + author='Mozilla', + author_email='l10n-drivers@mozilla.org', + license='APL 2', + url='https://hg.mozilla.org/l10n/fluent-migration/', + keywords=['fluent', 'localization', 'l10n'], + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.7', + ], + packages=['fluent', 'fluent.migrate'], + install_requires=[ + 'compare-locales >=7.6, <8.1', + 'fluent.syntax >=0.17.0, <0.18', + 'six', + ], + extras_require={ + 'hg': ['python-hglib',], + }, + tests_require=[ + 'mock', + ], + test_suite='tests.migrate' +) diff --git a/third_party/python/fluent.syntax/PKG-INFO b/third_party/python/fluent.syntax/PKG-INFO index 942022d34959..08d1ba4c407c 100644 --- a/third_party/python/fluent.syntax/PKG-INFO +++ b/third_party/python/fluent.syntax/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: fluent.syntax -Version: 0.15.1 +Version: 0.17.0 Summary: Localization library for expressive translations. Home-page: https://github.com/projectfluent/python-fluent Author: Mozilla diff --git a/third_party/python/fluent.syntax/fluent/syntax/errors.py b/third_party/python/fluent.syntax/fluent/syntax/errors.py index 58fbbda3c1f2..cd137871b88f 100644 --- a/third_party/python/fluent.syntax/fluent/syntax/errors.py +++ b/third_party/python/fluent.syntax/fluent/syntax/errors.py @@ -65,4 +65,6 @@ def get_error_message(code, args): return 'Unbalanced closing brace in TextElement.' if code == 'E0028': return 'Expected an inline expression' + if code == 'E0029': + return 'Expected simple expression as selector' return code diff --git a/third_party/python/fluent.syntax/fluent/syntax/parser.py b/third_party/python/fluent.syntax/fluent/syntax/parser.py index ece3f0da4a12..eded08d8dfd2 100644 --- a/third_party/python/fluent.syntax/fluent/syntax/parser.py +++ b/third_party/python/fluent.syntax/fluent/syntax/parser.py @@ -508,11 +508,20 @@ class FluentParser(object): else: raise ParseError('E0018') - if ( + elif ( isinstance(selector, ast.TermReference) - and selector.attribute is None ): - raise ParseError('E0017') + if selector.attribute is None: + raise ParseError('E0017') + elif not ( + isinstance(selector, ( + ast.StringLiteral, + ast.NumberLiteral, + ast.VariableReference, + ast.FunctionReference, + )) + ): + raise ParseError('E0029') ps.next() ps.next() @@ -555,17 +564,21 @@ class FluentParser(object): ps.next() attribute = self.get_identifier(ps) arguments = None - if ps.current_char == '(': + ps.peek_blank() + if ps.current_peek == '(': + ps.skip_to_peek() arguments = self.get_call_arguments(ps) return ast.TermReference(id, attribute, arguments) if ps.is_identifier_start(): id = self.get_identifier(ps) + ps.peek_blank() - if ps.current_char == '(': + if ps.current_peek == '(': # It's a Function. Ensure it's all upper-case. if not re.match('^[A-Z][A-Z0-9_-]*$', id.name): raise ParseError('E0008') + ps.skip_to_peek() args = self.get_call_arguments(ps) return ast.FunctionReference(id, args) diff --git a/third_party/python/fluent.syntax/setup.py b/third_party/python/fluent.syntax/setup.py index 9962489d5257..4b4eaceb3218 100755 --- a/third_party/python/fluent.syntax/setup.py +++ b/third_party/python/fluent.syntax/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup(name='fluent.syntax', - version='0.15.1', + version='0.17.0', description='Localization library for expressive translations.', long_description='See https://github.com/projectfluent/python-fluent/ for more info.', author='Mozilla', diff --git a/third_party/python/requirements.in b/third_party/python/requirements.in index c6bb655e352b..bff64117574b 100644 --- a/third_party/python/requirements.in +++ b/third_party/python/requirements.in @@ -19,11 +19,12 @@ attrs==18.1.0 biplist==1.0.3 blessings==1.7 -compare-locales==7.2.5 +compare-locales==8.0.0 cookies==2.2.1 distro==1.4.0 ecdsa==0.15 esprima==4.0.1 +fluent.migrate==0.9 jsmin==2.1.0 json-e==2.7.0 mozilla-version==0.3.0 diff --git a/third_party/python/requirements.txt b/third_party/python/requirements.txt index afb62cfa0411..75ff453bd733 100644 --- a/third_party/python/requirements.txt +++ b/third_party/python/requirements.txt @@ -22,10 +22,10 @@ click==7.0 \ --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \ # via pip-tools -compare-locales==7.2.5 \ - --hash=sha256:9d332ac3cb389724b02d725f724b54393b39499cf70fefcd8267b15b1e5ec281 \ - --hash=sha256:af035b6c2e53689a815e8a0d104a51359b2d45cf5109595348eda03e68ff9bec \ - # via -r requirements-mach-vendor-python.in +compare-locales==8.0.0 \ + --hash=sha256:077b007bd2c025284f73994970e7fada7fbdcbb4199ff010e378b305dee6d469 \ + --hash=sha256:ee02bdad012cdc9f6c6df24d7518ba2c5084f6bac0d176b4826156accc8d48d6 \ + # via -r requirements-mach-vendor-python.in, fluent.migrate cookies==2.2.1 \ --hash=sha256:15bee753002dff684987b8df8c235288eb8d45f8191ae056254812dfd42c81d3 \ --hash=sha256:d6b698788cae4cfa4e62ef8643a9ca332b79bd96cb314294b864ae8d7eb3ee8e \ @@ -41,10 +41,14 @@ ecdsa==0.15 \ esprima==4.0.1 \ --hash=sha256:08db1a876d3c2910db9cfaeb83108193af5411fc3a3a66ebefacd390d21323ee \ # via -r requirements-mach-vendor-python.in -fluent.syntax==0.15.1 \ - --hash=sha256:3d3c95b9de82df498172d9447576e787e809b753c6f7f5acf64a9ef8d7782c81 \ - --hash=sha256:e95bb3abfe7cf51b2c6d1e92d57335f07c737ba57bbbba18e57618bd15f78d4b \ - # via compare-locales +fluent.migrate==0.9 \ + --hash=sha256:735c86816ef7b7b03b32ff9985685f2d99cb0ed135351e4760a85236538f0beb \ + --hash=sha256:d42a001bd7292cef400e63f3d77c0c813a6a6162e7bd2dfa14eb01172d21e788 \ + # via -r requirements-mach-vendor-python.in +fluent.syntax==0.17.0 \ + --hash=sha256:ac3db2f77d62b032fdf1f17ef5c390b7801a9e9fb58d41eca3825c0d47b88d79 \ + --hash=sha256:e26be470aeebe4badd84f7bb0b648414e0f2ef95d26e5336d634af99e402ea61 \ + # via compare-locales, fluent.migrate jsmin==2.1.0 \ --hash=sha256:5d07bf0251a4128e5e8e8eef603849b6b5741c337bff087731a248f9cc774f56 \ # via -r requirements-mach-vendor-python.in