зеркало из https://github.com/mozilla/gecko-dev.git
421 строка
17 KiB
Python
421 строка
17 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import re
|
|
from difflib import SequenceMatcher
|
|
from xml import sax
|
|
try:
|
|
from cStringIO import StringIO
|
|
except ImportError:
|
|
from StringIO import StringIO
|
|
|
|
from compare_locales.parser import DTDParser, PropertiesParser
|
|
|
|
|
|
class Checker(object):
|
|
'''Abstract class to implement checks per file type.
|
|
'''
|
|
pattern = None
|
|
|
|
@classmethod
|
|
def use(cls, file):
|
|
return cls.pattern.match(file.file)
|
|
|
|
def check(self, refEnt, l10nEnt):
|
|
'''Given the reference and localized Entities, performs checks.
|
|
|
|
This is a generator yielding tuples of
|
|
- "warning" or "error", depending on what should be reported,
|
|
- tuple of line, column info for the error within the string
|
|
- description string to be shown in the report
|
|
'''
|
|
if True:
|
|
raise NotImplementedError("Need to subclass")
|
|
yield ("error", (0, 0), "This is an example error", "example")
|
|
|
|
|
|
class PrintfException(Exception):
|
|
def __init__(self, msg, pos):
|
|
self.pos = pos
|
|
self.msg = msg
|
|
|
|
|
|
class PropertiesChecker(Checker):
|
|
'''Tests to run on .properties files.
|
|
'''
|
|
pattern = re.compile('.*\.properties$')
|
|
printf = re.compile(r'%(?P<good>%|'
|
|
r'(?:(?P<number>[1-9][0-9]*)\$)?'
|
|
r'(?P<width>\*|[0-9]+)?'
|
|
r'(?P<prec>\.(?:\*|[0-9]+)?)?'
|
|
r'(?P<spec>[duxXosScpfg]))?')
|
|
|
|
def check(self, refEnt, l10nEnt):
|
|
'''Test for the different variable formats.
|
|
'''
|
|
refValue, l10nValue = refEnt.val, l10nEnt.val
|
|
refSpecs = None
|
|
# check for PluralForm.jsm stuff, should have the docs in the
|
|
# comment
|
|
if 'Localization_and_Plurals' in refEnt.pre_comment:
|
|
# For plurals, common variable pattern is #1. Try that.
|
|
pats = set(int(m.group(1)) for m in re.finditer('#([0-9]+)',
|
|
refValue))
|
|
if len(pats) == 0:
|
|
return
|
|
lpats = set(int(m.group(1)) for m in re.finditer('#([0-9]+)',
|
|
l10nValue))
|
|
if pats - lpats:
|
|
yield ('warning', 0, 'not all variables used in l10n',
|
|
'plural')
|
|
return
|
|
if lpats - pats:
|
|
yield ('error', 0, 'unreplaced variables in l10n',
|
|
'plural')
|
|
return
|
|
return
|
|
# check for lost escapes
|
|
raw_val = l10nEnt.raw_val
|
|
for m in PropertiesParser.escape.finditer(raw_val):
|
|
if m.group('single') and \
|
|
m.group('single') not in PropertiesParser.known_escapes:
|
|
yield ('warning', m.start(),
|
|
'unknown escape sequence, \\' + m.group('single'),
|
|
'escape')
|
|
try:
|
|
refSpecs = self.getPrintfSpecs(refValue)
|
|
except PrintfException:
|
|
refSpecs = []
|
|
if refSpecs:
|
|
for t in self.checkPrintf(refSpecs, l10nValue):
|
|
yield t
|
|
return
|
|
|
|
def checkPrintf(self, refSpecs, l10nValue):
|
|
try:
|
|
l10nSpecs = self.getPrintfSpecs(l10nValue)
|
|
except PrintfException, e:
|
|
yield ('error', e.pos, e.msg, 'printf')
|
|
return
|
|
if refSpecs != l10nSpecs:
|
|
sm = SequenceMatcher()
|
|
sm.set_seqs(refSpecs, l10nSpecs)
|
|
msgs = []
|
|
warn = None
|
|
for action, i1, i2, j1, j2 in sm.get_opcodes():
|
|
if action == 'equal':
|
|
continue
|
|
if action == 'delete':
|
|
# missing argument in l10n
|
|
if i2 == len(refSpecs):
|
|
# trailing specs missing, that's just a warning
|
|
warn = ', '.join('trailing argument %d `%s` missing' %
|
|
(i+1, refSpecs[i])
|
|
for i in xrange(i1, i2))
|
|
else:
|
|
for i in xrange(i1, i2):
|
|
msgs.append('argument %d `%s` missing' %
|
|
(i+1, refSpecs[i]))
|
|
continue
|
|
if action == 'insert':
|
|
# obsolete argument in l10n
|
|
for i in xrange(j1, j2):
|
|
msgs.append('argument %d `%s` obsolete' %
|
|
(i+1, l10nSpecs[i]))
|
|
continue
|
|
if action == 'replace':
|
|
for i, j in zip(xrange(i1, i2), xrange(j1, j2)):
|
|
msgs.append('argument %d `%s` should be `%s`' %
|
|
(j+1, l10nSpecs[j], refSpecs[i]))
|
|
if msgs:
|
|
yield ('error', 0, ', '.join(msgs), 'printf')
|
|
if warn is not None:
|
|
yield ('warning', 0, warn, 'printf')
|
|
|
|
def getPrintfSpecs(self, val):
|
|
hasNumber = False
|
|
specs = []
|
|
for m in self.printf.finditer(val):
|
|
if m.group("good") is None:
|
|
# found just a '%', signal an error
|
|
raise PrintfException('Found single %', m.start())
|
|
if m.group("good") == '%':
|
|
# escaped %
|
|
continue
|
|
if ((hasNumber and m.group('number') is None) or
|
|
(not hasNumber and specs and
|
|
m.group('number') is not None)):
|
|
# mixed style, numbered and not
|
|
raise PrintfException('Mixed ordered and non-ordered args',
|
|
m.start())
|
|
hasNumber = m.group('number') is not None
|
|
if hasNumber:
|
|
pos = int(m.group('number')) - 1
|
|
ls = len(specs)
|
|
if pos >= ls:
|
|
# pad specs
|
|
nones = pos - ls
|
|
specs[ls:pos] = nones*[None]
|
|
specs.append(m.group('spec'))
|
|
else:
|
|
if specs[pos] is not None:
|
|
raise PrintfException('Double ordered argument %d' %
|
|
(pos+1),
|
|
m.start())
|
|
specs[pos] = m.group('spec')
|
|
else:
|
|
specs.append(m.group('spec'))
|
|
# check for missing args
|
|
if hasNumber and not all(specs):
|
|
raise PrintfException('Ordered argument missing', 0)
|
|
return specs
|
|
|
|
|
|
class DTDChecker(Checker):
|
|
"""Tests to run on DTD files.
|
|
|
|
Uses xml.sax for the heavy lifting of xml parsing.
|
|
|
|
The code tries to parse until it doesn't find any unresolved entities
|
|
anymore. If it finds one, it tries to grab the key, and adds an empty
|
|
<!ENTITY key ""> definition to the header.
|
|
|
|
Also checks for some CSS and number heuristics in the values.
|
|
"""
|
|
pattern = re.compile('.*\.dtd$')
|
|
|
|
eref = re.compile('&(%s);' % DTDParser.Name)
|
|
tmpl = '''<!DOCTYPE elem [%s]>
|
|
<elem>%s</elem>
|
|
'''
|
|
xmllist = set(('amp', 'lt', 'gt', 'apos', 'quot'))
|
|
|
|
def __init__(self, reference):
|
|
self.reference = reference
|
|
self.__known_entities = None
|
|
|
|
def known_entities(self, refValue):
|
|
if self.__known_entities is None and self.reference is not None:
|
|
self.__known_entities = set()
|
|
for ent in self.reference:
|
|
self.__known_entities.update(self.entities_for_value(ent.val))
|
|
return self.__known_entities if self.__known_entities is not None \
|
|
else self.entities_for_value(refValue)
|
|
|
|
def entities_for_value(self, value):
|
|
reflist = set(m.group(1).encode('utf-8')
|
|
for m in self.eref.finditer(value))
|
|
reflist -= self.xmllist
|
|
return reflist
|
|
|
|
# Setup for XML parser, with default and text-only content handler
|
|
class TextContent(sax.handler.ContentHandler):
|
|
textcontent = ''
|
|
|
|
def characters(self, content):
|
|
self.textcontent += content
|
|
|
|
defaulthandler = sax.handler.ContentHandler()
|
|
texthandler = TextContent()
|
|
|
|
numPattern = r'([0-9]+|[0-9]*\.[0-9]+)'
|
|
num = re.compile('^%s$' % numPattern)
|
|
lengthPattern = '%s(em|px|ch|cm|in)' % numPattern
|
|
length = re.compile('^%s$' % lengthPattern)
|
|
spec = re.compile(r'((?:min\-)?(?:width|height))\s*:\s*%s' %
|
|
lengthPattern)
|
|
style = re.compile(r'^%(spec)s\s*(;\s*%(spec)s\s*)*;?$' %
|
|
{'spec': spec.pattern})
|
|
|
|
processContent = None
|
|
|
|
def check(self, refEnt, l10nEnt):
|
|
"""Try to parse the refvalue inside a dummy element, and keep
|
|
track of entities that we need to define to make that work.
|
|
|
|
Return a checker that offers just those entities.
|
|
"""
|
|
refValue, l10nValue = refEnt.val, l10nEnt.val
|
|
# find entities the refValue references,
|
|
# reusing markup from DTDParser.
|
|
reflist = self.known_entities(refValue)
|
|
entities = ''.join('<!ENTITY %s "">' % s for s in sorted(reflist))
|
|
parser = sax.make_parser()
|
|
parser.setFeature(sax.handler.feature_external_ges, False)
|
|
|
|
parser.setContentHandler(self.defaulthandler)
|
|
try:
|
|
parser.parse(StringIO(self.tmpl %
|
|
(entities, refValue.encode('utf-8'))))
|
|
# also catch stray %
|
|
parser.parse(StringIO(self.tmpl %
|
|
(refEnt.all.encode('utf-8') + entities,
|
|
'&%s;' % refEnt.key.encode('utf-8'))))
|
|
except sax.SAXParseException, e:
|
|
yield ('warning',
|
|
(0, 0),
|
|
"can't parse en-US value", 'xmlparse')
|
|
|
|
# find entities the l10nValue references,
|
|
# reusing markup from DTDParser.
|
|
l10nlist = self.entities_for_value(l10nValue)
|
|
missing = sorted(l10nlist - reflist)
|
|
_entities = entities + ''.join('<!ENTITY %s "">' % s for s in missing)
|
|
warntmpl = u'Referencing unknown entity `%s`'
|
|
if reflist:
|
|
warntmpl += ' (%s known)' % ', '.join(sorted(reflist))
|
|
if self.processContent is not None:
|
|
self.texthandler.textcontent = ''
|
|
parser.setContentHandler(self.texthandler)
|
|
try:
|
|
parser.parse(StringIO(self.tmpl % (_entities,
|
|
l10nValue.encode('utf-8'))))
|
|
# also catch stray %
|
|
# if this fails, we need to substract the entity definition
|
|
parser.setContentHandler(self.defaulthandler)
|
|
parser.parse(StringIO(self.tmpl % (
|
|
l10nEnt.all.encode('utf-8') + _entities,
|
|
'&%s;' % l10nEnt.key.encode('utf-8'))))
|
|
except sax.SAXParseException, e:
|
|
# xml parse error, yield error
|
|
# sometimes, the error is reported on our fake closing
|
|
# element, make that the end of the last line
|
|
lnr = e.getLineNumber() - 1
|
|
lines = l10nValue.splitlines()
|
|
if lnr > len(lines):
|
|
lnr = len(lines)
|
|
col = len(lines[lnr-1])
|
|
else:
|
|
col = e.getColumnNumber()
|
|
if lnr == 1:
|
|
# first line starts with <elem>, substract
|
|
col -= len("<elem>")
|
|
elif lnr == 0:
|
|
col -= len("<!DOCTYPE elem [") # first line is DOCTYPE
|
|
yield ('error', (lnr, col), ' '.join(e.args), 'xmlparse')
|
|
|
|
for key in missing:
|
|
yield ('warning', (0, 0), warntmpl % key.decode('utf-8'),
|
|
'xmlparse')
|
|
|
|
# Number check
|
|
if self.num.match(refValue) and not self.num.match(l10nValue):
|
|
yield ('warning', 0, 'reference is a number', 'number')
|
|
# CSS checks
|
|
# just a length, width="100em"
|
|
if self.length.match(refValue) and not self.length.match(l10nValue):
|
|
yield ('error', 0, 'reference is a CSS length', 'css')
|
|
# real CSS spec, style="width:100px;"
|
|
if self.style.match(refValue):
|
|
if not self.style.match(l10nValue):
|
|
yield ('error', 0, 'reference is a CSS spec', 'css')
|
|
else:
|
|
# warn if different properties or units
|
|
refMap = dict((s, u) for s, _, u in
|
|
self.spec.findall(refValue))
|
|
msgs = []
|
|
for s, _, u in self.spec.findall(l10nValue):
|
|
if s not in refMap:
|
|
msgs.insert(0, '%s only in l10n' % s)
|
|
continue
|
|
else:
|
|
ru = refMap.pop(s)
|
|
if u != ru:
|
|
msgs.append("units for %s don't match "
|
|
"(%s != %s)" % (s, u, ru))
|
|
for s in refMap.iterkeys():
|
|
msgs.insert(0, '%s only in reference' % s)
|
|
if msgs:
|
|
yield ('warning', 0, ', '.join(msgs), 'css')
|
|
|
|
if self.processContent is not None:
|
|
for t in self.processContent(self.texthandler.textcontent):
|
|
yield t
|
|
|
|
|
|
class PrincessAndroid(DTDChecker):
|
|
"""Checker for the string values that Android puts into an XML container.
|
|
|
|
http://developer.android.com/guide/topics/resources/string-resource.html#FormattingAndStyling # noqa
|
|
has more info. Check for unescaped apostrophes and bad unicode escapes.
|
|
"""
|
|
quoted = re.compile("(?P<q>[\"']).*(?P=q)$")
|
|
|
|
def unicode_escape(self, str):
|
|
"""Helper method to try to decode all unicode escapes in a string.
|
|
|
|
This code uses the standard python decode for unicode-escape, but
|
|
that's somewhat tricky, as its input needs to be ascii. To get to
|
|
ascii, the unicode string gets converted to ascii with
|
|
backslashreplace, i.e., all non-ascii unicode chars get unicode
|
|
escaped. And then we try to roll all of that back.
|
|
Now, when that hits an error, that's from the original string, and we
|
|
need to search for the actual error position in the original string,
|
|
as the backslashreplace code changes string positions quite badly.
|
|
See also the last check in TestAndroid.test_android_dtd, with a
|
|
lengthy chinese string.
|
|
"""
|
|
val = str.encode('ascii', 'backslashreplace')
|
|
try:
|
|
val.decode('unicode-escape')
|
|
except UnicodeDecodeError, e:
|
|
args = list(e.args)
|
|
badstring = args[1][args[2]:args[3]]
|
|
i = len(args[1][:args[2]].decode('unicode-escape'))
|
|
args[2] = i
|
|
args[3] = i + len(badstring)
|
|
raise UnicodeDecodeError(*args)
|
|
|
|
@classmethod
|
|
def use(cls, file):
|
|
"""Use this Checker only for DTD files in embedding/android."""
|
|
return (file.module in ("embedding/android",
|
|
"mobile/android/base")
|
|
and cls.pattern.match(file.file))
|
|
|
|
def processContent(self, val):
|
|
"""Actual check code.
|
|
Check for unicode escapes and unescaped quotes and apostrophes,
|
|
if string's not quoted.
|
|
"""
|
|
# first, try to decode unicode escapes
|
|
try:
|
|
self.unicode_escape(val)
|
|
except UnicodeDecodeError, e:
|
|
yield ('error', e.args[2], e.args[4], 'android')
|
|
# check for unescaped single or double quotes.
|
|
# first, see if the complete string is single or double quoted,
|
|
# that changes the rules
|
|
m = self.quoted.match(val)
|
|
if m:
|
|
q = m.group('q')
|
|
offset = 0
|
|
val = val[1:-1] # strip quotes
|
|
else:
|
|
q = "[\"']"
|
|
offset = -1
|
|
stray_quot = re.compile(r"[\\\\]*(%s)" % q)
|
|
|
|
for m in stray_quot.finditer(val):
|
|
if len(m.group(0)) % 2:
|
|
# found an unescaped single or double quote, which message?
|
|
if m.group(1) == '"':
|
|
msg = u"Quotes in Android DTDs need escaping with \\\" "\
|
|
u"or \\u0022, or put string in apostrophes."
|
|
else:
|
|
msg = u"Apostrophes in Android DTDs need escaping with "\
|
|
u"\\' or \\u0027, or use \u2019, or put string in "\
|
|
u"quotes."
|
|
yield ('error', m.end(0)+offset, msg, 'android')
|
|
|
|
|
|
def getChecker(file, reference=None):
|
|
if PropertiesChecker.use(file):
|
|
return PropertiesChecker()
|
|
if PrincessAndroid.use(file):
|
|
return PrincessAndroid(reference)
|
|
if DTDChecker.use(file):
|
|
return DTDChecker(reference)
|
|
return None
|