зеркало из https://github.com/mozilla/gecko-dev.git
522 строки
16 KiB
Python
522 строки
16 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import re
|
|
import codecs
|
|
import logging
|
|
from HTMLParser import HTMLParser
|
|
|
|
__constructors = []
|
|
|
|
|
|
class Entity(object):
|
|
'''
|
|
Abstraction layer for a localizable entity.
|
|
Currently supported are grammars of the form:
|
|
|
|
1: pre white space
|
|
2: pre comments
|
|
3: entity definition
|
|
4: entity key (name)
|
|
5: entity value
|
|
6: post comment (and white space) in the same line (dtd only)
|
|
<--[1]
|
|
<!-- pre comments --> <--[2]
|
|
<!ENTITY key "value"> <!-- comment -->
|
|
|
|
<-------[3]---------><------[6]------>
|
|
'''
|
|
def __init__(self, contents, pp,
|
|
span, pre_ws_span, pre_comment_span, def_span,
|
|
key_span, val_span, post_span):
|
|
self.contents = contents
|
|
self.span = span
|
|
self.pre_ws_span = pre_ws_span
|
|
self.pre_comment_span = pre_comment_span
|
|
self.def_span = def_span
|
|
self.key_span = key_span
|
|
self.val_span = val_span
|
|
self.post_span = post_span
|
|
self.pp = pp
|
|
pass
|
|
|
|
# getter helpers
|
|
|
|
def get_all(self):
|
|
return self.contents[self.span[0]:self.span[1]]
|
|
|
|
def get_pre_ws(self):
|
|
return self.contents[self.pre_ws_span[0]:self.pre_ws_span[1]]
|
|
|
|
def get_pre_comment(self):
|
|
return self.contents[self.pre_comment_span[0]:
|
|
self.pre_comment_span[1]]
|
|
|
|
def get_def(self):
|
|
return self.contents[self.def_span[0]:self.def_span[1]]
|
|
|
|
def get_key(self):
|
|
return self.contents[self.key_span[0]:self.key_span[1]]
|
|
|
|
def get_val(self):
|
|
return self.pp(self.contents[self.val_span[0]:self.val_span[1]])
|
|
|
|
def get_raw_val(self):
|
|
return self.contents[self.val_span[0]:self.val_span[1]]
|
|
|
|
def get_post(self):
|
|
return self.contents[self.post_span[0]:self.post_span[1]]
|
|
|
|
# getters
|
|
|
|
all = property(get_all)
|
|
pre_ws = property(get_pre_ws)
|
|
pre_comment = property(get_pre_comment)
|
|
definition = property(get_def)
|
|
key = property(get_key)
|
|
val = property(get_val)
|
|
raw_val = property(get_raw_val)
|
|
post = property(get_post)
|
|
|
|
def __repr__(self):
|
|
return self.key
|
|
|
|
|
|
class Junk(object):
|
|
'''
|
|
An almost-Entity, representing junk data that we didn't parse.
|
|
This way, we can signal bad content as stuff we don't understand.
|
|
And the either fix that, or report real bugs in localizations.
|
|
'''
|
|
junkid = 0
|
|
|
|
def __init__(self, contents, span):
|
|
self.contents = contents
|
|
self.span = span
|
|
self.pre_ws = self.pre_comment = self.definition = self.post = ''
|
|
self.__class__.junkid += 1
|
|
self.key = '_junk_%d_%d-%d' % (self.__class__.junkid, span[0], span[1])
|
|
|
|
# getter helpers
|
|
def get_all(self):
|
|
return self.contents[self.span[0]:self.span[1]]
|
|
|
|
# getters
|
|
all = property(get_all)
|
|
val = property(get_all)
|
|
|
|
def __repr__(self):
|
|
return self.key
|
|
|
|
|
|
class Parser:
|
|
canMerge = True
|
|
|
|
def __init__(self):
|
|
if not hasattr(self, 'encoding'):
|
|
self.encoding = 'utf-8'
|
|
pass
|
|
|
|
def readFile(self, file):
|
|
f = codecs.open(file, 'r', self.encoding)
|
|
try:
|
|
self.contents = f.read()
|
|
except UnicodeDecodeError, e:
|
|
(logging.getLogger('locales')
|
|
.error("Can't read file: " + file + '; ' + str(e)))
|
|
self.contents = u''
|
|
f.close()
|
|
|
|
def readContents(self, contents):
|
|
(self.contents, length) = codecs.getdecoder(self.encoding)(contents)
|
|
|
|
def parse(self):
|
|
l = []
|
|
m = {}
|
|
for e in self:
|
|
m[e.key] = len(l)
|
|
l.append(e)
|
|
return (l, m)
|
|
|
|
def postProcessValue(self, val):
|
|
return val
|
|
|
|
def __iter__(self):
|
|
contents = self.contents
|
|
offset = 0
|
|
self.header, offset = self.getHeader(contents, offset)
|
|
self.footer = ''
|
|
entity, offset = self.getEntity(contents, offset)
|
|
while entity:
|
|
yield entity
|
|
entity, offset = self.getEntity(contents, offset)
|
|
f = self.reFooter.match(contents, offset)
|
|
if f:
|
|
self.footer = f.group()
|
|
offset = f.end()
|
|
if len(contents) > offset:
|
|
yield Junk(contents, (offset, len(contents)))
|
|
pass
|
|
|
|
def getHeader(self, contents, offset):
|
|
header = ''
|
|
h = self.reHeader.match(contents)
|
|
if h:
|
|
header = h.group()
|
|
offset = h.end()
|
|
return (header, offset)
|
|
|
|
def getEntity(self, contents, offset):
|
|
m = self.reKey.match(contents, offset)
|
|
if m:
|
|
offset = m.end()
|
|
entity = self.createEntity(contents, m)
|
|
return (entity, offset)
|
|
# first check if footer has a non-empy match,
|
|
# 'cause then we don't find junk
|
|
m = self.reFooter.match(contents, offset)
|
|
if m and m.end() > offset:
|
|
return (None, offset)
|
|
m = self.reKey.search(contents, offset)
|
|
if m:
|
|
# we didn't match, but search, so there's junk between offset
|
|
# and start. We'll match() on the next turn
|
|
junkend = m.start()
|
|
return (Junk(contents, (offset, junkend)), junkend)
|
|
return (None, offset)
|
|
|
|
def createEntity(self, contents, m):
|
|
return Entity(contents, self.postProcessValue,
|
|
*[m.span(i) for i in xrange(7)])
|
|
|
|
|
|
def getParser(path):
|
|
for item in __constructors:
|
|
if re.search(item[0], path):
|
|
return item[1]
|
|
raise UserWarning("Cannot find Parser")
|
|
|
|
|
|
# Subgroups of the match will:
|
|
# 1: pre white space
|
|
# 2: pre comments
|
|
# 3: entity definition
|
|
# 4: entity key (name)
|
|
# 5: entity value
|
|
# 6: post comment (and white space) in the same line (dtd only)
|
|
# <--[1]
|
|
# <!-- pre comments --> <--[2]
|
|
# <!ENTITY key "value"> <!-- comment -->
|
|
#
|
|
# <-------[3]---------><------[6]------>
|
|
|
|
|
|
class DTDParser(Parser):
|
|
# http://www.w3.org/TR/2006/REC-xml11-20060816/#NT-NameStartChar
|
|
# ":" | [A-Z] | "_" | [a-z] |
|
|
# [#xC0-#xD6] | [#xD8-#xF6] | [#xF8-#x2FF] | [#x370-#x37D] | [#x37F-#x1FFF]
|
|
# | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] |
|
|
# [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] |
|
|
# [#x10000-#xEFFFF]
|
|
CharMinusDash = u'\x09\x0A\x0D\u0020-\u002C\u002E-\uD7FF\uE000-\uFFFD'
|
|
XmlComment = '<!--(?:-?[%s])*?-->' % CharMinusDash
|
|
NameStartChar = u':A-Z_a-z\xC0-\xD6\xD8-\xF6\xF8-\u02FF' + \
|
|
u'\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F' + \
|
|
u'\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD'
|
|
# + \U00010000-\U000EFFFF seems to be unsupported in python
|
|
|
|
# NameChar ::= NameStartChar | "-" | "." | [0-9] | #xB7 |
|
|
# [#x0300-#x036F] | [#x203F-#x2040]
|
|
NameChar = NameStartChar + ur'\-\.0-9' + u'\xB7\u0300-\u036F\u203F-\u2040'
|
|
Name = '[' + NameStartChar + '][' + NameChar + ']*'
|
|
reKey = re.compile('(?:(?P<pre>\s*)(?P<precomment>(?:' + XmlComment +
|
|
'\s*)*)(?P<entity><!ENTITY\s+(?P<key>' + Name +
|
|
')\s+(?P<val>\"[^\"]*\"|\'[^\']*\'?)\s*>)'
|
|
'(?P<post>[ \t]*(?:' + XmlComment + '\s*)*\n?)?)',
|
|
re.DOTALL)
|
|
# add BOM to DTDs, details in bug 435002
|
|
reHeader = re.compile(u'^\ufeff?'
|
|
u'(\s*<!--.*(http://mozilla.org/MPL/2.0/|'
|
|
u'LICENSE BLOCK)([^-]+-)*[^-]+-->)?', re.S)
|
|
reFooter = re.compile('\s*(<!--([^-]+-)*[^-]+-->\s*)*$')
|
|
rePE = re.compile('(?:(\s*)((?:' + XmlComment + '\s*)*)'
|
|
'(<!ENTITY\s+%\s+(' + Name +
|
|
')\s+SYSTEM\s+(\"[^\"]*\"|\'[^\']*\')\s*>\s*%' + Name +
|
|
';)([ \t]*(?:' + XmlComment + '\s*)*\n?)?)')
|
|
|
|
def getEntity(self, contents, offset):
|
|
'''
|
|
Overload Parser.getEntity to special-case ParsedEntities.
|
|
Just check for a parsed entity if that method claims junk.
|
|
|
|
<!ENTITY % foo SYSTEM "url">
|
|
%foo;
|
|
'''
|
|
entity, inneroffset = Parser.getEntity(self, contents, offset)
|
|
if (entity and isinstance(entity, Junk)) or entity is None:
|
|
m = self.rePE.match(contents, offset)
|
|
if m:
|
|
inneroffset = m.end()
|
|
entity = Entity(contents, self.postProcessValue,
|
|
*[m.span(i) for i in xrange(7)])
|
|
return (entity, inneroffset)
|
|
|
|
def createEntity(self, contents, m):
|
|
valspan = m.span('val')
|
|
valspan = (valspan[0]+1, valspan[1]-1)
|
|
return Entity(contents, self.postProcessValue, m.span(),
|
|
m.span('pre'), m.span('precomment'),
|
|
m.span('entity'), m.span('key'), valspan,
|
|
m.span('post'))
|
|
|
|
|
|
class PropertiesParser(Parser):
|
|
escape = re.compile(r'\\((?P<uni>u[0-9a-fA-F]{1,4})|'
|
|
'(?P<nl>\n\s*)|(?P<single>.))', re.M)
|
|
known_escapes = {'n': '\n', 'r': '\r', 't': '\t', '\\': '\\'}
|
|
|
|
def __init__(self):
|
|
self.reKey = re.compile('^(\s*)'
|
|
'((?:[#!].*?\n\s*)*)'
|
|
'([^#!\s\n][^=:\n]*?)\s*[:=][ \t]*', re.M)
|
|
self.reHeader = re.compile('^\s*([#!].*\s*)+')
|
|
self.reFooter = re.compile('\s*([#!].*\s*)*$')
|
|
self._escapedEnd = re.compile(r'\\+$')
|
|
self._trailingWS = re.compile(r'[ \t]*$')
|
|
Parser.__init__(self)
|
|
|
|
def getHeader(self, contents, offset):
|
|
header = ''
|
|
h = self.reHeader.match(contents, offset)
|
|
if h:
|
|
candidate = h.group()
|
|
if 'http://mozilla.org/MPL/2.0/' in candidate or \
|
|
'LICENSE BLOCK' in candidate:
|
|
header = candidate
|
|
offset = h.end()
|
|
return (header, offset)
|
|
|
|
def getEntity(self, contents, offset):
|
|
# overwritten to parse values line by line
|
|
m = self.reKey.match(contents, offset)
|
|
if m:
|
|
offset = m.end()
|
|
while True:
|
|
endval = nextline = contents.find('\n', offset)
|
|
if nextline == -1:
|
|
endval = offset = len(contents)
|
|
break
|
|
# is newline escaped?
|
|
_e = self._escapedEnd.search(contents, offset, nextline)
|
|
offset = nextline + 1
|
|
if _e is None:
|
|
break
|
|
# backslashes at end of line, if 2*n, not escaped
|
|
if len(_e.group()) % 2 == 0:
|
|
break
|
|
# strip trailing whitespace
|
|
ws = self._trailingWS.search(contents, m.end(), offset)
|
|
if ws:
|
|
endval -= ws.end() - ws.start()
|
|
entity = Entity(contents, self.postProcessValue,
|
|
(m.start(), offset), # full span
|
|
m.span(1), # leading whitespan
|
|
m.span(2), # leading comment span
|
|
(m.start(3), offset), # entity def span
|
|
m.span(3), # key span
|
|
(m.end(), endval), # value span
|
|
(offset, offset)) # post comment span, empty
|
|
return (entity, offset)
|
|
m = self.reKey.search(contents, offset)
|
|
if m:
|
|
# we didn't match, but search, so there's junk between offset
|
|
# and start. We'll match() on the next turn
|
|
junkend = m.start()
|
|
return (Junk(contents, (offset, junkend)), junkend)
|
|
return (None, offset)
|
|
|
|
def postProcessValue(self, val):
|
|
|
|
def unescape(m):
|
|
found = m.groupdict()
|
|
if found['uni']:
|
|
return unichr(int(found['uni'][1:], 16))
|
|
if found['nl']:
|
|
return ''
|
|
return self.known_escapes.get(found['single'], found['single'])
|
|
val = self.escape.sub(unescape, val)
|
|
return val
|
|
|
|
|
|
class DefinesParser(Parser):
|
|
# can't merge, #unfilter needs to be the last item, which we don't support
|
|
canMerge = False
|
|
|
|
def __init__(self):
|
|
self.reKey = re.compile('^(\s*)((?:^#(?!define\s).*\s*)*)'
|
|
'(#define[ \t]+(\w+)[ \t]+(.*?))([ \t]*$\n?)',
|
|
re.M)
|
|
self.reHeader = re.compile('^\s*(#(?!define\s).*\s*)*')
|
|
self.reFooter = re.compile('\s*(#(?!define\s).*\s*)*$', re.M)
|
|
Parser.__init__(self)
|
|
|
|
|
|
class IniParser(Parser):
|
|
'''
|
|
Parse files of the form:
|
|
# initial comment
|
|
[cat]
|
|
whitespace*
|
|
#comment
|
|
string=value
|
|
...
|
|
'''
|
|
def __init__(self):
|
|
self.reHeader = re.compile('^((?:\s*|[;#].*)\n)*\[.+?\]\n', re.M)
|
|
self.reKey = re.compile('(\s*)((?:[;#].*\n\s*)*)((.+?)=(.*))(\n?)')
|
|
self.reFooter = re.compile('\s*')
|
|
Parser.__init__(self)
|
|
|
|
|
|
DECL, COMMENT, START, END, CONTENT = range(5)
|
|
|
|
|
|
class BookmarksParserInner(HTMLParser):
|
|
|
|
class Token(object):
|
|
_type = None
|
|
content = ''
|
|
|
|
def __str__(self):
|
|
return self.content
|
|
|
|
class DeclToken(Token):
|
|
_type = DECL
|
|
|
|
def __init__(self, decl):
|
|
self.content = decl
|
|
pass
|
|
|
|
def __str__(self):
|
|
return '<!%s>' % self.content
|
|
pass
|
|
|
|
class CommentToken(Token):
|
|
_type = COMMENT
|
|
|
|
def __init__(self, comment):
|
|
self.content = comment
|
|
pass
|
|
|
|
def __str__(self):
|
|
return '<!--%s-->' % self.content
|
|
pass
|
|
|
|
class StartToken(Token):
|
|
_type = START
|
|
|
|
def __init__(self, tag, attrs, content):
|
|
self.tag = tag
|
|
self.attrs = dict(attrs)
|
|
self.content = content
|
|
pass
|
|
pass
|
|
|
|
class EndToken(Token):
|
|
_type = END
|
|
|
|
def __init__(self, tag):
|
|
self.tag = tag
|
|
pass
|
|
|
|
def __str__(self):
|
|
return '</%s>' % self.tag.upper()
|
|
pass
|
|
|
|
class ContentToken(Token):
|
|
_type = CONTENT
|
|
|
|
def __init__(self, content):
|
|
self.content = content
|
|
pass
|
|
pass
|
|
|
|
def __init__(self):
|
|
HTMLParser.__init__(self)
|
|
self.tokens = []
|
|
|
|
def parse(self, contents):
|
|
self.tokens = []
|
|
self.feed(contents)
|
|
self.close()
|
|
return self.tokens
|
|
|
|
# Called when we hit an end DL tag to reset the folder selections
|
|
def handle_decl(self, decl):
|
|
self.tokens.append(self.DeclToken(decl))
|
|
|
|
# Called when we hit an end DL tag to reset the folder selections
|
|
def handle_comment(self, comment):
|
|
self.tokens.append(self.CommentToken(comment))
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
self.tokens.append(self.StartToken(tag, attrs,
|
|
self.get_starttag_text()))
|
|
|
|
# Called when text data is encountered
|
|
def handle_data(self, data):
|
|
if self.tokens[-1]._type == CONTENT:
|
|
self.tokens[-1].content += data
|
|
else:
|
|
self.tokens.append(self.ContentToken(data))
|
|
|
|
def handle_charref(self, data):
|
|
self.handle_data('&#%s;' % data)
|
|
|
|
def handle_entityref(self, data):
|
|
self.handle_data('&%s;' % data)
|
|
|
|
# Called when we hit an end DL tag to reset the folder selections
|
|
def handle_endtag(self, tag):
|
|
self.tokens.append(self.EndToken(tag))
|
|
|
|
|
|
class BookmarksParser(Parser):
|
|
canMerge = False
|
|
|
|
class BMEntity(object):
|
|
def __init__(self, key, val):
|
|
self.key = key
|
|
self.val = val
|
|
|
|
def __iter__(self):
|
|
p = BookmarksParserInner()
|
|
tks = p.parse(self.contents)
|
|
i = 0
|
|
k = []
|
|
for i in xrange(len(tks)):
|
|
t = tks[i]
|
|
if t._type == START:
|
|
k.append(t.tag)
|
|
keys = t.attrs.keys()
|
|
keys.sort()
|
|
for attrname in keys:
|
|
yield self.BMEntity('.'.join(k) + '.@' + attrname,
|
|
t.attrs[attrname])
|
|
if i + 1 < len(tks) and tks[i+1]._type == CONTENT:
|
|
i += 1
|
|
t = tks[i]
|
|
v = t.content.strip()
|
|
if v:
|
|
yield self.BMEntity('.'.join(k), v)
|
|
elif t._type == END:
|
|
k.pop()
|
|
|
|
|
|
__constructors = [('\\.dtd$', DTDParser()),
|
|
('\\.properties$', PropertiesParser()),
|
|
('\\.ini$', IniParser()),
|
|
('\\.inc$', DefinesParser()),
|
|
('bookmarks\\.html$', BookmarksParser())]
|