зеркало из https://github.com/mozilla/bedrock.git
rewrite l10n_check command to generate locale files with else statements
This commit is contained in:
James Long
Родитель
3840410244
Коммит
f984637ba9
|
|
@ -3,14 +3,12 @@
|
|||
{% block content %}
|
||||
<h1>Hello world!</h1>
|
||||
|
||||
{% l10n foo, 20110630 %}
|
||||
<p>Localizable, unversioned.</p>
|
||||
{% l10n foo, 20110915 %}
|
||||
<p>This needs localization!</p>
|
||||
{% endl10n %}
|
||||
|
||||
{% l10n bar, 20110715 %}
|
||||
<p>Localizable, versioned.</p>
|
||||
{% else %}
|
||||
<p>This should always be shown.</p>
|
||||
{% l10n bar, 20110910 %}
|
||||
<p>This was changed a while ago.</p>
|
||||
{% endl10n %}
|
||||
|
||||
{#
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ for error and avoids unnecessary code duplication across various locales.
|
|||
|
||||
.. figure:: images/l10n-blocks.jpg
|
||||
:alt: L10n blocks example
|
||||
|
||||
n
|
||||
This is an example of what parts of a page would be expressed as localizable
|
||||
blocks.
|
||||
|
||||
|
|
@ -123,3 +123,34 @@ Done! The template engine will automatically use these translated blocks and
|
|||
put them into the source template in the appropriate places when rendering
|
||||
the German version of this page.
|
||||
|
||||
Generating the locale files
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
./manage.py l10n_check
|
||||
|
||||
This command will check which blocks need to be translated and update
|
||||
the locale templates with needed translations. It will copy the
|
||||
English blocks into the locale files if a translation is needed.
|
||||
|
||||
It uses the version of a block to determine if a translation is
|
||||
needed. You need to update this version (which is a date in the format
|
||||
YYYYMMDD) if you want it to be re-localized.
|
||||
|
||||
New blocks will simply appear in english in the locale files. For
|
||||
example, in the German template, it will look like:
|
||||
|
||||
{% l10n foo %}
|
||||
<h1>This is an English string that needs translating.</h1>
|
||||
{% endl10n %}
|
||||
|
||||
If there was a previous translation, it will be kept in the file so
|
||||
the the page will still display it:
|
||||
|
||||
{% l10n foo %}
|
||||
<h1>This is an English string that needs translating.</h1>
|
||||
{% else %}
|
||||
<h1>Dies ist ein English string wurde nicht.</h1>
|
||||
{% endl10n %}
|
||||
|
||||
The localizer needs to translate the English string and remove the
|
||||
else block and previous translation.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import datetime
|
||||
import itertools
|
||||
import os
|
||||
import re
|
||||
import os, errno
|
||||
from os import path
|
||||
from optparse import make_option
|
||||
import codecs
|
||||
|
|
@ -7,11 +9,234 @@ import codecs
|
|||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.conf import settings
|
||||
|
||||
from jingo import get_env
|
||||
from jinja2 import Environment, TemplateNotFound
|
||||
from jinja2.parser import Parser
|
||||
|
||||
|
||||
def l10n_file(*args):
|
||||
return path.join(settings.ROOT, 'locale', *args)
|
||||
|
||||
|
||||
def l10n_tmpl(tmpl, lang):
|
||||
return l10n_file(lang, 'templates', tmpl)
|
||||
|
||||
|
||||
def app_tmpl(tmpl):
|
||||
app = tmpl[0:tmpl.index('/')]
|
||||
return path.join(settings.ROOT, 'apps', app, 'templates', tmpl)
|
||||
|
||||
|
||||
def list_templates():
|
||||
"""List all the templates in all the installed apps"""
|
||||
|
||||
for app in settings.INSTALLED_APPS:
|
||||
tmpl_dir = path.join(settings.ROOT, 'apps', app, 'templates')
|
||||
|
||||
if path.exists(tmpl_dir):
|
||||
# Find all the .html files
|
||||
for root, dirs, files in os.walk(tmpl_dir):
|
||||
for filename in files:
|
||||
name, ext = os.path.splitext(filename)
|
||||
|
||||
if ext == '.html':
|
||||
full_path = os.path.join(root, filename)
|
||||
yield full_path.replace(tmpl_dir, '').lstrip('/')
|
||||
|
||||
|
||||
def update_templates(langs):
|
||||
"""List templates with outdated/incorrect l10n blocks"""
|
||||
|
||||
for tmpl in list_templates():
|
||||
for lang in langs:
|
||||
if path.exists(l10n_tmpl(tmpl, lang)):
|
||||
update_template(tmpl, lang)
|
||||
else:
|
||||
copy_template(tmpl, lang)
|
||||
|
||||
|
||||
def update_template(tmpl, lang):
|
||||
"""Detect outdated/incorrect l10n block and notify"""
|
||||
|
||||
parser = L10nParser()
|
||||
blocks = parser.parse_template(app_tmpl(tmpl))
|
||||
|
||||
t = l10n_tmpl(tmpl, lang)
|
||||
l10n_blocks = parser.parse_template(t, strict=False)
|
||||
l10n_version = parser.parse_tmpl_version(t)
|
||||
|
||||
for name, data in blocks.iteritems():
|
||||
if name in l10n_blocks:
|
||||
old = l10n_blocks[name]
|
||||
|
||||
if l10n_version < data['version']:
|
||||
# Move the main content to the else content only if it
|
||||
# doesn't already exist, and then update the main content
|
||||
if not old['else_content']:
|
||||
old['else_content'] = old['main_content']
|
||||
old['main_content'] = data['main_content']
|
||||
else:
|
||||
l10n_blocks[name] = data
|
||||
|
||||
write_l10n_template(l10n_blocks, tmpl, lang)
|
||||
|
||||
|
||||
def write_l10n_template(blocks, tmpl, lang):
|
||||
"""Write out blocks to an l10n template"""
|
||||
dest = l10n_tmpl(tmpl, lang)
|
||||
|
||||
# Make sure the template dir exists
|
||||
try:
|
||||
os.makedirs(os.path.dirname(dest))
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EEXIST: pass
|
||||
else: raise
|
||||
|
||||
with open(dest, 'w+') as file:
|
||||
today = datetime.date.today()
|
||||
file.write('{# Version: %s #}\n' % today.strftime('%Y%m%d'))
|
||||
file.write('{# DO NOT MODIFY THE ABOVE LINE #}\n\n')
|
||||
|
||||
for (name, data) in blocks.iteritems():
|
||||
file.write('{%% l10n %s %%}\n' % name)
|
||||
file.write(data['main_content'])
|
||||
if(data['else_content']):
|
||||
file.write('\n{% else %}\n')
|
||||
file.write(data['else_content'])
|
||||
file.write('\n{% endl10n %}\n\n')
|
||||
|
||||
|
||||
def copy_template(tmpl, lang):
|
||||
"""Create a new l10n template by copying the l10n blocks"""
|
||||
|
||||
parser = L10nParser()
|
||||
blocks = parser.parse_template(app_tmpl(tmpl))
|
||||
if blocks:
|
||||
write_l10n_template(blocks, tmpl, lang)
|
||||
|
||||
|
||||
class L10nParser():
|
||||
|
||||
file_version_re = re.compile('{# Version: (\d+) #}')
|
||||
|
||||
def __init__(self):
|
||||
self.tmpl = None
|
||||
|
||||
def parse_tmpl_version(self, tmpl):
|
||||
line = codecs.open(tmpl, encoding='utf-8').readline().strip()
|
||||
matches = self.file_version_re.match(line)
|
||||
if matches:
|
||||
return int(matches.group(1))
|
||||
return None
|
||||
|
||||
def parse_template(self, tmpl, strict=True):
|
||||
"""Read a template and parse the l10n blocks"""
|
||||
self.tmpl = tmpl
|
||||
return self.parse(codecs.open(tmpl, encoding='utf-8').read(),
|
||||
strict=strict)
|
||||
|
||||
def parse(self, src, strict=True):
|
||||
"""Analyze a template and get the l10n block information"""
|
||||
|
||||
env = Environment()
|
||||
self.tokens = env.lex(src)
|
||||
blocks = {}
|
||||
|
||||
for name, version in self.parse_blocks(strict):
|
||||
main_content, else_content = self.block_content()
|
||||
|
||||
blocks[name] = {'version': version,
|
||||
'main_content': main_content,
|
||||
'else_content': else_content}
|
||||
|
||||
return blocks
|
||||
|
||||
def parse_blocks(self, strict):
|
||||
"""Extract tags from the l10n block"""
|
||||
|
||||
while self.scan_until('block_begin'):
|
||||
self.scan_ignore('whitespace')
|
||||
name = self.scan_next('name')
|
||||
|
||||
if name != 'l10n':
|
||||
continue
|
||||
|
||||
self.scan_ignore('whitespace')
|
||||
|
||||
block_name = self.scan_next('name')
|
||||
block_version = None
|
||||
|
||||
if self.scan_next('operator') == ',':
|
||||
self.scan_ignore('whitespace')
|
||||
block_version = self.scan_next('integer')
|
||||
error = False
|
||||
|
||||
# Version must be in the date format YYYYMMDD
|
||||
if len(block_version) != 8:
|
||||
error = True
|
||||
|
||||
try:
|
||||
block_version = int(block_version)
|
||||
except ValueError:
|
||||
error = True
|
||||
|
||||
if error:
|
||||
raise Exception("Invalid l10n block declaration: "
|
||||
"bad version '%s' in %s"
|
||||
% (block_name, self.tmpl))
|
||||
elif strict:
|
||||
raise Exception("Invalid l10n block declaration: "
|
||||
"missing date for block '%s' in %s"
|
||||
% (block_name, self.tmpl))
|
||||
|
||||
self.scan_until('block_end')
|
||||
yield [block_name, block_version]
|
||||
|
||||
def block_content(self):
|
||||
"""Parse the content from an l10n block"""
|
||||
|
||||
in_else = False
|
||||
main_content = []
|
||||
else_content = []
|
||||
|
||||
for token in self.tokens:
|
||||
if token[1] == 'block_begin':
|
||||
self.scan_ignore('whitespace')
|
||||
name = self.scan_next('name')
|
||||
|
||||
if name == 'endl10n':
|
||||
self.scan_until('block_end')
|
||||
break
|
||||
elif name == 'else':
|
||||
in_else = True
|
||||
self.scan_until('block_end')
|
||||
continue
|
||||
|
||||
buffer = else_content if in_else else main_content
|
||||
buffer.append(token[2])
|
||||
|
||||
return [''.join(x).replace('\\n', '\n').strip()
|
||||
for x in [main_content, else_content]]
|
||||
|
||||
def scan_until(self, name):
|
||||
for token in self.tokens:
|
||||
if token[1] == name:
|
||||
return True
|
||||
return False
|
||||
|
||||
def scan_ignore(self, name):
|
||||
for token in self.tokens:
|
||||
if token[1] != name:
|
||||
# Put it back on the list
|
||||
self.tokens = itertools.chain([token], self.tokens)
|
||||
break
|
||||
|
||||
def scan_next(self, name):
|
||||
token = self.tokens.next()
|
||||
if token and token[1] == name:
|
||||
return token[2]
|
||||
return False
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
args = ''
|
||||
help = 'Checks which content needs to be localized.'
|
||||
|
|
@ -29,126 +254,6 @@ class Command(BaseCommand):
|
|||
if args:
|
||||
langs = args
|
||||
else:
|
||||
langs = os.listdir(self.l10n_file())
|
||||
langs = os.listdir(l10n_file())
|
||||
|
||||
if options['templates']:
|
||||
self.check_templates(langs)
|
||||
else:
|
||||
self.check_blocks(langs)
|
||||
|
||||
def l10n_file(self, *args):
|
||||
return path.join(settings.ROOT, 'locale', *args)
|
||||
|
||||
def check_templates(self, langs):
|
||||
"""List templates that don't exist in the localized folders"""
|
||||
|
||||
for tmpl in self.list_templates():
|
||||
for lang in langs:
|
||||
fullpath = self.l10n_file(lang, 'templates', tmpl)
|
||||
if not path.exists(fullpath):
|
||||
print "[%s] %s" % (lang, tmpl)
|
||||
|
||||
|
||||
def check_blocks(self, langs):
|
||||
"""List templates with outdated/incorrect l10n blocks"""
|
||||
|
||||
for tmpl in self.list_templates():
|
||||
for lang in langs:
|
||||
fullpath = self.l10n_file(lang, 'templates', tmpl)
|
||||
|
||||
if path.exists(fullpath):
|
||||
blocks = self.parse_template(tmpl)
|
||||
l10n_blocks = self.parse_template(fullpath)
|
||||
|
||||
self.compare_versions(tmpl, lang, blocks, l10n_blocks)
|
||||
|
||||
|
||||
def compare_versions(self, tmpl, lang, latest, localized):
|
||||
"""Detect outdated/incorrect l10n block and notify"""
|
||||
|
||||
for name, version in latest.iteritems():
|
||||
if version:
|
||||
if not name in localized:
|
||||
print "%s: %s needs %s localized" % (tmpl, lang, name)
|
||||
elif not localized[name]:
|
||||
print ("%s: %s has unversioned %s but should be v%s"
|
||||
% (tmpl, lang, name, version))
|
||||
elif localized[name] < version:
|
||||
print ("%s: %s has %s at v%s but needs to be at v%s"
|
||||
% (tmpl, lang, name, localized[name], version))
|
||||
|
||||
def list_templates(self):
|
||||
"""List all the templates in all the installed apps"""
|
||||
|
||||
for app in settings.INSTALLED_APPS:
|
||||
tmpl_dir = path.join(settings.ROOT, 'apps', app, 'templates')
|
||||
|
||||
if path.exists(tmpl_dir):
|
||||
# Find all the .html files
|
||||
for root, dirs, files in os.walk(tmpl_dir):
|
||||
for filename in files:
|
||||
name, ext = os.path.splitext(filename)
|
||||
|
||||
if ext == '.html':
|
||||
full_path = os.path.join(root, filename)
|
||||
|
||||
# Strip the path to get just the
|
||||
# namespaced template name
|
||||
yield full_path.replace(tmpl_dir, '').lstrip('/')
|
||||
|
||||
def parse_template(self, tmpl):
|
||||
"""Analyze a template and get the l10n block information"""
|
||||
|
||||
env = get_env()
|
||||
|
||||
try:
|
||||
src = env.loader.get_source(env, tmpl)
|
||||
except TemplateNotFound:
|
||||
src = codecs.open(tmpl, encoding='utf-8').read()
|
||||
|
||||
self.tokens = env.lex(src)
|
||||
blocks = {}
|
||||
|
||||
while self.scan_until('block_begin'):
|
||||
self.scan_ignore('whitespace')
|
||||
blockname = self.scan_next('name')
|
||||
|
||||
if blockname == 'l10n':
|
||||
self.scan_ignore('whitespace')
|
||||
name = self.scan_next('name')
|
||||
self.scan_ignore('whitespace')
|
||||
|
||||
if self.scan_next('operator') == ',':
|
||||
self.scan_ignore('whitespace')
|
||||
|
||||
version = self.scan_next('integer')
|
||||
|
||||
try:
|
||||
version = int(version)
|
||||
except ValueError:
|
||||
raise Exception("Invalid l10n block declaration '%s' in %s"
|
||||
% (name, tmpl))
|
||||
|
||||
blocks[name] = version
|
||||
else:
|
||||
blocks[name] = False
|
||||
return blocks
|
||||
|
||||
def scan_until(self, name):
|
||||
for token in self.tokens:
|
||||
if token[1] == name:
|
||||
return True
|
||||
return False
|
||||
|
||||
def scan_ignore(self, name):
|
||||
for token in self.tokens:
|
||||
if token[1] != name:
|
||||
# Put it back on the list
|
||||
self.tokens = itertools.chain([token], self.tokens)
|
||||
break
|
||||
|
||||
def scan_next(self, name):
|
||||
token = self.tokens.next()
|
||||
if token and token[1] == name:
|
||||
return token[2]
|
||||
return False
|
||||
update_templates(langs)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
import unittest
|
||||
from management.commands.l10n_check import list_templates, L10nParser
|
||||
|
||||
class TestL10nCheck(unittest.TestCase):
|
||||
|
||||
def test_list_templates(self):
|
||||
tmpls = filter(lambda tmpl: 'mozorg/channel.html' in tmpl,
|
||||
list_templates())
|
||||
print list(list_templates())
|
||||
assert tmpls
|
||||
|
||||
def test_parse_templates(self):
|
||||
parser = L10nParser()
|
||||
blocks = parser.parse('foo bar bizzle what? '
|
||||
'{% l10n baz, 20110914 %}'
|
||||
'mumble'
|
||||
'{% else %}'
|
||||
'elsed'
|
||||
'{% endl10n %}'
|
||||
'qux')
|
||||
baz = blocks['baz']
|
||||
|
||||
self.assertEqual(baz['main_content'], 'mumble')
|
||||
self.assertEqual(baz['else_content'], 'elsed')
|
||||
self.assertEqual(baz['version'], 20110914)
|
||||
|
||||
blocks = parser.parse('foo bar bizzle what? '
|
||||
'{% l10n baz, 20110914 %}'
|
||||
'mumble'
|
||||
'{% endl10n %}'
|
||||
'qux')
|
||||
baz = blocks['baz']
|
||||
self.assertEqual(baz['main_content'], 'mumble')
|
||||
Загрузка…
Ссылка в новой задаче