From 3a19fe3346576b05ee5fcb402959df612171a688 Mon Sep 17 00:00:00 2001 From: Mathieu Pillard Date: Mon, 11 Apr 2022 14:13:54 +0200 Subject: [PATCH] Remove puente: add necessary babel configuration instead (#19097) * Remove puente: add necessary babel configuration instead * Update comment * We don't actually need lstrip, and it breaks a bunch of our templates * Comment why we are doing this * Remove obsolete test (the setting is gone, this is done in babel.cfg now) * Add tests. We don't actually need the silent bit * Set ORGANIZATION, limit paths to src/olympia --- babel.cfg | 6 ++ babeljs.cfg | 6 ++ locale/templates/LC_MESSAGES/django.pot | 10 ++- locale/templates/LC_MESSAGES/djangojs.pot | 10 ++- requirements/prod.txt | 3 - scripts/extract-l10n.sh | 8 ++- setup.py | 7 +- src/olympia/core/babel.py | 68 +++++++++++++++++++ src/olympia/core/tests/test_babel.py | 26 +++++++ src/olympia/discovery/tests/test_commands.py | 5 -- src/olympia/lib/settings_base.py | 65 +++++------------- .../templates/translations/trans-menu.html | 6 +- 12 files changed, 154 insertions(+), 66 deletions(-) create mode 100644 babel.cfg create mode 100644 babeljs.cfg create mode 100644 src/olympia/core/babel.py create mode 100644 src/olympia/core/tests/test_babel.py diff --git a/babel.cfg b/babel.cfg new file mode 100644 index 0000000000..18880252cd --- /dev/null +++ b/babel.cfg @@ -0,0 +1,6 @@ +[python: src/olympia/**.py] +[jinja2_custom: src/olympia/discovery/strings.jinja2] +[django: src/olympia/**/templates/**/emails/**.*] +[django: src/olympia/**/templates/admin/**.html] +[django: src/olympia/**/templates/devhub/forms/widgets/compat_app_input_option.html] +[jinja2_custom: src/olympia/**/templates/**.html] diff --git a/babeljs.cfg b/babeljs.cfg new file mode 100644 index 0000000000..690a87d2e1 --- /dev/null +++ b/babeljs.cfg @@ -0,0 +1,6 @@ +[ignore: static/js/**-all.js] +[ignore: static/js/**-min.js] +[javascript: static/js/*.js] +[javascript: static/js/common/**.js] +[javascript: static/js/stats/**.js] +[javascript: static/js/zamboni/**.js] diff --git a/locale/templates/LC_MESSAGES/django.pot b/locale/templates/LC_MESSAGES/django.pot index edf9b19d36..69b6d7f03e 100644 --- a/locale/templates/LC_MESSAGES/django.pot +++ b/locale/templates/LC_MESSAGES/django.pot @@ -1,10 +1,14 @@ - +# Translations template for addons-server. +# Copyright (C) 2022 Mozilla +# This file is distributed under the same license as the addons-server project. +# FIRST AUTHOR , 2022. +# #, fuzzy msgid "" msgstr "" -"Project-Id-Version: PROJECT 1.0\n" +"Project-Id-Version: addons-server 1.0\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2022-04-05 17:49+0000\n" +"POT-Creation-Date: 2022-04-11 11:53+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" diff --git a/locale/templates/LC_MESSAGES/djangojs.pot b/locale/templates/LC_MESSAGES/djangojs.pot index 460a74701f..8b8977b87d 100644 --- a/locale/templates/LC_MESSAGES/djangojs.pot +++ b/locale/templates/LC_MESSAGES/djangojs.pot @@ -1,10 +1,14 @@ - +# Translations template for addons-server. +# Copyright (C) 2022 Mozilla +# This file is distributed under the same license as the addons-server project. +# FIRST AUTHOR , 2022. +# #, fuzzy msgid "" msgstr "" -"Project-Id-Version: PROJECT 1.0\n" +"Project-Id-Version: addons-server 1.0\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2022-04-05 17:49+0000\n" +"POT-Creation-Date: 2022-04-11 11:53+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" diff --git a/requirements/prod.txt b/requirements/prod.txt index 19ca890246..d66fc96fca 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -381,9 +381,6 @@ polib==1.1.1 \ prompt_toolkit==3.0.28 \ --hash=sha256:30129d870dcb0b3b6a53efdc9d0a83ea96162ffd28ffe077e94215b233dc670c \ --hash=sha256:9f1cd16b1e86c2968f2519d7fb31dd9d669916f515612c269d14e9ed52b51650 -puente==0.5.0 \ - --hash=sha256:7ba1d07f9cee9657adf874bd94879b343fea81a783fdfa8e53885520477bf1ea \ - --hash=sha256:4a17958f7d6a83cb9ff92593c40f34911abafaec6ad959916b754aaa3869f11f # python-dateutil is required by elasticsearch-dsl python-dateutil==2.8.2 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 \ diff --git a/scripts/extract-l10n.sh b/scripts/extract-l10n.sh index ab8093ac8d..c94fc0d4ad 100755 --- a/scripts/extract-l10n.sh +++ b/scripts/extract-l10n.sh @@ -19,6 +19,11 @@ set -o pipefail # Treat unset variables as an error an exit immediately. set -u +# Extraction needs our django settings for jinja, so we need a django settings +# module set. Since this command is meant to be run in local envs, we use +# "settings". +DJANGO_SETTINGS_MODULE=settings + INITIAL_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) GIT_CHANGES=$(git status --porcelain) GIT_REMOTE="https://github.com/mozilla/addons-server.git" # Upstream. @@ -81,7 +86,8 @@ function init_environment { function run_l10n_extraction { python3 manage.py extract_content_strings - python3 manage.py extract + PYTHONPATH=. DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} pybabel extract -F babel.cfg -o locale/templates/LC_MESSAGES/django.pot -c 'L10n:' -w 80 --version=1.0 --project=addons-server --copyright-holder=Mozilla . + PYTHONPATH=. DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} pybabel extract -F babeljs.cfg -o locale/templates/LC_MESSAGES/djangojs.pot -c 'L10n:' -w 80 --version=1.0 --project=addons-server --copyright-holder=Mozilla . pushd locale > /dev/null diff --git a/setup.py b/setup.py index 9a0066f53f..26aba775af 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,11 @@ setup( 'Framework :: Django', 'Topic :: Internet :: WWW/HTTP :: Browsers', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.9', ], + entry_points={ + 'babel.extractors': [ + 'jinja2_custom = olympia.core.babel:extract_jinja', + ] + }, ) diff --git a/src/olympia/core/babel.py b/src/olympia/core/babel.py new file mode 100644 index 0000000000..bbc9d38918 --- /dev/null +++ b/src/olympia/core/babel.py @@ -0,0 +1,68 @@ +import django +from django.conf import settings +from jinja2.ext import babel_extract + + +# List of settings jinja2.ext.babel_extract() cares about and that are safe +# to pass as options as a result. +RELEVANT_SETTINGS = [ + 'block_start_string', + 'block_end_string', + 'variable_start_string', + 'variable_end_string', + 'comment_start_string', + 'comment_end_string', + 'line_statement_prefix', + 'line_comment_prefix', + 'trim_blocks', + 'lstrip_blocks', + 'keep_trailing_newline', + 'extensions', +] + + +def generate_option(value): + """ + Generate option to pass to babel_extract() from a TEMPLATES['OPTION'] value + setting. + + babel_extract() options are meant to be coming from babel config files, so + everything is based on strings. + """ + if isinstance(value, bool): + return 'true' if value else 'false' + elif isinstance(value, (list, tuple)): + return ','.join(value) + return value + + +def extract_jinja(fileobj, keywords, comment_tags, options): + """ + Wrapper around jinja2's babel_extract() that sets the relevant options by + looking at our django settings. + + This is necessary because jinja2's babel_extract() loads a default + environement which doesn't have our extensions and doesn't set the options + we need for trimming, so it can't process all our templates and generates + a po file that doesn't correspond to our gettext calls because of the + whitespace differences. + """ + # django needs to be configured for the jinja extensions to be imported, + # since at least one imports our models. + django.setup() + + for TEMPLATE in settings.TEMPLATES: + if TEMPLATE.get('NAME') == 'jinja2': + overrides = { + key: generate_option(TEMPLATE['OPTIONS'][key]) + for key in RELEVANT_SETTINGS + if key in TEMPLATE['OPTIONS'] + } + options.update(overrides) + # Special case: `trimmed` is configured through an environment policy, + # but babel_extract() considers it's an option. + options['trimmed'] = generate_option( + TEMPLATE['OPTIONS'].get('policies', {}).get('ext.i18n.trimmed', False) + ) + break + return babel_extract(fileobj, keywords, comment_tags, options) diff --git a/src/olympia/core/tests/test_babel.py b/src/olympia/core/tests/test_babel.py new file mode 100644 index 0000000000..f4d6933c89 --- /dev/null +++ b/src/olympia/core/tests/test_babel.py @@ -0,0 +1,26 @@ +from io import BytesIO + +import pytest +from babel.messages.extract import DEFAULT_KEYWORDS + +from olympia.core.babel import extract_jinja, generate_option + + +@pytest.mark.parametrize( + 'value,expected', + [ + (True, 'true'), + (False, 'false'), + ('foo', 'foo'), + (['foo', 'bar'], 'foo,bar'), + (('abc', 'def'), 'abc,def'), + ], +) +def test_generate_option(value, expected): + assert generate_option(value) == expected + + +def test_extract_jinja_no_options(): + # Doesn't actually extract, just tests that extract_jinja() doesn't fail + # when converting our settings in options for the underlying function. + extract_jinja(BytesIO(), DEFAULT_KEYWORDS, ['L10n:'], {}) diff --git a/src/olympia/discovery/tests/test_commands.py b/src/olympia/discovery/tests/test_commands.py index dcb67c37f8..ab45844a71 100644 --- a/src/olympia/discovery/tests/test_commands.py +++ b/src/olympia/discovery/tests/test_commands.py @@ -111,11 +111,6 @@ expected_content = """{# L10n: editorial content for the discovery pane. #} class TestExtractDiscoStringsCommand(TestCase): - def test_settings(self): - assert (settings.EDITORIAL_CONTENT_FILENAME, 'jinja2') in settings.PUENTE[ - 'DOMAIN_METHODS' - ]['django'] - def test_basic(self): responses.add( responses.GET, diff --git a/src/olympia/lib/settings_base.py b/src/olympia/lib/settings_base.py index 8a06c126a2..d6ce9b6557 100644 --- a/src/olympia/lib/settings_base.py +++ b/src/olympia/lib/settings_base.py @@ -308,7 +308,7 @@ SECRET_CDN_TOKEN = env('SECRET_CDN_TOKEN', default=None) # Templates configuration. # List of path patterns for which we should be using Django Template Language. -# If you add things here, don't forget to also change PUENTE config below. +# If you add things here, don't forget to also change babel.cfg ! JINJA_EXCLUDE_TEMPLATE_PATHS = ( # All emails should be processed with Django for consistency. r'^.*\/emails\/', @@ -326,6 +326,8 @@ JINJA_EXCLUDE_TEMPLATE_PATHS = ( TEMPLATES = [ { 'BACKEND': 'django_jinja.backend.Jinja2', + # This is used by olympia.core.babel to find the template configuration + # for jinja2 templates. 'NAME': 'jinja2', 'APP_DIRS': True, 'DIRS': ( @@ -352,6 +354,7 @@ TEMPLATES = [ ), 'extensions': ( 'jinja2.ext.do', + 'jinja2.ext.i18n', 'jinja2.ext.loopcontrols', 'django_jinja.builtins.extensions.CsrfExtension', 'django_jinja.builtins.extensions.DjangoFiltersExtension', @@ -359,9 +362,11 @@ TEMPLATES = [ 'django_jinja.builtins.extensions.TimezoneExtension', 'django_jinja.builtins.extensions.UrlsExtension', 'olympia.amo.templatetags.jinja_helpers.Spaceless', - 'puente.ext.i18n', 'waffle.jinja.WaffleExtension', ), + 'policies': { + 'ext.i18n.trimmed': True, + }, 'finalize': lambda x: x if x is not None else '', 'translation_engine': 'django.utils.translation', 'autoescape': True, @@ -519,7 +524,6 @@ INSTALLED_APPS = ( 'rest_framework', 'waffle', 'django_jinja', - 'puente', 'rangefilter', 'nobot', # Django contrib apps @@ -551,52 +555,10 @@ HOMEPAGE_SHELVES_EDITORIAL_CONTENT_API = ( 'https://addons.mozilla.org/api/v5/shelves/editorial' ) -# Filename where the strings will be stored. Used in puente config below. +# Filename where the strings will be stored. Used in extract_content_strings +# management command, but note that the filename is hardcoded in babel.cfg. EDITORIAL_CONTENT_FILENAME = 'src/olympia/discovery/strings.jinja2' -# Tells the extract script what files to look for l10n in and what function -# handles the extraction. The puente library expects this. -PUENTE = { - 'BASE_DIR': ROOT, - # Tells the extract script what files to look for l10n in and what function - # handles the extraction. - 'DOMAIN_METHODS': { - 'django': [ - ('src/olympia/**.py', 'python'), - # Extract the generated file containing editorial content for all - # disco pane recommendations using jinja2 parser. It's not a real - # template, but it uses jinja2 syntax for convenience, hence why - # it's not in templates/ with a .html extension. - (EDITORIAL_CONTENT_FILENAME, 'jinja2'), - # Make sure we're parsing django-admin & email templates with the - # django template extractor. This should match the behavior of - # JINJA_EXCLUDE_TEMPLATE_PATHS - ( - 'src/olympia/**/templates/**/emails/**.*', - 'enmerkar.extract.extract_django', - ), - ('**/templates/admin/**.html', 'enmerkar.extract.extract_django'), - ( - '**/templates/devhub/forms/widgets/compat_app_input_option.html', - 'enmerkar.extract.extract_django', - ), - ('src/olympia/**/templates/**.html', 'jinja2'), - ], - 'djangojs': [ - # We can't say **.js because that would dive into mochikit - # and timeplot and all the other baggage we're carrying. - # Timeplot, in particular, crashes the extractor with bad - # unicode data. - ('static/js/**-all.js', 'ignore'), - ('static/js/**-min.js', 'ignore'), - ('static/js/*.js', 'javascript'), - ('static/js/common/**.js', 'javascript'), - ('static/js/stats/**.js', 'javascript'), - ('static/js/zamboni/**.js', 'javascript'), - ], - }, -} - # Bundles is a dictionary of two dictionaries, css and js, which list css files # and js files that can be bundled together by the minify app. MINIFY_BUNDLES = { @@ -999,10 +961,19 @@ LOGGING = { 'level': 'ERROR', 'class': 'django_statsd.loggers.errors.StatsdHandler', }, + 'console': { + 'class': 'logging.StreamHandler', + }, }, 'root': {'handlers': ['mozlog'], 'level': logging.INFO}, 'loggers': { 'amqp': {'handlers': ['null'], 'level': logging.WARNING, 'propagate': False}, + 'babel': {'handlers': ['console'], 'level': logging.INFO, 'propagate': False}, + 'blib2to3.pgen2.driver': { + 'handlers': ['null'], + 'level': logging.INFO, + 'propagate': False, + }, 'caching': {'handlers': ['mozlog'], 'level': logging.ERROR, 'propagate': False}, 'caching.invalidation': { 'handlers': ['null'], diff --git a/src/olympia/translations/templates/translations/trans-menu.html b/src/olympia/translations/templates/translations/trans-menu.html index 97020fbd1b..8d7408762f 100644 --- a/src/olympia/translations/templates/translations/trans-menu.html +++ b/src/olympia/translations/templates/translations/trans-menu.html @@ -3,7 +3,7 @@ data-default="{{ default_locale }}" id="l10n-menu"> {% set dl = languages[default_locale] %}

- {# l10n: {0} is a language name, like 'French' #} + {# L10n: {0} is a language name, like 'French' #} {% trans %} Localize for: {{ dl }} {% endtrans %} @@ -38,7 +38,7 @@