Remove puente: add necessary babel configuration instead (#19097)

* Remove puente: add necessary babel configuration instead * Update comment * We don't actually need lstrip, and it breaks a bunch of our templates * Comment why we are doing this * Remove obsolete test (the setting is gone, this is done in babel.cfg now) * Add tests. We don't actually need the silent bit * Set ORGANIZATION, limit paths to src/olympia
2022-04-11 14:13:54 +02:00 · 2022-04-11 14:13:54 +02:00 · 3a19fe3346
--- a/babel.cfg
+++ b/babel.cfg
@ -0,0 +1,6 @@
+[python: src/olympia/**.py]
+[jinja2_custom: src/olympia/discovery/strings.jinja2]
+[django: src/olympia/**/templates/**/emails/**.*]
+[django: src/olympia/**/templates/admin/**.html]
+[django: src/olympia/**/templates/devhub/forms/widgets/compat_app_input_option.html]
+[jinja2_custom: src/olympia/**/templates/**.html]
--- a/babeljs.cfg
+++ b/babeljs.cfg
@ -0,0 +1,6 @@
+[ignore: static/js/**-all.js]
+[ignore: static/js/**-min.js]
+[javascript: static/js/*.js]
+[javascript: static/js/common/**.js]
+[javascript: static/js/stats/**.js]
+[javascript: static/js/zamboni/**.js]
--- a/locale/templates/LC_MESSAGES/django.pot
+++ b/locale/templates/LC_MESSAGES/django.pot
@ -1,10 +1,14 @@
-
+# Translations template for addons-server.
+# Copyright (C) 2022 Mozilla
+# This file is distributed under the same license as the addons-server project.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2022.
+#
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: PROJECT 1.0\n"
+"Project-Id-Version: addons-server 1.0\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-04-05 17:49+0000\n"
+"POT-Creation-Date: 2022-04-11 11:53+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
--- a/locale/templates/LC_MESSAGES/djangojs.pot
+++ b/locale/templates/LC_MESSAGES/djangojs.pot
@ -1,10 +1,14 @@
-
+# Translations template for addons-server.
+# Copyright (C) 2022 Mozilla
+# This file is distributed under the same license as the addons-server project.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2022.
+#
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: PROJECT 1.0\n"
+"Project-Id-Version: addons-server 1.0\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-04-05 17:49+0000\n"
+"POT-Creation-Date: 2022-04-11 11:53+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
--- a/requirements/prod.txt
+++ b/requirements/prod.txt
@ -381,9 +381,6 @@ polib==1.1.1 \
 prompt_toolkit==3.0.28 \
    --hash=sha256:30129d870dcb0b3b6a53efdc9d0a83ea96162ffd28ffe077e94215b233dc670c \
    --hash=sha256:9f1cd16b1e86c2968f2519d7fb31dd9d669916f515612c269d14e9ed52b51650
-puente==0.5.0 \
-    --hash=sha256:7ba1d07f9cee9657adf874bd94879b343fea81a783fdfa8e53885520477bf1ea \
-    --hash=sha256:4a17958f7d6a83cb9ff92593c40f34911abafaec6ad959916b754aaa3869f11f
 # python-dateutil is required by elasticsearch-dsl
 python-dateutil==2.8.2 \
    --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 \
--- a/scripts/extract-l10n.sh
+++ b/scripts/extract-l10n.sh
@ -19,6 +19,11 @@ set -o pipefail
 # Treat unset variables as an error an exit immediately.
 set -u

+# Extraction needs our django settings for jinja, so we need a django settings
+# module set. Since this command is meant to be run in local envs, we use
+# "settings".
+DJANGO_SETTINGS_MODULE=settings
+
 INITIAL_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 GIT_CHANGES=$(git status --porcelain)
 GIT_REMOTE="https://github.com/mozilla/addons-server.git"  # Upstream.
@ -81,7 +86,8 @@ function init_environment {

 function run_l10n_extraction {
    python3 manage.py extract_content_strings
-    python3 manage.py extract
+    PYTHONPATH=. DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} pybabel extract -F babel.cfg -o locale/templates/LC_MESSAGES/django.pot -c 'L10n:' -w 80 --version=1.0 --project=addons-server --copyright-holder=Mozilla .
+    PYTHONPATH=. DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} pybabel extract -F babeljs.cfg -o locale/templates/LC_MESSAGES/djangojs.pot -c 'L10n:' -w 80 --version=1.0 --project=addons-server --copyright-holder=Mozilla .

    pushd locale > /dev/null

--- a/setup.py
+++ b/setup.py
@ -25,6 +25,11 @@ setup(
        'Framework :: Django',
        'Topic :: Internet :: WWW/HTTP :: Browsers',
        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.9',
    ],
+    entry_points={
+        'babel.extractors': [
+            'jinja2_custom = olympia.core.babel:extract_jinja',
+        ]
+    },
 )
--- a/src/olympia/core/babel.py
+++ b/src/olympia/core/babel.py
@ -0,0 +1,68 @@
+import django
+from django.conf import settings
+from jinja2.ext import babel_extract
+
+
+# List of settings jinja2.ext.babel_extract() cares about and that are safe
+# to pass as options as a result.
+RELEVANT_SETTINGS = [
+    'block_start_string',
+    'block_end_string',
+    'variable_start_string',
+    'variable_end_string',
+    'comment_start_string',
+    'comment_end_string',
+    'line_statement_prefix',
+    'line_comment_prefix',
+    'trim_blocks',
+    'lstrip_blocks',
+    'keep_trailing_newline',
+    'extensions',
+]
+
+
+def generate_option(value):
+    """
+    Generate option to pass to babel_extract() from a TEMPLATES['OPTION'] value
+    setting.
+
+    babel_extract() options are meant to be coming from babel config files, so
+    everything is based on strings.
+    """
+    if isinstance(value, bool):
+        return 'true' if value else 'false'
+    elif isinstance(value, (list, tuple)):
+        return ','.join(value)
+    return value
+
+
+def extract_jinja(fileobj, keywords, comment_tags, options):
+    """
+    Wrapper around jinja2's babel_extract() that sets the relevant options by
+    looking at our django settings.
+
+    This is necessary because jinja2's babel_extract() loads a default
+    environement which doesn't have our extensions and doesn't set the options
+    we need for trimming, so it can't process all our templates and generates
+    a po file that doesn't correspond to our gettext calls because of the
+    whitespace differences.
+    """
+    # django needs to be configured for the jinja extensions to be imported,
+    # since at least one imports our models.
+    django.setup()
+
+    for TEMPLATE in settings.TEMPLATES:
+        if TEMPLATE.get('NAME') == 'jinja2':
+            overrides = {
+                key: generate_option(TEMPLATE['OPTIONS'][key])
+                for key in RELEVANT_SETTINGS
+                if key in TEMPLATE['OPTIONS']
+            }
+            options.update(overrides)
+            # Special case: `trimmed` is configured through an environment policy,
+            # but babel_extract() considers it's an option.
+            options['trimmed'] = generate_option(
+                TEMPLATE['OPTIONS'].get('policies', {}).get('ext.i18n.trimmed', False)
+            )
+            break
+    return babel_extract(fileobj, keywords, comment_tags, options)
--- a/src/olympia/core/tests/test_babel.py
+++ b/src/olympia/core/tests/test_babel.py
@ -0,0 +1,26 @@
+from io import BytesIO
+
+import pytest
+from babel.messages.extract import DEFAULT_KEYWORDS
+
+from olympia.core.babel import extract_jinja, generate_option
+
+
+@pytest.mark.parametrize(
+    'value,expected',
+    [
+        (True, 'true'),
+        (False, 'false'),
+        ('foo', 'foo'),
+        (['foo', 'bar'], 'foo,bar'),
+        (('abc', 'def'), 'abc,def'),
+    ],
+)
+def test_generate_option(value, expected):
+    assert generate_option(value) == expected
+
+
+def test_extract_jinja_no_options():
+    # Doesn't actually extract, just tests that extract_jinja() doesn't fail
+    # when converting our settings in options for the underlying function.
+    extract_jinja(BytesIO(), DEFAULT_KEYWORDS, ['L10n:'], {})
--- a/src/olympia/discovery/tests/test_commands.py
+++ b/src/olympia/discovery/tests/test_commands.py
@ -111,11 +111,6 @@ expected_content = """{# L10n: editorial content for the discovery pane. #}


 class TestExtractDiscoStringsCommand(TestCase):
-    def test_settings(self):
-        assert (settings.EDITORIAL_CONTENT_FILENAME, 'jinja2') in settings.PUENTE[
-            'DOMAIN_METHODS'
-        ]['django']
-
    def test_basic(self):
        responses.add(
            responses.GET,
--- a/src/olympia/lib/settings_base.py
+++ b/src/olympia/lib/settings_base.py
@ -308,7 +308,7 @@ SECRET_CDN_TOKEN = env('SECRET_CDN_TOKEN', default=None)

 # Templates configuration.
 # List of path patterns for which we should be using Django Template Language.
-# If you add things here, don't forget to also change PUENTE config below.
+# If you add things here, don't forget to also change babel.cfg !
 JINJA_EXCLUDE_TEMPLATE_PATHS = (
    # All emails should be processed with Django for consistency.
    r'^.*\/emails\/',
@ -326,6 +326,8 @@ JINJA_EXCLUDE_TEMPLATE_PATHS = (
 TEMPLATES = [
    {
        'BACKEND': 'django_jinja.backend.Jinja2',
+        # This is used by olympia.core.babel to find the template configuration
+        # for jinja2 templates.
        'NAME': 'jinja2',
        'APP_DIRS': True,
        'DIRS': (
@ -352,6 +354,7 @@ TEMPLATES = [
            ),
            'extensions': (
                'jinja2.ext.do',
+                'jinja2.ext.i18n',
                'jinja2.ext.loopcontrols',
                'django_jinja.builtins.extensions.CsrfExtension',
                'django_jinja.builtins.extensions.DjangoFiltersExtension',
@ -359,9 +362,11 @@ TEMPLATES = [
                'django_jinja.builtins.extensions.TimezoneExtension',
                'django_jinja.builtins.extensions.UrlsExtension',
                'olympia.amo.templatetags.jinja_helpers.Spaceless',
-                'puente.ext.i18n',
                'waffle.jinja.WaffleExtension',
            ),
+            'policies': {
+                'ext.i18n.trimmed': True,
+            },
            'finalize': lambda x: x if x is not None else '',
            'translation_engine': 'django.utils.translation',
            'autoescape': True,
@ -519,7 +524,6 @@ INSTALLED_APPS = (
    'rest_framework',
    'waffle',
    'django_jinja',
-    'puente',
    'rangefilter',
    'nobot',
    # Django contrib apps
@ -551,52 +555,10 @@ HOMEPAGE_SHELVES_EDITORIAL_CONTENT_API = (
    'https://addons.mozilla.org/api/v5/shelves/editorial'
 )

-# Filename where the strings will be stored. Used in puente config below.
+# Filename where the strings will be stored. Used in extract_content_strings
+# management command, but note that the filename is hardcoded in babel.cfg.
 EDITORIAL_CONTENT_FILENAME = 'src/olympia/discovery/strings.jinja2'

-# Tells the extract script what files to look for l10n in and what function
-# handles the extraction. The puente library expects this.
-PUENTE = {
-    'BASE_DIR': ROOT,
-    # Tells the extract script what files to look for l10n in and what function
-    # handles the extraction.
-    'DOMAIN_METHODS': {
-        'django': [
-            ('src/olympia/**.py', 'python'),
-            # Extract the generated file containing editorial content for all
-            # disco pane recommendations using jinja2 parser. It's not a real
-            # template, but it uses jinja2 syntax for convenience, hence why
-            # it's not in templates/ with a .html extension.
-            (EDITORIAL_CONTENT_FILENAME, 'jinja2'),
-            # Make sure we're parsing django-admin & email templates with the
-            # django template extractor. This should match the behavior of
-            # JINJA_EXCLUDE_TEMPLATE_PATHS
-            (
-                'src/olympia/**/templates/**/emails/**.*',
-                'enmerkar.extract.extract_django',
-            ),
-            ('**/templates/admin/**.html', 'enmerkar.extract.extract_django'),
-            (
-                '**/templates/devhub/forms/widgets/compat_app_input_option.html',
-                'enmerkar.extract.extract_django',
-            ),
-            ('src/olympia/**/templates/**.html', 'jinja2'),
-        ],
-        'djangojs': [
-            # We can't say **.js because that would dive into mochikit
-            # and timeplot and all the other baggage we're carrying.
-            # Timeplot, in particular, crashes the extractor with bad
-            # unicode data.
-            ('static/js/**-all.js', 'ignore'),
-            ('static/js/**-min.js', 'ignore'),
-            ('static/js/*.js', 'javascript'),
-            ('static/js/common/**.js', 'javascript'),
-            ('static/js/stats/**.js', 'javascript'),
-            ('static/js/zamboni/**.js', 'javascript'),
-        ],
-    },
-}
-
 # Bundles is a dictionary of two dictionaries, css and js, which list css files
 # and js files that can be bundled together by the minify app.
 MINIFY_BUNDLES = {
@ -999,10 +961,19 @@ LOGGING = {
            'level': 'ERROR',
            'class': 'django_statsd.loggers.errors.StatsdHandler',
        },
+        'console': {
+            'class': 'logging.StreamHandler',
+        },
    },
    'root': {'handlers': ['mozlog'], 'level': logging.INFO},
    'loggers': {
        'amqp': {'handlers': ['null'], 'level': logging.WARNING, 'propagate': False},
+        'babel': {'handlers': ['console'], 'level': logging.INFO, 'propagate': False},
+        'blib2to3.pgen2.driver': {
+            'handlers': ['null'],
+            'level': logging.INFO,
+            'propagate': False,
+        },
        'caching': {'handlers': ['mozlog'], 'level': logging.ERROR, 'propagate': False},
        'caching.invalidation': {
            'handlers': ['null'],
--- a/src/olympia/translations/templates/translations/trans-menu.html
+++ b/src/olympia/translations/templates/translations/trans-menu.html
@ -3,7 +3,7 @@
     data-default="{{ default_locale }}" id="l10n-menu">
  {% set dl = languages[default_locale] %}
  <p>
-  {# l10n: {0} is a language name, like 'French' #}
+  {# L10n: {0} is a language name, like 'French' #}
  {% trans %}
    Localize for: <a id="change-locale" href="#">{{ dl }}</a>
  {% endtrans %}
@ -38,7 +38,7 @@
  </div>
  <div id="modal-l10n-unsaved" class="modal hidden">
    <p class="msg">
-    {# l10n: {0} is a language name, like 'French' #}
+    {# L10n: {0} is a language name, like 'French' #}
    {% trans %}
      You have unsaved changes in the <b>{0}</b> locale.
      Would you like to save your changes before switching locales?
@ -52,7 +52,7 @@
  </div>
  <div id="modal-l10n-rm" class="modal hidden">
    <p class="msg">
-    {# l10n: {0} is a language name, like 'French' #}
+    {# L10n: {0} is a language name, like 'French' #}
    {% trans %}
      Are you sure you want to remove all <b>{0}</b> translations? This cannot be undone.
    {% endtrans %}