Move lang files into the data directory

This commit is contained in:
Paul McLanahan 2020-05-29 10:59:57 -04:00 коммит произвёл Paul McLanahan
Родитель d0fb5d4daa
Коммит 1ecb3c0807
18 изменённых файлов: 71 добавлений и 52 удалений

Просмотреть файл

@ -4,7 +4,7 @@ import shutil
from django.conf import settings
from django.test import TestCase
from bedrock.settings.base import get_dev_languages, path
from bedrock.settings.base import get_dev_languages, data_path
class AcceptedLocalesTest(TestCase):
@ -15,8 +15,8 @@ class AcceptedLocalesTest(TestCase):
DEV_LANGUAGES or PROD_LANGUAGES should be used.
"""
locale = path('locale')
locale_bkp = path('locale_bkp')
locale = data_path('locale')
locale_bkp = data_path('locale_bkp')
@classmethod
def setup_class(cls):

Просмотреть файл

@ -19,8 +19,9 @@ from bedrock.base.config_manager import config
# ROOT path of the project. A pathlib.Path object.
DATA_PATH = config('DATA_PATH', default='data')
ROOT_PATH = Path(__file__).resolve().parents[2]
GIT_REPOS_PATH = ROOT_PATH / 'git-repos'
DATA_PATH = ROOT_PATH / DATA_PATH
ROOT = str(ROOT_PATH)
@ -28,8 +29,8 @@ def path(*args):
return abspath(str(ROOT_PATH.joinpath(*args)))
def git_repo_path(*args):
return abspath(str(GIT_REPOS_PATH.joinpath(*args)))
def data_path(*args):
return abspath(str(DATA_PATH.joinpath(*args)))
# Is this a dev instance?
@ -41,7 +42,7 @@ DEBUG = config('DEBUG', parser=bool, default='false')
DATABASES = {
'default': {
'ENGINE': 'django_prometheus.db.backends.sqlite3',
'NAME': path('bedrock.db'),
'NAME': data_path('bedrock.db'),
},
}
@ -112,7 +113,7 @@ PROD_DETAILS_STORAGE = config('PROD_DETAILS_STORAGE',
default='product_details.storage.PDDatabaseStorage')
# path into which to clone the p-d json repo
PROD_DETAILS_JSON_REPO_PATH = config('PROD_DETAILS_JSON_REPO_PATH',
default=git_repo_path('product_details_json'))
default=data_path('product_details_json'))
PROD_DETAILS_JSON_REPO_URI = config('PROD_DETAILS_JSON_REPO_URI',
default='https://github.com/mozilla-releng/product-details.git')
PROD_DETAILS_JSON_REPO_BRANCH = config('PROD_DETAILS_JSON_REPO_BRANCH', default='production')
@ -134,7 +135,7 @@ PROD_LANGUAGES = ('ach', 'af', 'an', 'ar', 'ast', 'az', 'azz', 'be', 'bg',
'sr', 'sv-SE', 'ta', 'te', 'th', 'tl', 'tr', 'trs', 'uk', 'ur',
'uz', 'vi', 'xh', 'zh-CN', 'zh-TW', 'zu')
LOCALES_PATH = ROOT_PATH / 'locale'
LOCALES_PATH = DATA_PATH / 'locale'
default_locales_repo = 'www.mozilla.org' if DEV else 'bedrock-l10n'
default_locales_repo = 'https://github.com/mozilla-l10n/{}'.format(default_locales_repo)
LOCALES_REPO = config('LOCALES_REPO', default=default_locales_repo)
@ -260,13 +261,13 @@ FLUENT_DEFAULT_FILES = [
FLUENT_DEFAULT_PERCENT_REQUIRED = config('FLUENT_DEFAULT_PERCENT_REQUIRED', default='80', parser=int)
FLUENT_REPO = config('FLUENT_REPO', default='mozmeao/www-l10n')
FLUENT_REPO_URL = f'https://github.com/{FLUENT_REPO}'
FLUENT_REPO_PATH = GIT_REPOS_PATH / 'www-l10n'
FLUENT_REPO_PATH = DATA_PATH / 'www-l10n'
# will be something like "<github username>:<github token>"
FLUENT_REPO_AUTH = config('FLUENT_REPO_AUTH', default='')
FLUENT_LOCAL_PATH = ROOT_PATH / 'l10n'
FLUENT_L10N_TEAM_REPO = config('FLUENT_L10N_TEAM_REPO', default='mozilla-l10n/www-l10n')
FLUENT_L10N_TEAM_REPO_URL = f'https://github.com/{FLUENT_L10N_TEAM_REPO}'
FLUENT_L10N_TEAM_REPO_PATH = GIT_REPOS_PATH / 'l10n-team'
FLUENT_L10N_TEAM_REPO_PATH = DATA_PATH / 'l10n-team'
# 10 seconds during dev and 10 min in prod
FLUENT_CACHE_TIMEOUT = config('FLUENT_CACHE_TIMEOUT', default='10' if DEBUG else '600', parser=int)
# order matters. first sting found wins.
@ -703,7 +704,7 @@ EMAIL_HOST_PASSWORD = config('EMAIL_HOST_PASSWORD', default='')
# Google Analytics
GA_ACCOUNT_CODE = ''
EXTERNAL_FILES_PATH = config('EXTERNAL_FILES_PATH', default=git_repo_path('community_data'))
EXTERNAL_FILES_PATH = config('EXTERNAL_FILES_PATH', default=data_path('community_data'))
EXTERNAL_FILES_BRANCH = config('EXTERNAL_FILES_BRANCH', default='master')
EXTERNAL_FILES_REPO = config('EXTERNAL_FILES_REPO', default='https://github.com/mozilla/community-data.git')
EXTERNAL_FILES = {
@ -1352,16 +1353,16 @@ if DEV:
else:
content_cards_default_branch = 'prod-processed'
CONTENT_CARDS_PATH = config('CONTENT_CARDS_PATH', default=git_repo_path('content_cards'))
CONTENT_CARDS_PATH = config('CONTENT_CARDS_PATH', default=data_path('content_cards'))
CONTENT_CARDS_REPO = config('CONTENT_CARDS_REPO', default='https://github.com/mozmeao/www-admin.git')
CONTENT_CARDS_BRANCH = config('CONTENT_CARDS_BRANCH', default=content_cards_default_branch)
CONTENT_CARDS_URL = config('CONTENT_CARDS_URL', default=STATIC_URL)
RELEASE_NOTES_PATH = config('RELEASE_NOTES_PATH', default=git_repo_path('release_notes'))
RELEASE_NOTES_PATH = config('RELEASE_NOTES_PATH', default=data_path('release_notes'))
RELEASE_NOTES_REPO = config('RELEASE_NOTES_REPO', default='https://github.com/mozilla/release-notes.git')
RELEASE_NOTES_BRANCH = config('RELEASE_NOTES_BRANCH', default='master')
WWW_CONFIG_PATH = config('WWW_CONFIG_PATH', default=git_repo_path('www_config'))
WWW_CONFIG_PATH = config('WWW_CONFIG_PATH', default=data_path('www_config'))
WWW_CONFIG_REPO = config('WWW_CONFIG_REPO', default='https://github.com/mozmeao/www-config.git')
WWW_CONFIG_BRANCH = config('WWW_CONFIG_BRANCH', default='master')
@ -1372,7 +1373,7 @@ LEGAL_DOCS_DMS_URL = config('LEGAL_DOCS_DMS_URL', default='')
LEGAL_DOCS_CACHE_TIMEOUT = config('LEGAL_DOCS_CACHE_TIMEOUT', default='60' if DEV else '600', parser=int)
MOFO_SECURITY_ADVISORIES_PATH = config('MOFO_SECURITY_ADVISORIES_PATH',
default=git_repo_path('mofo_security_advisories'))
default=data_path('mofo_security_advisories'))
MOFO_SECURITY_ADVISORIES_REPO = config('MOFO_SECURITY_ADVISORIES_REPO',
default='https://github.com/mozilla/'
'foundation-security-advisories.git')

Просмотреть файл

@ -4,8 +4,9 @@ from os import getenv
from subprocess import check_output, CalledProcessError
JSON_DATA_FILE = getenv('AWS_DB_JSON_DATA_FILE', 'bedrock_db_info.json')
DB_FILE = 'bedrock.db'
DATA_PATH = getenv('DATA_PATH', 'data')
JSON_DATA_FILE = getenv('AWS_DB_JSON_DATA_FILE', f'{DATA_PATH}/bedrock_db_info.json')
DB_FILE = f'{DATA_PATH}/bedrock.db'
CACHE = {}
BLOCKSIZE = 65536

Просмотреть файл

@ -12,6 +12,7 @@ from db_s3_utils import (
set_db_data,
JSON_DATA_FILE,
DB_FILE,
DATA_PATH,
)
@ -29,7 +30,7 @@ def get_file_url(filename):
def download_db_info():
try:
resp = requests.get(get_file_url(JSON_DATA_FILE))
resp = requests.get(get_file_url(os.path.basename(JSON_DATA_FILE)))
resp.raise_for_status()
except requests.RequestException:
return None
@ -42,7 +43,7 @@ def download_db_info():
def download_db_file(filename):
resp = requests.get(get_file_url(filename), stream=True)
resp = requests.get(get_file_url(os.path.basename(filename)), stream=True)
with open(filename, 'wb') as fp:
for chunk in resp.iter_content(chunk_size=128):
fp.write(chunk)
@ -76,6 +77,7 @@ def main(args):
return 0
new_db_file = db_info['file_name']
new_db_file = f'{DATA_PATH}/{new_db_file}'
download_db_file(new_db_file)
checksum = get_db_checksum(new_db_file)
if checksum == db_info['checksum']:

Просмотреть файл

@ -6,5 +6,8 @@ if [ ! -e ./manage.py ]; then
cd $script_parent_dir
fi
# ensure the data dir exists
mkdir -p data
# use honcho to inject the proper env vars
honcho run --env docker/envfiles/prod.env ./bin/sync-all.sh

2
data/.gitignore поставляемый Normal file
Просмотреть файл

@ -0,0 +1,2 @@
**/*
!.gitignore

Просмотреть файл

@ -43,7 +43,6 @@ services:
- ./wsgi/:/app/wsgi:delegated
- ./locale/:/app/locale:delegated
- ./l10n/:/app/l10n:delegated
- ./git-repos/:/app/git-repos:delegated
# run the tests against local changes
test:
@ -61,7 +60,6 @@ services:
- ./tests/:/app/tests:delegated
- ./wsgi/:/app/wsgi:delegated
- ./l10n/:/app/l10n:delegated
- ./git-repos/:/app/git-repos:delegated
# run tests with no volumes
test-image:
@ -93,7 +91,6 @@ services:
- ./wsgi/:/app/wsgi:delegated
- ./locale/:/app/locale:delegated
- ./l10n/:/app/l10n:delegated
- ./git-repos/:/app/git-repos:delegated
builder:
build:

Просмотреть файл

@ -163,7 +163,7 @@ recipe path.
Before you run this, make sure to update the local clones of your l10n repositories.
This command will use the Fluent file you edited as template, read the legacy translations
from ``locale`` and write the generated Fluent files for each locale into the ``git-repos/www-l10n/`` directory.
from ``locale`` and write the generated Fluent files for each locale into the ``data/www-l10n/`` directory.
Append each locale you wish to convert in a space-separated list. You can find the full
list of supported locales in the `Pontoon configuration file`_ and below in a space-separated

Просмотреть файл

@ -20,7 +20,7 @@ If you don't already have a ``locale`` directory, this command will clone the
git repo containing the .lang translation files (either the dev or prod files
depending on your ``DEV`` setting). If the folder is already present, it will
update the repository to the latest version. It do the same thing for the
repository for the .ftl translation files in ``git-repos/www-l10n`` directory.
repository for the .ftl translation files in ``data/www-l10n`` directory.
Fluent
======

Просмотреть файл

@ -114,11 +114,9 @@ def translate(text, files):
for file_ in files:
key = "dotlang-%s-%s" % (lang, file_)
rel_path = os.path.join('locale', lang, '%s.lang' % file_)
path = str(settings.LOCALES_PATH / lang / f'{file_}.lang')
trans = cache.get(key)
if trans is None:
path = os.path.join(settings.ROOT, rel_path)
trans = parse(path)
cache.set(key, trans)
@ -130,7 +128,7 @@ def translate(text, files):
'replaced text (aka %s)')
message = '%s\n\n%s\n%s' % (explanation, text,
trans[tweaked_text])
mail_error(rel_path, message)
mail_error(path, message)
return Markup(text)
return Markup(trans[tweaked_text])
return Markup(text)
@ -225,12 +223,11 @@ def lang_file_tag_set(path, lang=None):
return ALL_THE_THINGS
lang = lang or translation.get_language(True)
rel_path = os.path.join('locale', lang, '%s.lang' % path)
cache_key = 'tag:%s' % rel_path
fpath = settings.LOCALES_PATH / lang / f'{path}.lang'
cache_key = f'tag:locale/{lang}/{path}.lang'
tag_set = cache.get(cache_key)
if tag_set is None:
tag_set = set()
fpath = os.path.join(settings.ROOT, rel_path)
try:
with codecs.open(fpath, 'r', 'utf-8', errors='replace') as lines:
for line in lines:

Просмотреть файл

@ -76,18 +76,18 @@ def parse_po(path):
def po_msgs(domain):
return parse_po(join(settings.ROOT, 'locale', 'templates', 'LC_MESSAGES',
'{}.pot'.format(domain)))
path = settings.LOCALES_PATH / 'templates' / 'LC_MESSAGES' / f'{domain}.pot'
return parse_po(str(path))
def translated_strings(file_):
path = join(settings.ROOT, 'locale', 'templates', file_)
path = str(settings.LOCALES_PATH / 'templates', file_)
trans = list(parse_lang(path, skip_untranslated=False).keys())
return trans
def lang_file(name, lang):
return join(settings.ROOT, 'locale', lang, name)
return str(settings.LOCALES_PATH / lang / name)
def is_template(path):

Просмотреть файл

@ -20,7 +20,7 @@ from jinja2 import Environment
def l10n_file(*args):
return path.join(settings.ROOT, 'locale', *args)
return str(settings.LOCALES_PATH.joinpath(*args))
def list_templates():

Просмотреть файл

@ -150,8 +150,7 @@ class Command(BaseCommand):
filenames = options['filenames']
if filenames:
# mimics puente.management.commands.extract for a list of files
outputdir = os.path.join(settings.ROOT, 'locale', 'templates',
'LC_MESSAGES')
outputdir = str(settings.LOCALES_PATH / 'templates' / 'LC_MESSAGES')
if not os.path.isdir(outputdir):
os.makedirs(outputdir)

Просмотреть файл

@ -20,7 +20,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
langs = options['langs']
if not langs:
langs = os.listdir(os.path.join(settings.ROOT, 'locale'))
langs = os.listdir(str(settings.LOCALES_PATH))
langs = [x for x in langs if x != 'templates']
langs = [x for x in langs if x[0] != '.']

Просмотреть файл

@ -6,6 +6,7 @@
import codecs
from os import path
from io import StringIO, IOBase
from pathlib import Path
from textwrap import dedent
from django.conf import settings
@ -23,9 +24,10 @@ from lib.l10n_utils.management.commands.l10n_check import (
from lib.l10n_utils.management.commands.l10n_extract import extract_from_files
from lib.l10n_utils.tests import capture_stdio
ROOT = path.join(path.dirname(path.abspath(__file__)), 'test_files')
TEMPLATE_DIRS = (path.join(ROOT, 'templates'),)
ROOT_PATH = Path(__file__).with_name('test_files')
ROOT = str(ROOT_PATH)
LOCALES_PATH = ROOT_PATH / 'locale'
TEMPLATE_DIRS = (str(ROOT_PATH / 'templates'),)
METHODS = [
('templates/**.html',
@ -38,7 +40,7 @@ TRUE_MOCK = Mock()
TRUE_MOCK.return_value = True
@override_settings(ROOT=ROOT)
@override_settings(ROOT=ROOT, LOCALES_PATH=LOCALES_PATH)
class TestL10nExtract(TestCase):
def test_extract_from_files(self):
"""
@ -116,7 +118,7 @@ class TestL10nExtract(TestCase):
callback.assert_called_once_with(testfile[0], METHODS[0][1], ANY)
@override_settings(ROOT=ROOT)
@override_settings(LOCALES_PATH=LOCALES_PATH)
class TestL10nCheck(TestCase):
def _get_block(self, blocks, name):
"""Out of all blocks, grab the one with the specified name."""
@ -328,7 +330,7 @@ class TestL10nCheck(TestCase):
self.assertEqual(open_buffer.getvalue(), good_value)
@override_settings(ROOT=ROOT)
@override_settings(LOCALES_PATH=LOCALES_PATH)
class Testl10nMerge(TestCase):
@patch('lib.l10n_utils.gettext.settings.ROOT', ROOT)
@patch('lib.l10n_utils.gettext._append_to_lang_file')
@ -338,7 +340,7 @@ class Testl10nMerge(TestCase):
Bug 861168.
"""
merge_lang_files(['de'])
dest_file = path.join(ROOT, 'locale', 'de', 'firefox', 'fx.lang')
dest_file = str(LOCALES_PATH / 'de' / 'firefox' / 'fx.lang')
write_mock.assert_called_once_with(dest_file,
[[None, u'Find out if your device is '
u'supported &nbsp;\xbb']])

Просмотреть файл

@ -25,6 +25,7 @@ from lib.l10n_utils.extract import extract_python
LANG_FILES = 'test_file'
ROOT_PATH = Path(__file__).with_name('test_files')
LOCALES_PATH = ROOT_PATH / 'locale'
ROOT = str(ROOT_PATH)
TEMPLATE_DIRS = [str(ROOT_PATH.joinpath('templates'))]
jinja_env = Jinja2.get_default().env
@ -34,6 +35,7 @@ jinja_env = Jinja2.get_default().env
DEV=False,
ROOT_URLCONF='lib.l10n_utils.tests.test_files.urls',
ROOT=ROOT,
LOCALES_PATH=LOCALES_PATH,
LANGUAGE_CODE='en-US', # Triggers reset of Django's language cache
)
@patch.object(jinja_env.loader, 'searchpath', TEMPLATE_DIRS)
@ -201,6 +203,7 @@ class TestDotlang(TestCase):
@override_settings(
DEV=False,
ROOT=ROOT,
LOCALES_PATH=LOCALES_PATH,
LANGUAGE_CODE='en-US', # Triggers reset of Django's language cache
EMAIL_SUBJECT_PREFIX='[bedrock] ',
MANAGERS=('dude@example.com',),
@ -212,12 +215,13 @@ class TestDotlang(TestCase):
result = translate(expected, [path])
assert expected == result
assert len(mail.outbox) == 1
assert mail.outbox[0].subject == '[bedrock] locale/fr/%s.lang is corrupted' % path
assert mail.outbox[0].subject == f'[bedrock] {LOCALES_PATH}/fr/{path}.lang is corrupted'
mail.outbox = []
@override_settings(
DEV=False,
ROOT=ROOT,
LOCALES_PATH=LOCALES_PATH,
LANGUAGE_CODE='en-US', # Triggers reset of Django's language cache
)
def test_format_identifier_order(self):
@ -236,6 +240,7 @@ class TestDotlang(TestCase):
@override_settings(
ROOT_URLCONF='lib.l10n_utils.tests.test_files.urls',
ROOT=ROOT,
LOCALES_PATH=LOCALES_PATH,
LANGUAGE_CODE='en-US', # Triggers reset of Django's language cache
)
def test_lang_files_queried_in_order(self):
@ -248,6 +253,7 @@ class TestDotlang(TestCase):
@override_settings(
DEV=False,
ROOT=ROOT,
LOCALES_PATH=LOCALES_PATH,
LANGUAGE_CODE='en-US', # Triggers reset of Django's language cache
)
def test_extract_message_tweaks_do_not_break(self):
@ -427,6 +433,7 @@ class TestDotlang(TestCase):
DEV=False,
ROOT_URLCONF='lib.l10n_utils.tests.test_files.urls',
ROOT=ROOT,
LOCALES_PATH=LOCALES_PATH,
LANGUAGE_CODE='en-US', # Triggers reset of Django's language cache
)
class TestTranslationList(TestCase):

Просмотреть файл

@ -5,6 +5,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
from pathlib import Path
from django.conf import settings
from django.core.cache import caches
@ -20,7 +21,9 @@ from lib.l10n_utils.gettext import (_append_to_lang_file, langfiles_for_path,
from lib.l10n_utils.tests import TempFileMixin
cache = caches['l10n']
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'test_files')
ROOT_PATH = Path(__file__).with_name('test_files')
LOCALES_PATH = ROOT_PATH / 'locale'
ROOT = str(ROOT_PATH)
TEMPLATE_DIRS = (os.path.join(ROOT, 'templates'))
DOTLANG_FILES = ['dude', 'walter', 'donny']
@ -83,7 +86,7 @@ class TestPOFiles(TestCase):
[None, u'The Dude minds!'],
]
@override_settings(ROOT=ROOT)
@override_settings(ROOT=ROOT, LOCALES_PATH=LOCALES_PATH)
def test_parse_po(self):
"""Should return correct messages"""
msgs = po_msgs('messages')
@ -100,7 +103,7 @@ class TestPOFiles(TestCase):
}
self.assertDictEqual(msgs, expected)
@override_settings(ROOT=ROOT)
@override_settings(ROOT=ROOT, LOCALES_PATH=LOCALES_PATH)
@patch('lib.l10n_utils.gettext._append_to_lang_file')
@patch('lib.l10n_utils.gettext.langfiles_for_path')
def test_po_to_langfiles(self, langfiles_mock, append_mock):
@ -154,7 +157,9 @@ class TestPOFiles(TestCase):
_append_to_lang_file(path_new, {})
assert md_mock.called
@override_settings(ROOT=ROOT, DOTLANG_FILES=DOTLANG_FILES)
@override_settings(ROOT=ROOT,
LOCALES_PATH=LOCALES_PATH,
DOTLANG_FILES=DOTLANG_FILES)
@patch('lib.l10n_utils.gettext.parse_lang')
@patch('lib.l10n_utils.gettext.codecs', MagicMock())
def test_uses_default_lang_files(self, pl_mock):

Просмотреть файл

@ -17,6 +17,7 @@ from bedrock.mozorg.tests import TestCase
ROOT_PATH = Path(__file__).with_name('test_files')
LOCALES_PATH = ROOT_PATH / 'locale'
ROOT = str(ROOT_PATH)
TEMPLATE_DIRS = [str(ROOT_PATH.joinpath('templates'))]
jinja_env = Jinja2.get_default().env
@ -38,6 +39,7 @@ class TestL10nBlocks(TestCase):
@patch.object(jinja_env.loader, 'searchpath', TEMPLATE_DIRS)
@override_settings(
LOCALES_PATH=LOCALES_PATH,
ROOT=ROOT,
ROOT_URLCONF='lib.l10n_utils.tests.test_files.urls',
)
@ -64,6 +66,7 @@ class TestTransBlocks(TestCase):
@patch.object(jinja_env.loader, 'searchpath', TEMPLATE_DIRS)
@override_settings(
LOCALES_PATH=LOCALES_PATH,
ROOT=ROOT,
ROOT_URLCONF='lib.l10n_utils.tests.test_files.urls',
DOTLANG_FILES=['download_button', 'main', 'footer']