From aa1fcbd4352d7ed4e7388ea5923d1640c4a1134d Mon Sep 17 00:00:00 2001 From: Christopher Grebs Date: Wed, 4 Oct 2017 08:35:31 +0200 Subject: [PATCH] Remove HIDDEN_LANGUAGES, sync with .po files and product-details. (#6536) * Remove HIDDEN_LANGUAGES, sync with .po files and product-details. Fixes #6534, Fixes #6535 * Remove test referencing cy language, imho it doesn't make sense to have languages lying around in the code that aren't in product-details anyway, just adds another layer of writing things down needlessly * Add comment about bidi languages * Add test * Remove fa-IR from languages, we never supported it in the past. * Remove custom locales from test settings. --- locale/testing.milos | 0 settings_test.py | 14 ----- src/olympia/addons/models.py | 4 +- src/olympia/amo/tests/test_amo_utils.py | 1 - src/olympia/amo/tests/test_utils_.py | 12 +++- src/olympia/amo/utils.py | 6 +- src/olympia/lib/settings_base.py | 58 ++++++++++++++++--- src/olympia/pages/views.py | 4 +- src/olympia/translations/fields.py | 3 +- src/olympia/translations/tests/test_models.py | 10 ---- 10 files changed, 66 insertions(+), 46 deletions(-) delete mode 100644 locale/testing.milos diff --git a/locale/testing.milos b/locale/testing.milos deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/settings_test.py b/settings_test.py index 39dc509d27..5ed79af4b2 100644 --- a/settings_test.py +++ b/settings_test.py @@ -80,20 +80,6 @@ ALLOW_SELF_REVIEWS = True # Make sure the debug toolbar isn't used during the tests. INSTALLED_APPS = [app for app in INSTALLED_APPS if app != 'debug_toolbar'] -# These are the default languages. If you want a constrainted set for your -# tests, you should add those in the tests. -AMO_LANGUAGES = ( - 'af', 'ar', 'bg', 'ca', 'cs', 'da', 'de', 'el', 'en-US', 'en-GB', 'es', - 'eu', 'fa', 'fi', 'fr', 'ga-IE', 'he', 'hu', 'id', 'it', 'ja', 'ko', 'mn', - 'nl', 'pl', 'pt-BR', 'pt-PT', 'ro', 'ru', 'sk', 'sl', 'sq', 'sv-SE', 'uk', - 'vi', 'zh-CN', 'zh-TW', -) - -# Make sure we run our tests with debug languages. -AMO_LANGUAGES = AMO_LANGUAGES + DEBUG_LANGUAGES - -LANGUAGES = lazy(lazy_langs, dict)(AMO_LANGUAGES) -LANGUAGE_URL_MAP = dict([(i.lower(), i) for i in AMO_LANGUAGES]) TASK_USER_ID = 1337 ES_DEFAULT_NUM_REPLICAS = 0 diff --git a/src/olympia/addons/models.py b/src/olympia/addons/models.py index e992162356..f939d985cf 100644 --- a/src/olympia/addons/models.py +++ b/src/olympia/addons/models.py @@ -547,9 +547,7 @@ class Addon(OnChangeMixin, ModelBase): addon.status = amo.STATUS_NULL locale_is_set = (addon.default_locale and - addon.default_locale in ( - settings.AMO_LANGUAGES + - settings.HIDDEN_LANGUAGES) and + addon.default_locale in settings.AMO_LANGUAGES and data.get('default_locale') == addon.default_locale) if not locale_is_set: addon.default_locale = to_language(trans_real.get_language()) diff --git a/src/olympia/amo/tests/test_amo_utils.py b/src/olympia/amo/tests/test_amo_utils.py index 31bc27ce7f..78b479d16b 100644 --- a/src/olympia/amo/tests/test_amo_utils.py +++ b/src/olympia/amo/tests/test_amo_utils.py @@ -110,7 +110,6 @@ def test_to_language(test_input, expected): ('en-us', 'en-US'), ('en_US', 'en-US'), ('en', 'en-US'), - ('cy', 'cy'), # A hidden language. ('FR', 'fr'), ('es-ES', None), # We don't go from specific to generic. ('xxx', None), diff --git a/src/olympia/amo/tests/test_utils_.py b/src/olympia/amo/tests/test_utils_.py index c20544b235..e90ae90786 100644 --- a/src/olympia/amo/tests/test_utils_.py +++ b/src/olympia/amo/tests/test_utils_.py @@ -184,7 +184,11 @@ def test_get_locale_from_lang(lang): locale = get_locale_from_lang(lang) debug_languages = ('dbg', 'dbr', 'dbl') - expected_language = lang[:2] if lang not in debug_languages else 'en' + long_languages = ('hsb', 'dsb', 'kab') + expected_language = ( + lang[:3] if lang in long_languages else ( + lang[:2] if lang not in debug_languages else 'en' + )) assert isinstance(locale, Locale) assert locale.language == expected_language @@ -194,3 +198,9 @@ def test_get_locale_from_lang(lang): if separator: territory = lang.split(separator)[1] assert locale.territory == territory + + +@pytest.mark.parametrize('lang', settings.LANGUAGES_BIDI) +def test_bidi_language_in_amo_languages(lang): + """Make sure all bidi marked locales are in AMO_LANGUAGES too.""" + assert lang in settings.AMO_LANGUAGES or lang in settings.DEBUG_LANGUAGES diff --git a/src/olympia/amo/utils.py b/src/olympia/amo/utils.py index ca7191d85e..c0b8e0b34e 100644 --- a/src/olympia/amo/utils.py +++ b/src/olympia/amo/utils.py @@ -827,9 +827,7 @@ def find_language(locale): if not locale: return None - LANGS = settings.AMO_LANGUAGES + settings.HIDDEN_LANGUAGES - - if locale in LANGS: + if locale in settings.AMO_LANGUAGES: return locale # Check if locale has a short equivalent. @@ -839,7 +837,7 @@ def find_language(locale): # Check if locale is something like en_US that needs to be converted. locale = to_language(locale) - if locale in LANGS: + if locale in settings.AMO_LANGUAGES: return locale return None diff --git a/src/olympia/lib/settings_base.py b/src/olympia/lib/settings_base.py index 6fed67bbad..9d97316714 100644 --- a/src/olympia/lib/settings_base.py +++ b/src/olympia/lib/settings_base.py @@ -153,21 +153,62 @@ LANGUAGE_CODE = 'en-US' # Note: If you update this list, don't forget to also update the locale # permissions in the database. AMO_LANGUAGES = ( - 'af', 'ar', 'bg', 'bn-BD', 'ca', 'cs', 'da', 'de', 'dsb', - 'el', 'en-GB', 'en-US', 'es', 'eu', 'fa', 'fi', 'fr', 'ga-IE', 'he', 'hu', - 'hsb', 'id', 'it', 'ja', 'ka', 'kab', 'ko', 'nn-NO', 'mk', 'mn', 'nl', - 'pl', 'pt-BR', 'pt-PT', 'ro', 'ru', 'sk', 'sl', 'sq', 'sv-SE', 'uk', 'ur', - 'vi', 'zh-CN', 'zh-TW', + 'af', # Afrikaans + 'ar', # Arabic + 'bg', # Bulgarian + 'bn-BD', # Bengali (Bangladesh) + 'ca', # Catalan + 'cs', # Czech + 'da', # Danish + 'de', # German + 'dsb', # Lower Sorbian + 'el', # Greek + 'en-GB', # English (British) + 'en-US', # English (US) + 'es', # Spanish + 'eu', # Basque + 'fa', # Persian + 'fi', # Finnish + 'fr', # French + 'ga-IE', # Irish + 'he', # Hebrew + 'hsb', # Upper Sorbian + 'hu', # Hungarian + 'id', # Indonesian + 'it', # Italian + 'ja', # Japanese + 'ka', # Georgian + 'kab', # Kabyle + 'ko', # Korean + 'mk', # Macedonian + 'mn', # Mongolian + 'nl', # Dutch + 'nn-NO', # Norwegian (Nynorsk) + 'pl', # Polish + 'pt-BR', # Portuguese (Brazilian) + 'pt-PT', # Portuguese (Portugal) + 'ro', # Romanian + 'ru', # Russian + 'sk', # Slovak + 'sl', # Slovenian + 'sq', # Albanian + 'sv-SE', # Swedish + 'uk', # Ukrainian + 'ur', # Urdu + 'vi', # Vietnamese + 'zh-CN', # Chinese (Simplified) + 'zh-TW', # Chinese (Traditional) ) +# Bidirectional languages. +# Locales in here *must* be in `AMO_LANGUAGES` too. +LANGUAGES_BIDI = ('ar', 'fa', 'he', 'dbr', 'ur') + # Explicit conversion of a shorter language code into a more specific one. SHORTER_LANGUAGES = { 'en': 'en-US', 'ga': 'ga-IE', 'pt': 'pt-PT', 'sv': 'sv-SE', 'zh': 'zh-CN' } -# Not shown on the site, but .po files exist and these are available on the -# L10n dashboard. Generally languages start here and move into AMO_LANGUAGES. -HIDDEN_LANGUAGES = ('cy', 'hr', 'sr', 'sr-Latn', 'tr') DEBUG_LANGUAGES = ('dbr', 'dbl') @@ -198,7 +239,6 @@ PROD_DETAILS_STORAGE = 'olympia.lib.product_details_backend.NoCachePDFileStorage # Override Django's built-in with our native names LANGUAGES = lazy(lazy_langs, dict)(AMO_LANGUAGES) -LANGUAGES_BIDI = ('ar', 'fa', 'fa-IR', 'he', 'dbr', 'ur') LANGUAGE_URL_MAP = dict([(i.lower(), i) for i in AMO_LANGUAGES]) diff --git a/src/olympia/pages/views.py b/src/olympia/pages/views.py index 2bfb20483d..bd03681afa 100644 --- a/src/olympia/pages/views.py +++ b/src/olympia/pages/views.py @@ -29,9 +29,7 @@ def credits(request): .order_by('display_name') .distinct()) - languages = sorted(list( - set(settings.AMO_LANGUAGES + settings.HIDDEN_LANGUAGES) - - set(['en-US']))) + languages = sorted(list(set(settings.AMO_LANGUAGES) - set(['en-US']))) localizers = [] for lang in languages: diff --git a/src/olympia/translations/fields.py b/src/olympia/translations/fields.py index aceacc0d17..e2c2b9ba5b 100644 --- a/src/olympia/translations/fields.py +++ b/src/olympia/translations/fields.py @@ -196,8 +196,9 @@ class TranslationDescriptor(related.ReverseSingleRelatedObjectDescriptor): rv = None for locale, string in dict_.items(): loc = amo_to_language(locale) - if loc not in settings.AMO_LANGUAGES + settings.HIDDEN_LANGUAGES: + if loc not in settings.AMO_LANGUAGES: continue + # The Translation is created and saved in here. trans = self.translation_from_string(instance, locale, string) diff --git a/src/olympia/translations/tests/test_models.py b/src/olympia/translations/tests/test_models.py index 1f627f22c2..2c478a364d 100644 --- a/src/olympia/translations/tests/test_models.py +++ b/src/olympia/translations/tests/test_models.py @@ -230,16 +230,6 @@ class TranslationTestCase(BaseTestCase): translation.activate('fr') self.trans_eq(get_model().name, 'oui', 'fr') - def test_dict_with_hidden_locale(self): - with self.settings(HIDDEN_LANGUAGES=('xxx',)): - o = TranslatedModel.objects.get(id=1) - o.name = {'en-US': 'active language', 'xxx': 'hidden language', - 'de': 'another language'} - o.save() - ts = Translation.objects.filter(id=o.name_id) - assert sorted(ts.values_list('locale', flat=True)) == ( - ['de', 'en-US', 'xxx']) - def test_dict_bad_locale(self): m = TranslatedModel.objects.get(id=1) m.name = {'de': 'oof', 'xxx': 'bam', 'es': 'si'}