Remove HIDDEN_LANGUAGES, sync with .po files and product-details. (#6536)

* Remove HIDDEN_LANGUAGES, sync with .po files and product-details.

Fixes #6534, Fixes #6535 

* Remove test referencing cy language, imho it doesn't make sense to have languages lying around in the code that aren't in product-details anyway, just adds another layer of writing things down needlessly

* Add comment about bidi languages

* Add test

* Remove fa-IR from languages, we never supported it in the past.

* Remove custom locales from test settings.

settings_test.py

@@ -80,20 +80,6 @@ ALLOW_SELF_REVIEWS = True
 # Make sure the debug toolbar isn't used during the tests.
 INSTALLED_APPS = [app for app in INSTALLED_APPS if app != 'debug_toolbar']
 
-# These are the default languages. If you want a constrainted set for your
-# tests, you should add those in the tests.
-AMO_LANGUAGES = (
-    'af', 'ar', 'bg', 'ca', 'cs', 'da', 'de', 'el', 'en-US', 'en-GB', 'es',
-    'eu', 'fa', 'fi', 'fr', 'ga-IE', 'he', 'hu', 'id', 'it', 'ja', 'ko', 'mn',
-    'nl', 'pl', 'pt-BR', 'pt-PT', 'ro', 'ru', 'sk', 'sl', 'sq', 'sv-SE', 'uk',
-    'vi', 'zh-CN', 'zh-TW',
-)
-
-# Make sure we run our tests with debug languages.
-AMO_LANGUAGES = AMO_LANGUAGES + DEBUG_LANGUAGES
-
-LANGUAGES = lazy(lazy_langs, dict)(AMO_LANGUAGES)
-LANGUAGE_URL_MAP = dict([(i.lower(), i) for i in AMO_LANGUAGES])
 TASK_USER_ID = 1337
 
 ES_DEFAULT_NUM_REPLICAS = 0

src/olympia/addons/models.py

@@ -547,9 +547,7 @@ class Addon(OnChangeMixin, ModelBase):
 
         addon.status = amo.STATUS_NULL
         locale_is_set = (addon.default_locale and
-                         addon.default_locale in (
-                             settings.AMO_LANGUAGES +
-                             settings.HIDDEN_LANGUAGES) and
+                         addon.default_locale in settings.AMO_LANGUAGES and
                          data.get('default_locale') == addon.default_locale)
         if not locale_is_set:
             addon.default_locale = to_language(trans_real.get_language())

src/olympia/amo/tests/test_amo_utils.py

@@ -110,7 +110,6 @@ def test_to_language(test_input, expected):
     ('en-us', 'en-US'),
     ('en_US', 'en-US'),
     ('en', 'en-US'),
-    ('cy', 'cy'),  # A hidden language.
     ('FR', 'fr'),
     ('es-ES', None),  # We don't go from specific to generic.
     ('xxx', None),

src/olympia/amo/tests/test_utils_.py

@@ -184,7 +184,11 @@ def test_get_locale_from_lang(lang):
     locale = get_locale_from_lang(lang)
 
     debug_languages = ('dbg', 'dbr', 'dbl')
-    expected_language = lang[:2] if lang not in debug_languages else 'en'
+    long_languages = ('hsb', 'dsb', 'kab')
+    expected_language = (
+        lang[:3] if lang in long_languages else (
+            lang[:2] if lang not in debug_languages else 'en'
+        ))
 
     assert isinstance(locale, Locale)
     assert locale.language == expected_language
@@ -194,3 +198,9 @@ def test_get_locale_from_lang(lang):
     if separator:
         territory = lang.split(separator)[1]
         assert locale.territory == territory
+
+
+@pytest.mark.parametrize('lang', settings.LANGUAGES_BIDI)
+def test_bidi_language_in_amo_languages(lang):
+    """Make sure all bidi marked locales are in AMO_LANGUAGES too."""
+    assert lang in settings.AMO_LANGUAGES or lang in settings.DEBUG_LANGUAGES

src/olympia/amo/utils.py

@@ -827,9 +827,7 @@ def find_language(locale):
     if not locale:
         return None
 
-    LANGS = settings.AMO_LANGUAGES + settings.HIDDEN_LANGUAGES
-
-    if locale in LANGS:
+    if locale in settings.AMO_LANGUAGES:
         return locale
 
     # Check if locale has a short equivalent.
@@ -839,7 +837,7 @@ def find_language(locale):
 
     # Check if locale is something like en_US that needs to be converted.
     locale = to_language(locale)
-    if locale in LANGS:
+    if locale in settings.AMO_LANGUAGES:
         return locale
 
     return None

src/olympia/lib/settings_base.py

@@ -153,21 +153,62 @@ LANGUAGE_CODE = 'en-US'
 # Note: If you update this list, don't forget to also update the locale
 # permissions in the database.
 AMO_LANGUAGES = (
-    'af', 'ar', 'bg', 'bn-BD', 'ca', 'cs', 'da', 'de', 'dsb',
-    'el', 'en-GB', 'en-US', 'es', 'eu', 'fa', 'fi', 'fr', 'ga-IE', 'he', 'hu',
-    'hsb', 'id', 'it', 'ja', 'ka', 'kab', 'ko', 'nn-NO', 'mk', 'mn', 'nl',
-    'pl', 'pt-BR', 'pt-PT', 'ro', 'ru', 'sk', 'sl', 'sq', 'sv-SE', 'uk', 'ur',
-    'vi', 'zh-CN', 'zh-TW',
+    'af',  # Afrikaans
+    'ar',  # Arabic
+    'bg',  # Bulgarian
+    'bn-BD',  # Bengali (Bangladesh)
+    'ca',  # Catalan
+    'cs',  # Czech
+    'da',  # Danish
+    'de',  # German
+    'dsb',  # Lower Sorbian
+    'el',  # Greek
+    'en-GB',  # English (British)
+    'en-US',  # English (US)
+    'es',  # Spanish
+    'eu',  # Basque
+    'fa',  # Persian
+    'fi',  # Finnish
+    'fr',  # French
+    'ga-IE',  # Irish
+    'he',  # Hebrew
+    'hsb',  # Upper Sorbian
+    'hu',  # Hungarian
+    'id',  # Indonesian
+    'it',  # Italian
+    'ja',  # Japanese
+    'ka',  # Georgian
+    'kab',  # Kabyle
+    'ko',  # Korean
+    'mk',  # Macedonian
+    'mn',  # Mongolian
+    'nl',  # Dutch
+    'nn-NO',  # Norwegian (Nynorsk)
+    'pl',  # Polish
+    'pt-BR',  # Portuguese (Brazilian)
+    'pt-PT',  # Portuguese (Portugal)
+    'ro',  # Romanian
+    'ru',  # Russian
+    'sk',  # Slovak
+    'sl',  # Slovenian
+    'sq',  # Albanian
+    'sv-SE',  # Swedish
+    'uk',  # Ukrainian
+    'ur',  # Urdu
+    'vi',  # Vietnamese
+    'zh-CN',  # Chinese (Simplified)
+    'zh-TW',  # Chinese (Traditional)
 )
 
+# Bidirectional languages.
+# Locales in here *must* be in `AMO_LANGUAGES` too.
+LANGUAGES_BIDI = ('ar', 'fa', 'he', 'dbr', 'ur')
+
 # Explicit conversion of a shorter language code into a more specific one.
 SHORTER_LANGUAGES = {
     'en': 'en-US', 'ga': 'ga-IE', 'pt': 'pt-PT', 'sv': 'sv-SE', 'zh': 'zh-CN'
 }
 
-# Not shown on the site, but .po files exist and these are available on the
-# L10n dashboard.  Generally languages start here and move into AMO_LANGUAGES.
-HIDDEN_LANGUAGES = ('cy', 'hr', 'sr', 'sr-Latn', 'tr')
 
 DEBUG_LANGUAGES = ('dbr', 'dbl')
 
@@ -198,7 +239,6 @@ PROD_DETAILS_STORAGE = 'olympia.lib.product_details_backend.NoCachePDFileStorage
 
 # Override Django's built-in with our native names
 LANGUAGES = lazy(lazy_langs, dict)(AMO_LANGUAGES)
-LANGUAGES_BIDI = ('ar', 'fa', 'fa-IR', 'he', 'dbr', 'ur')
 
 LANGUAGE_URL_MAP = dict([(i.lower(), i) for i in AMO_LANGUAGES])
 

src/olympia/pages/views.py

@@ -29,9 +29,7 @@ def credits(request):
                       .order_by('display_name')
                       .distinct())
 
-    languages = sorted(list(
-        set(settings.AMO_LANGUAGES + settings.HIDDEN_LANGUAGES) -
-        set(['en-US'])))
+    languages = sorted(list(set(settings.AMO_LANGUAGES) - set(['en-US'])))
 
     localizers = []
     for lang in languages:

src/olympia/translations/fields.py

@@ -196,8 +196,9 @@ class TranslationDescriptor(related.ReverseSingleRelatedObjectDescriptor):
         rv = None
         for locale, string in dict_.items():
             loc = amo_to_language(locale)
-            if loc not in settings.AMO_LANGUAGES + settings.HIDDEN_LANGUAGES:
+            if loc not in settings.AMO_LANGUAGES:
                 continue
+
             # The Translation is created and saved in here.
             trans = self.translation_from_string(instance, locale, string)
 

src/olympia/translations/tests/test_models.py

@@ -230,16 +230,6 @@ class TranslationTestCase(BaseTestCase):
         translation.activate('fr')
         self.trans_eq(get_model().name, 'oui', 'fr')
 
-    def test_dict_with_hidden_locale(self):
-        with self.settings(HIDDEN_LANGUAGES=('xxx',)):
-            o = TranslatedModel.objects.get(id=1)
-            o.name = {'en-US': 'active language', 'xxx': 'hidden language',
-                      'de': 'another language'}
-            o.save()
-        ts = Translation.objects.filter(id=o.name_id)
-        assert sorted(ts.values_list('locale', flat=True)) == (
-            ['de', 'en-US', 'xxx'])
-
     def test_dict_bad_locale(self):
         m = TranslatedModel.objects.get(id=1)
         m.name = {'de': 'oof', 'xxx': 'bam', 'es': 'si'}