Fix missing languages in stats from product_details. (#10595)

* Fix missing languages in stats from product_details.

Some parts actually... need all the languages.

Fixes mozilla/addons#891

* Remove dbg locale, fix ordering for py3

src/olympia/browse/tests.py

@@ -66,8 +66,10 @@ def test_locale_display_name():
         actual = locale_display_name(locale)
         assert actual == (english, native)
 
-    check('el', 'Greek', u'Ελληνικά')
-    check('el-XX', 'Greek', u'Ελληνικά')
+    check('el', u'Greek', u'Ελληνικά')
+    check('el-XX', u'Greek', u'Ελληνικά')
+    check('wo', u'Wolof', u'Wolof')
+    check('es-MX', u'Spanish', u'Espa\xf1ol')
     pytest.raises(KeyError, check, 'fake-lang', '', '')
 
 

src/olympia/browse/views.py

@@ -18,7 +18,7 @@ from olympia.addons.views import BaseFilter
 from olympia.amo.models import manual_order
 from olympia.amo.urlresolvers import reverse
 from olympia.amo.utils import render
-from olympia.core.languages import LANGUAGE_MAPPING
+from olympia.core.languages import ALL_LANGUAGES
 
 
 PAGINATE_PERSONAS_BY = 30
@@ -34,8 +34,8 @@ def locale_display_name(locale):
     if not locale:
         raise KeyError
 
-    if locale.lower() in LANGUAGE_MAPPING:
-        v = LANGUAGE_MAPPING[locale.lower()]
+    if locale.lower() in ALL_LANGUAGES:
+        v = ALL_LANGUAGES[locale.lower()]
         return v['english'], v['native']
     else:
         # Take out the regional portion and try again.
@@ -92,7 +92,7 @@ def _get_locales(addons):
             english, native = locale_display_name(locale)
             # Add the locale as a differentiator if we had to strip the
             # regional portion.
-            if locale not in LANGUAGE_MAPPING:
+            if locale not in ALL_LANGUAGES:
                 native = '%s (%s)' % (native, locale)
             addon.locale_display, addon.locale_native = english, native
         except KeyError:

src/olympia/core/languages.py

@@ -1,3 +1,7 @@
+# Languages that we support on AMO, e.g we have translations for them
+# and they're tested.
+# They're automatically added to `settings.AMO_LANGUAGES` in
+# `lib.settings_base`
 LANGUAGE_MAPPING = {
     'af': {
         'english': u'Afrikaans',
@@ -232,3 +236,395 @@ LANGUAGE_MAPPING = {
         'native': u'\u6b63\u9ad4\u4e2d\u6587 (\u7e41\u9ad4)'
     }
 }
+
+# Languages supported by product-details that we don't have translations for
+# and haven't been tested. It's fine to move languages up to `LANGUAGE_MAPPING`
+# but make sure they're tested.
+# Languages in here are used for example in statistics views.
+ADDITIONAL_PRODUCT_LANGUAGES = {
+    'ach': {
+        'english': u'Acholi',
+        'native': u'Acholi'
+    },
+    u'ak': {
+        'english': u'Akan',
+        'native': u'Akan'
+    },
+    u'am-et': {
+        'english': u'Amharic',
+        'native': u'\u12a0\u121b\u122d\u129b'
+    },
+    u'an': {
+        'english': u'Aragonese',
+        'native': u'aragon\xe9s'
+    },
+    u'as': {
+        'english': u'Assamese',
+        'native': u'\u0985\u09b8\u09ae\u09c0\u09af\u09bc\u09be'
+    },
+    u'azz': {
+        'english': u'Highland Puebla Nahuatl',
+        'native': u'nahuatl sierra norte Puebla'
+    },
+    u'be': {
+        'english': u'Belarusian',
+        'native': u'\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f'  # noqa
+    },
+    u'bm': {
+        'english': u'Bambara',
+        'native': u'Bamanankan'
+    },
+    u'bn-IN': {
+        'english': u'Bengali (India)',
+        'native': u'\u09ac\u09be\u0982\u09b2\u09be (\u09ad\u09be\u09b0\u09a4)'
+    },
+    u'br': {
+        'english': u'Breton',
+        'native': u'Brezhoneg'
+    },
+    u'brx': {
+        'english': u'Bodo',
+        'native': u"\u092c\u0930'"
+    },
+    u'ca-valencia': {
+        'english': u'Catalan (Valencian)',
+        'native': u'catal\xe0 (valenci\xe0)'
+    },
+    u'csb': {
+        'english': u'Kashubian',
+        'native': u'Kasz\xebbsczi'
+    },
+    u'cy': {
+        'english': u'Welsh',
+        'native': u'Cymraeg'
+    },
+    u'de-AT': {
+        'english': u'German (Austria)',
+        'native': u'Deutsch (\xd6sterreich)'
+    },
+    u'de-CH': {
+        'english': u'German (Switzerland)',
+        'native': u'Deutsch (Schweiz)'
+    },
+    u'de-DE': {
+        'english': u'German (Germany)',
+        'native': u'Deutsch (Deutschland)'
+    },
+    u'ee': {
+        'english': u'Ewe',
+        'native': u'E\u028be'
+    },
+    u'en-AU': {
+        'english': u'English (Australian)',
+               'native': u'English (Australian)'},
+    u'en-NZ': {
+        'english': u'English (New Zealand)',
+               'native': u'English (New Zealand)'},
+    u'en-ZA': {
+        'english': u'English (South African)',
+               'native': u'English (South African)'},
+    u'eo': {
+        'english': u'Esperanto',
+        'native': u'Esperanto'
+    },
+    u'es-AR': {
+        'english': u'Spanish (Argentina)',
+               'native': u'Espa\xf1ol (de Argentina)'},
+    u'es-CL': {
+        'english': u'Spanish (Chile)',
+        'native': u'Espa\xf1ol (de Chile)'
+    },
+    u'es-ES': {
+        'english': u'Spanish (Spain)',
+        'native': u'Espa\xf1ol (de Espa\xf1a)'},
+    u'es-MX': {
+        'english': u'Spanish (Mexico)',
+        'native': u'Espa\xf1ol (de M\xe9xico)'},
+    u'et': {
+        'english': u'Estonian',
+        'native': u'Eesti keel'
+    },
+    u'ff': {
+        'english': u'Fulah',
+        'native': u'Pulaar-Fulfulde'
+    },
+    u'fj-FJ': {
+        'english': u'Fijian',
+        'native': u'Vosa vaka-Viti'
+    },
+    u'fur-IT': {
+        'english': u'Friulian',
+        'native': u'Furlan'
+    },
+    u'ga': {
+        'english': u'Irish',
+        'native': u'Gaeilge'
+    },
+    u'gd': {
+        'english': u'Gaelic (Scotland)',
+        'native': u'G\xe0idhlig'
+    },
+    u'gl': {
+        'english': u'Galician',
+        'native': u'Galego'
+    },
+    u'gn': {
+        'english': u'Guarani',
+        'native': u"Ava\xf1e'\u1ebd"
+    },
+    u'gu-IN': {
+        'english': u'Gujarati (India)',
+        'native': u'\u0a97\u0ac1\u0a9c\u0ab0\u0abe\u0aa4\u0ac0 (\u0aad\u0abe\u0ab0\u0aa4)'  # noqa
+    },
+    u'ha': {
+        'english': u'Hausa',
+        'native': u'Hausa'
+    },
+    u'hi': {
+        'english': u'Hindi',
+        'native': u'\u0939\u093f\u0928\u094d\u0926\u0940'},
+    u'hi-IN': {
+        'english': u'Hindi (India)',
+        'native': u'\u0939\u093f\u0928\u094d\u0926\u0940 (\u092d\u093e\u0930\u0924)'  # noqa
+    },
+    u'hr': {
+        'english': u'Croatian',
+        'native': u'Hrvatski'
+    },
+    u'hy-AM': {
+        'english': u'Armenian',
+        'native': u'\u0540\u0561\u0575\u0565\u0580\u0565\u0576'
+    },
+    u'ig': {
+        'english': u'Igbo',
+        'native': u'Igbo'
+    },
+    u'is': {
+        'english': u'Icelandic',
+        'native': u'\xedslenska'
+    },
+    u'ja-JP-mac': {
+        'english': u'Japanese',
+        'native': u'\u65e5\u672c\u8a9e'
+    },
+    u'kk': {
+        'english': u'Kazakh',
+        'native': u'\u049a\u0430\u0437\u0430\u049b'
+    },
+    u'km': {
+        'english': u'Khmer',
+        'native': u'\u1781\u17d2\u1798\u17c2\u179a'
+    },
+    u'kn': {
+        'english': u'Kannada',
+        'native': u'\u0c95\u0ca8\u0ccd\u0ca8\u0ca1'
+    },
+    u'kok': {
+        'english': u'Konkani',
+        'native': u'\u0915\u094b\u0902\u0915\u0928\u0940'
+    },
+    u'ks': {
+        'english': u'Kashmiri',
+        'native': u'\u0643\u0634\u0645\u06cc\u0631\u06cc'
+    },
+    u'ku': {
+        'english': u'Kurdish',
+        'native': u'Kurd\xee'
+    },
+    u'la': {
+        'english': u'Latin',
+        'native': u'Latina'
+    },
+    u'lg': {
+        'english': u'Luganda',
+        'native': u'Luganda'
+    },
+    u'lij': {
+        'english': u'Ligurian',
+        'native': u'Ligure'
+    },
+    u'ln': {
+        'english': u'Lingala',
+        'native': u'Ling\xe1la'
+    },
+    u'lo': {
+        'english': u'Lao',
+        'native': u'\u0e9e\u0eb2\u0eaa\u0eb2\u0ea5\u0eb2\u0ea7'
+    },
+    u'lt': {
+        'english': u'Lithuanian',
+        'native': u'Lietuvi\u0173'
+    },
+    u'ltg': {
+        'english': u'Latgalian',
+        'native': u'Latgalie\u0161u valoda'
+    },
+    u'lv': {
+        'english': u'Latvian',
+        'native': u'Latvie\u0161u'
+    },
+    u'mai': {
+        'english': u'Maithili',
+        'native': u'\u092e\u0948\u0925\u093f\u0932\u0940 \u09ae\u09c8\u09a5\u09bf\u09b2\u09c0'  # noqa
+    },
+    u'mg': {
+        'english': u'Malagasy',
+        'native': u'Malagasy'
+    },
+    u'mi': {
+        'english': u'Maori (Aotearoa)',
+        'native': u'M\u0101ori (Aotearoa)'
+    },
+    u'ml': {
+        'english': u'Malayalam',
+        'native': u'\u0d2e\u0d32\u0d2f\u0d3e\u0d33\u0d02'
+    },
+    u'mr': {
+        'english': u'Marathi',
+        'native': u'\u092e\u0930\u093e\u0920\u0940'
+    },
+    u'my': {
+        'english': u'Burmese',
+        'native': u'\u1019\u103c\u1014\u103a\u1019\u102c\u1018\u102c\u101e\u102c'  # noqa
+    },
+    u'ne-NP': {
+        'english': u'Nepali',
+        'native': u'\u0928\u0947\u092a\u093e\u0932\u0940'
+    },
+    u'nr': {
+        'english': u'Ndebele, South',
+        'native': u'isiNdebele'
+    },
+    u'nso': {
+        'english': u'Northern Sotho',
+        'native': u'Sepedi'
+    },
+    u'oc': {
+        'english': u'Occitan (Lengadocian)',
+        'native': u'occitan (lengadocian)'},
+    u'or': {
+        'english': u'Odia',
+        'native': u'\u0b13\u0b21\u0b3c\u0b3f\u0b06'
+    },
+    u'pa': {
+        'english': u'Punjabi',
+        'native': u'\u0a2a\u0a70\u0a1c\u0a3e\u0a2c\u0a40'
+    },
+    u'rm': {
+        'english': u'Romansh',
+        'native': u'rumantsch'
+    },
+    u'rw': {
+        'english': u'Kinyarwanda',
+        'native': u'Ikinyarwanda'
+    },
+    u'sa': {
+        'english': u'Sanskrit',
+        'native': u'\u0938\u0902\u0938\u094d\u0915\u0943\u0924'
+    },
+    u'sah': {
+        'english': u'Sakha',
+        'native': u'\u0421\u0430\u0445\u0430\u043b\u044b\u044b'
+    },
+    u'sat': {
+        'english': u'Santali',
+        'native': u'\u0938\u0902\u0924\u093e\u0932\u0940'
+    },
+    u'si': {
+        'english': u'Sinhala',
+        'native': u'\u0dc3\u0dd2\u0d82\u0dc4\u0dbd'
+    },
+    u'son': {
+        'english': u'Songhai',
+        'native': u'So\u014bay'
+    },
+    u'sr': {
+        'english': u'Serbian',
+        'native': u'\u0421\u0440\u043f\u0441\u043a\u0438'
+    },
+    u'sr-Cyrl': {
+        'english': u'Serbian',
+        'native': u'\u0421\u0440\u043f\u0441\u043a\u0438'
+    },
+    u'sr-Latn': {
+        'english': u'Serbian',
+        'native': u'Srpski'
+    },
+    u'ss': {
+        'english': u'Siswati',
+        'native': u'siSwati'
+    },
+    u'st': {
+        'english': u'Southern Sotho',
+        'native': u'Sesotho'
+    },
+    u'sw': {
+        'english': u'Swahili',
+        'native': u'Kiswahili'
+    },
+    u'ta': {
+        'english': u'Tamil',
+        'native': u'\u0ba4\u0bae\u0bbf\u0bb4\u0bcd'
+    },
+    u'ta-IN': {
+        'english': u'Tamil (India)',
+        'native': u'\u0ba4\u0bae\u0bbf\u0bb4\u0bcd (\u0b87\u0ba8\u0bcd\u0ba4\u0bbf\u0baf\u0bbe)'  # noqa
+    },
+    u'ta-LK': {
+        'english': u'Tamil (Sri Lanka)',
+        'native': u'\u0ba4\u0bae\u0bbf\u0bb4\u0bcd (\u0b87\u0bb2\u0b99\u0bcd\u0b95\u0bc8)'  # noqa
+    },
+    u'tl': {
+        'english': u'Tagalog',
+        'native': u'Tagalog'
+    },
+    u'tn': {
+        'english': u'Tswana',
+        'native': u'Setswana'
+    },
+    u'ts': {
+        'english': u'Tsonga',
+        'native': u'Xitsonga'
+    },
+    u'tsz': {
+        'english': u'Pur\xe9pecha',
+        'native': u'Pur\xe9pecha'
+    },
+    u'tt-RU': {
+        'english': u'Tatar',
+        'native': u'Tatar\xe7a'
+    },
+    u'uz': {
+        'english': u'Uzbek',
+        'native': u'O\u02bbzbek tili'
+    },
+    u've': {
+        'english': u'Venda',
+        'native': u'Tshiven\u1e13a'
+    },
+    u'wo': {
+        'english': u'Wolof',
+        'native': u'Wolof'
+    },
+    u'x-testing': {
+        'english': u'Testing',
+        'native': u'\u0166\u1e17\u015f\u0167\u012b\u019e\u0260'
+    },
+    u'xh': {
+        'english': u'Xhosa',
+        'native': u'isiXhosa'
+    },
+    u'yo': {
+        'english': u'Yoruba',
+        'native': u'Yor\xf9b\xe1'
+    },
+    u'zu': {
+        'english': u'Zulu',
+        'native': u'isiZulu'
+    }
+ }
+
+
+ALL_LANGUAGES = {}
+ALL_LANGUAGES.update(LANGUAGE_MAPPING)
+ALL_LANGUAGES.update(ADDITIONAL_PRODUCT_LANGUAGES)

src/olympia/stats/fixtures/stats/test_models.json

@@ -161,7 +161,7 @@
             "statuses": "{\"userEnabled\": 950, \"userDisabled\": 50}",
             "applications": "{\"{ec8030f7-c20a-464f-9b0e-13a3a9e97384}\": {\"4.0\": 1000}}",
             "oses": "{\"WINNT\": 400, \"Linux\": 300}",
-            "locales": "{\"el\": 400, \"en-US\": 300}",
+            "locales": "{\"el\": 400, \"es-MX\": 400, \"en-US\": 300}",
             "date": "2009-06-01"
         }
     },
@@ -175,7 +175,7 @@
             "statuses": "{\"userEnabled\": 1370, \"userDisabled\": 130}",
             "applications": "{\"{ec8030f7-c20a-464f-9b0e-13a3a9e97384}\": {\"4.0\": 1500}}",
             "oses": "{\"WINNT\": 500, \"Linux\": 400}",
-            "locales": "{\"el\": 400, \"en-US\": 300}",
+            "locales": "{\"el\": 400, \"es-MX\": 400, \"en-US\": 300}",
             "date": "2009-06-02"
         }
     },
@@ -189,7 +189,7 @@
             "statuses": "{\"userEnabled\": 950, \"userDisabled\": 50}",
             "applications": "{\"{ec8030f7-c20a-464f-9b0e-13a3a9e97384}\": {\"4.0\": 1500}}",
             "oses": "{\"WINNT\": 500, \"Linux\": 400}",
-            "locales": "{\"el\": 400, \"en-US\": 300}",
+            "locales": "{\"el\": 400, \"es-MX\": 400, \"en-US\": 300}",
             "date": "2007-01-01"
         }
     },

src/olympia/stats/tests/test_views.py

@@ -307,9 +307,9 @@ class TestCSVs(ESStatsTest):
         assert response.status_code == 200
         self.csv_eq(
             response,
-            u"""date,count,English (US) (en-us),Ελληνικά (el)
-               2009-06-02,1500,300,400
-               2009-06-01,1000,300,400""")
+            u"""date,count,English (US) (en-us),Espa\xf1ol (de M\xe9xico) (es-mx),Ελληνικά (el)
+               2009-06-02,1500,300,400,400
+               2009-06-01,1000,300,400,400""")  # noqa
 
     def test_statuses_series(self):
         response = self.get_view_response('stats.statuses_series',
@@ -469,6 +469,7 @@ class TestResponses(ESStatsTest):
                 "end": "2009-06-02",
                 "data": {
                     u"Ελληνικά (el)": 400,
+                    u'Espa\xf1ol (de M\xe9xico) (es-mx)': 400,
                     u"English (US) (en-us)": 300
                 }
             },
@@ -478,18 +479,20 @@ class TestResponses(ESStatsTest):
                 "end": "2009-06-01",
                 "data": {
                     u"Ελληνικά (el)": 400,
+                    u'Espa\xf1ol (de M\xe9xico) (es-mx)': 400,
                     u"English (US) (en-us)": 300
                 }
-            }
+            },
         ])
 
     def test_usage_by_locale_csv(self):
         response = self.get_view_response(
             'stats.locales_series', group='day', format='csv')
         assert response.status_code == 200
-        self.csv_eq(response, """date,count,English (US) (en-us),Ελληνικά (el)
-                                 2009-06-02,1500,300,400
-                                 2009-06-01,1000,300,400""")
+        self.csv_eq(response,
+            u"""date,count,English (US) (en-us),Espa\xf1ol (de M\xe9xico) (es-mx),Ελληνικά (el)
+               2009-06-02,1500,300,400,400
+               2009-06-01,1000,300,400,400""")  # noqa
 
     def test_usage_by_os_json(self):
         response = self.get_view_response(

src/olympia/stats/views.py

@@ -26,7 +26,7 @@ from olympia.access import acl
 from olympia.amo.decorators import allow_cross_site_request
 from olympia.amo.urlresolvers import reverse
 from olympia.amo.utils import AMOJSONEncoder, render
-from olympia.core.languages import LANGUAGE_MAPPING
+from olympia.core.languages import ALL_LANGUAGES
 from olympia.lib.cache import memoize
 from olympia.stats.decorators import addon_view_stats
 from olympia.stats.forms import DateForm
@@ -292,7 +292,7 @@ def process_locales(series):
     """Convert locale codes to pretty names, skip any unknown locales."""
     languages = {
         key.lower(): value['native']
-        for key, value in LANGUAGE_MAPPING.items()}
+        for key, value in ALL_LANGUAGES.items()}
 
     for row in series:
         if 'data' in row: