Add support for downloading Firefox language packs (#506)

bin/make-langpacks.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+"""
+This script builds up-to-date versions of Firefox Developer Edition (Aurora)
+language packs from mozilla source code and localizations and makes them
+available in a public repository.
+
+A `langpacks` folder is required on the same directory level as this script,
+with the following contents:
+* `mozilla-aurora`: SOURCE_REPOSITORY checkout
+* `mozilla-aurora/.mozconfig`: file set to reflect this folder structure, see
+  https://developer.mozilla.org/en-US/docs/Mozilla/Creating_a_language_pack#Pre-build_steps
+* `mozilla-aurora/langpacks/l10n`: contains L10N_REPOSITORIES checkouts
+* `mozilla-aurora/langpacks/build`: empty folder to contain build
+* `pontoon-langpacks`: TARGET_REPOSITORY checkout (you need write access)
+
+Required libraries:
+* lxml
+* requests
+"""
+
+
+from __future__ import print_function
+from lxml import html
+
+import datetime
+import os
+import requests
+import shutil
+import subprocess
+
+
+SOURCE_REPOSITORY = 'ssh://hg.mozilla.org/releases/mozilla-aurora/'
+TARGET_REPOSITORY = 'git@github.com:mathjazz/pontoon-langpacks.git'
+
+L10N_REPOSITORIES = [{
+    'url': 'ssh://hg.mozilla.org/releases/l10n/mozilla-aurora/{locale_code}/',
+    'locales_url': 'https://hg.mozilla.org/releases/l10n/mozilla-aurora/'
+}, {
+    'url': 'ssh://hg@bitbucket.org/mozilla-l10n/{locale_code}/',
+    'locales_url': 'https://api.bitbucket.org/2.0/repositories/mozilla-l10n'
+}]
+
+
+def write(text):
+    timestamp = datetime.datetime.now().strftime('[%Y-%m-%d %H:%M:%S] ')
+    print(timestamp + text)
+
+
+def execute(command, cwd=None):
+    try:
+        st = subprocess.PIPE
+        proc = subprocess.Popen(
+            args=command, stdout=st, stderr=st, stdin=st, cwd=cwd)
+
+        (output, error) = proc.communicate()
+        code = proc.returncode
+        return code, output, error
+
+    except OSError as error:
+        return -1, '', error
+
+
+def pull(url, target):
+    # Undo local changes
+    execute(['hg', 'revert', '--all', '--no-backup'], target)
+
+    # Pull
+    code, output, error = execute(['hg', 'pull'], target)
+    code, output, error = execute(['hg', 'update', '-c'], target)
+    if code == 0:
+        write('Repository at ' + url + ' updated.')
+
+    # Clone
+    else:
+        write(unicode(error))
+        write('Clone instead.')
+
+        code, output, error = execute(['hg', 'clone', url, target])
+        if code == 0:
+            write('Repository at ' + url + ' cloned.')
+        else:
+            write(unicode(error))
+
+
+write('Building langpacks initiated.')
+
+# Change working directory to the langpacks folder
+abspath = os.path.abspath(__file__)
+dname = os.path.dirname(abspath)
+os.chdir(os.path.join(dname, 'langpacks'))
+
+# Update source repository
+target = 'mozilla-aurora'
+pull(SOURCE_REPOSITORY, target)
+
+# Configure
+write('Running ./mach configure. Please wait...')
+execute(['sh', 'mach', 'configure'], 'mozilla-aurora')
+write('Configuration complete.')
+
+for repository in L10N_REPOSITORIES:
+    # Update locale list
+    write('Updating locale lists for {url}...'.format(url=repository['url']))
+    r = requests.get(repository['locales_url'], verify=False)
+    try:
+        repository['locales'] = [repo['slug'] for repo in r.json()['values']]
+    except ValueError:
+        tree = html.fromstring(r.content)
+        repository['locales'] = tree.xpath('//b/text()')
+    write('Repository locale lists updated.')
+
+    for locale in repository['locales']:
+        # Update locale repositories
+        url = repository['url'].format(locale_code=locale)
+        target = os.path.join('mozilla-aurora/langpacks/l10n', locale)
+        pull(url, target)
+
+        # Build locale langpacks
+        write('Building langpack for {locale}...'.format(locale=locale))
+        target = 'mozilla-aurora/langpacks/build/browser/locales/'
+        execute(['make', 'merge-' + locale, 'LOCALE_MERGEDIR=$(pwd)/mergedir'], target)
+        execute(['make', 'langpack-' + locale, 'LOCALE_MERGEDIR=$(pwd)/mergedir'], target)
+        write('Langpack for {locale} built!'.format(locale=locale))
+
+write('Pushing changes to target repository...')
+
+# Reset target repository to remote state
+execute(["git", "fetch", "--all"], 'pontoon-langpacks')
+code, output, error = execute(["git", "reset", "--hard", "origin"], 'pontoon-langpacks')
+if code != 0:
+    execute(["git", "clone", TARGET_REPOSITORY, 'pontoon-langpacks'])
+
+# Move langpack to target repository
+source = 'mozilla-aurora/langpacks/build/dist/linux-x86_64/xpi'
+for filename in os.listdir(source):
+    shutil.copyfile(
+        os.path.join(source, filename),
+        os.path.join('pontoon-langpacks', filename.split('.')[2] + '.xpi')
+    )
+
+# Push target repository
+git_cmd = ['git', '-c', 'user.name=Mozilla Pontoon', '-c', 'user.email=pontoon@mozilla.com']
+execute(git_cmd + ['add', '-A', '--'], 'pontoon-langpacks')
+execute(git_cmd + ['commit', '-m', 'Update language packs'], 'pontoon-langpacks')
+code, output, error = execute(["git", "push"], 'pontoon-langpacks')
+
+if 'Everything up-to-date' in error:
+    write('Nothing to commit.')
+else:
+    write('Changes pushed.')
+
+write('Building langpacks complete.')

pontoon/base/admin.py

@@ -63,7 +63,7 @@ class ProjectAdmin(admin.ModelAdmin):
 
     fieldsets = (
         (None, {
-            'fields': ('name', 'slug', 'info_brief', 'disabled',),
+            'fields': ('name', 'slug', 'info_brief', 'langpack_url', 'disabled',),
         }),
         ('WEBSITE', {
             'fields': ('url', 'width', 'links'),

pontoon/base/migrations/0071_auto_20161123_2349.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.10 on 2016-11-23 23:49
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0070_auto_20161110_1336'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='langpack_url',
+            field=models.URLField(blank=True, help_text=b'\n        URL pattern for downloading language packs. Leave empty if language packs\n        not available for the project. Supports {locale_code} wildcard.\n    ', null=True, verbose_name=b'Language pack URL'),
+        ),
+        migrations.AlterField(
+            model_name='resource',
+            name='format',
+            field=models.CharField(blank=True, choices=[(b'po', b'po'), (b'xliff', b'xliff'), (b'xlf', b'xliff'), (b'properties', b'properties'), (b'dtd', b'dtd'), (b'inc', b'inc'), (b'ini', b'ini'), (b'lang', b'lang'), (b'ftl', b'ftl')], max_length=20, verbose_name=b'Format'),
+        ),
+    ]

pontoon/base/models.py

@@ -588,6 +588,11 @@ class Project(AggregatedStats):
     links = models.BooleanField(
         'Keep links on the project website clickable', default=False)
 
+    langpack_url = models.URLField('Language pack URL', blank=True, null=True, help_text="""
+        URL pattern for downloading language packs. Leave empty if language packs
+        not available for the project. Supports {locale_code} wildcard.
+    """)
+
     # Disable project instead of deleting to keep translation memory & attributions
     disabled = models.BooleanField(default=False)
 
@@ -619,6 +624,7 @@ class Project(AggregatedStats):
             'url': self.url,
             'width': self.width or '',
             'links': self.links or '',
+            'langpack_url': self.langpack_url or '',
         }
 
     def save(self, *args, **kwargs):
@@ -900,6 +906,7 @@ class Repository(models.Model):
         select any localized file on GitHub, click Raw and replace locale code
         and the following bits in the URL with `{locale_code}`.
     """)
+
     """
     Mapping of locale codes to VCS revisions of each repo at the last
     sync. If this isn't a multi-locale repo, the mapping has a single
@@ -1023,15 +1030,10 @@ class Repository(models.Model):
                 checkout_path = self.locale_checkout_path(locale)
 
                 try:
-                    update_from_vcs(
-                        repo_type,
-                        url,
-                        checkout_path
-                    )
+                    update_from_vcs(repo_type, url, checkout_path)
                     current_revisions[locale.code] = get_revision(repo_type, checkout_path)
                 except PullFromRepositoryException as e:
                     log.error('%s Pull Error for %s: %s' % (repo_type.upper(), url, e))
-                    pass
 
             return current_revisions
 

pontoon/base/static/js/translate.js

@@ -2745,6 +2745,9 @@ var Pontoon = (function (my) {
     updateProfileMenu: function () {
       $('#profile .admin-current-project a').attr('href', '/admin/projects/' + this.project.slug + '/');
       $('#profile .upload').toggle(this.state.paths && this.user.isTranslator && this.part !== 'all-resources');
+      $('#profile .langpack')
+        .toggle(this.project.langpack_url !== '')
+        .find('a').attr('href', this.project.langpack_url.replace('{locale_code}', this.locale.code));
     },
 
 
@@ -3205,7 +3208,8 @@ var Pontoon = (function (my) {
         slug: self.getProjectData('slug'),
         info: self.getProjectData('info'),
         width: self.getProjectWidth(),
-        links: self.getProjectData('links') === 'True' ? true : false
+        links: self.getProjectData('links') === 'True' ? true : false,
+        langpack_url: self.getProjectData('langpack_url')
       };
 
       this.part = this.getSelectedPart();

pontoon/base/templates/translate.html

@@ -108,6 +108,7 @@
             <li class="horizontal-separator"></li>
             {% endif %}
 
+            <li class="langpack"><a href=""><i class="fa fa-cloud-download fa-fw"></i>Download Language Pack</a></li>
             <li class="download"><i class="fa fa-cloud-download fa-fw"></i>Download Translations</i></li>
             <li class="upload"><i class="fa fa-cloud-upload fa-fw"></i>Upload Translations</li>