Bug 1077381 - Encode generate_browsersearch verbose output as utf-8. r=gps

We need codecs for the print function and unicode_literals to let the
format() string stay undecorated.

python/moz.build

@@ -23,6 +23,7 @@ PYTHON_UNIT_TESTS += [
     'mozbuild/dumbmake/test/test_dumbmake.py',
     'mozbuild/mozbuild/test/__init__.py',
     'mozbuild/mozbuild/test/action/test_buildlist.py',
+    'mozbuild/mozbuild/test/action/test_generate_browsersearch.py',
     'mozbuild/mozbuild/test/backend/__init__.py',
     'mozbuild/mozbuild/test/backend/common.py',
     'mozbuild/mozbuild/test/backend/test_android_eclipse.py',

python/mozbuild/mozbuild/action/generate_browsersearch.py

@@ -22,9 +22,13 @@ browsersearch.json in the locale-specific raw resource directory
 e.g. raw/browsersearch.json, raw-pt-rBR/browsersearch.json.
 '''
 
-from __future__ import print_function
+from __future__ import (
+    print_function,
+    unicode_literals,
+)
 
 import argparse
+import codecs
 import json
 import re
 import sys
@@ -72,8 +76,9 @@ def main(args):
     engines = properties.get_list('browser.search.order')
 
     if opts.verbose:
-        print('Read {len} engines: {engines}'.format(len=len(engines), engines=engines))
-        print("Default engine is '{default}'.".format(default=default))
+        writer = codecs.getwriter('utf-8')(sys.stdout)
+        print('Read {len} engines: {engines}'.format(len=len(engines), engines=engines), file=writer)
+        print("Default engine is '{default}'.".format(default=default), file=writer)
 
     browsersearch = {}
     browsersearch['default'] = default

python/mozbuild/mozbuild/test/action/data/invalid/region.properties

@@ -0,0 +1,12 @@
+# A region.properties file with invalid unicode byte sequences.  The
+# sequences were cribbed from Markus Kuhn's "UTF-8 decoder capability
+# and stress test", available at
+# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
+
+# 3.5  Impossible bytes                                                         |
+#                                                                               |
+# The following two bytes cannot appear in a correct UTF-8 string               |
+#                                                                               |
+# 3.5.1  fe = "þ"                                                               |
+# 3.5.2  ff = "ÿ"                                                               |
+# 3.5.3  fe fe ff ff = "þþÿÿ"                                                   |

python/mozbuild/mozbuild/test/action/data/valid-zh-CN/region.properties

@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Default search engine
+browser.search.defaultenginename=百度
+
+# Search engine order (order displayed in the search bar dropdown)s
+browser.search.order.1=百度
+browser.search.order.2=Google
+
+# This is the default set of web based feed handlers shown in the reader
+# selection UI
+browser.contentHandlers.types.0.title=Bloglines
+browser.contentHandlers.types.0.uri=http://www.bloglines.com/login?r=/sub/%s
+
+# increment this number when anything gets changed in the list below.  This will
+# cause Firefox to re-read these prefs and inject any new handlers into the
+# profile database.  Note that "new" is defined as "has a different URL"; this
+# means that it's not possible to update the name of existing handler, so
+# don't make any spelling errors here.
+gecko.handlerService.defaultHandlersVersion=3
+
+# The default set of protocol handlers for webcal:
+gecko.handlerService.schemes.webcal.0.name=30 Boxes
+gecko.handlerService.schemes.webcal.0.uriTemplate=https://30boxes.com/external/widget?refer=ff&url=%s
+
+# The default set of protocol handlers for mailto:
+gecko.handlerService.schemes.mailto.0.name=Yahoo! 邮件
+gecko.handlerService.schemes.mailto.0.uriTemplate=https://compose.mail.yahoo.com/?To=%s
+gecko.handlerService.schemes.mailto.1.name=Gmail
+gecko.handlerService.schemes.mailto.1.uriTemplate=https://mail.google.com/mail/?extsrc=mailto&url=%s
+
+# This is the default set of web based feed handlers shown in the reader
+# selection UI
+browser.contentHandlers.types.0.title=My Yahoo!
+browser.contentHandlers.types.0.uri=http://www.bloglines.com/login?r=/sub/%s

python/mozbuild/mozbuild/test/action/test_generate_browsersearch.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+from __future__ import unicode_literals
+
+import json
+import os
+import unittest
+
+import mozunit
+
+import mozbuild.action.generate_browsersearch as generate_browsersearch
+
+from mozfile.mozfile import (
+    NamedTemporaryFile,
+    TemporaryDirectory,
+)
+
+import mozpack.path as mozpath
+
+
+test_data_path = mozpath.abspath(mozpath.dirname(__file__))
+test_data_path = mozpath.join(test_data_path, 'data')
+
+
+class TestGenerateBrowserSearch(unittest.TestCase):
+    """
+    Unit tests for generate_browsersearch.py.
+    """
+
+    def _test_one(self, name):
+        with TemporaryDirectory() as tmpdir:
+            with NamedTemporaryFile(mode='rw') as temp:
+                srcdir = os.path.join(test_data_path, name)
+
+                generate_browsersearch.main([
+                    '--verbose',
+                    '--srcdir', srcdir,
+                    temp.name])
+                return json.load(temp)
+
+    def test_valid_unicode(self):
+        o = self._test_one('valid-zh-CN')
+        self.assertEquals(o['default'], '百度')
+        self.assertEquals(o['engines'], ['百度', 'Google'])
+
+    def test_invalid_unicode(self):
+        with self.assertRaises(UnicodeDecodeError):
+            self._test_one('invalid')
+
+
+if __name__ == '__main__':
+    mozunit.main()