Bug 1471648 - [mozlog] Add support for Python 3; r=raphael

Differential Revision: https://phabricator.services.mozilla.com/D18069

--HG--
extra : moz-landing-system : lando

testing/mozbase/mozlog/mozlog/formatters/html/html.py

@@ -161,8 +161,11 @@ class HTMLFormatter(base.BaseFormatter):
                     # Encode base64 to avoid that some browsers (such as Firefox, Opera)
                     # treats '#' as the start of another link if it is contained in the data URL.
                     # Use 'charset=utf-8' to show special characters like Chinese.
-                    utf_encoded = six.text_type(content).encode('utf-8', 'xmlcharrefreplace')
+                    utf8_encoded_bytes = six.text_type(content).encode('utf-8',
-                    href = 'data:text/html;charset=utf-8;base64,%s' % base64.b64encode(utf_encoded)
+                                                                       'xmlcharrefreplace')
+                    b64_encoded_bytes = base64.b64encode(utf8_encoded_bytes)
+                    b64_encoded_str = b64_encoded_bytes.decode()
+                    href = "data:text/html;charset=utf-8;base64,{0}".format(b64_encoded_str)
 
                 links_html.append(html.a(
                     name.title(),

testing/mozbase/mozlog/mozlog/formatters/machformatter.py

@@ -122,7 +122,7 @@ class MachFormatter(base.BaseFormatter):
         # Format check counts
         checks = self.summary.aggregate('count', count)
         rv.append("Ran {} checks ({})".format(sum(checks.values()),
-                  ', '.join(['{} {}s'.format(v, k) for k, v in checks.items() if v])))
+                  ', '.join(['{} {}s'.format(v, k) for k, v in sorted(checks.items()) if v])))
 
         # Format expected counts
         checks = self.summary.aggregate('expected', count, include_skip=False)
@@ -146,7 +146,7 @@ class MachFormatter(base.BaseFormatter):
                 if not count[key]['unexpected']:
                     continue
                 status_str = ", ".join(["{} {}".format(n, s)
-                                        for s, n in count[key]['unexpected'].items()])
+                                        for s, n in sorted(count[key]['unexpected'].items())])
                 rv.append("  {}: {} ({})".format(
                           key, sum(count[key]['unexpected'].values()), status_str))
 

testing/mozbase/mozlog/mozlog/handlers/base.py

@@ -78,14 +78,15 @@ class StreamHandler(BaseHandler):
     def __init__(self, stream, formatter):
         BaseHandler.__init__(self, formatter)
         assert stream is not None
-        # This is a hack to deal with the case where we are passed a
+        if six.PY2:
-        # StreamWriter (e.g. by mach for stdout). A StreamWriter requires
+            # This is a hack to deal with the case where we are passed a
-        # the code to handle unicode in exactly the opposite way compared
+            # StreamWriter (e.g. by mach for stdout). A StreamWriter requires
-        # to a normal stream i.e. you always have to pass in a Unicode
+            # the code to handle unicode in exactly the opposite way compared
-        # object rather than a string object. Cope with that by extracting
+            # to a normal stream i.e. you always have to pass in a Unicode
-        # the underlying raw stream.
+            # object rather than a string object. Cope with that by extracting
-        if isinstance(stream, codecs.StreamWriter):
+            # the underlying raw stream.
-            stream = stream.stream
+            if isinstance(stream, codecs.StreamWriter):
+                stream = stream.stream
 
         self.formatter = formatter
         self.stream = stream
@@ -98,11 +99,23 @@ class StreamHandler(BaseHandler):
         if not formatted:
             return
         with self._lock:
-            if isinstance(formatted, six.text_type):
+            if six.PY3:
-                self.stream.write(formatted.encode("utf-8", "replace"))
+                import io
-            elif isinstance(formatted, str):
+                import mozfile
+                if isinstance(self.stream, io.StringIO) and isinstance(formatted, bytes):
+                    formatted = formatted.decode()
+                elif (
+                     isinstance(self.stream, io.BytesIO)
+                     or isinstance(self.stream, mozfile.NamedTemporaryFile)
+                     ) and isinstance(formatted, str):
+                    formatted = formatted.encode()
                 self.stream.write(formatted)
             else:
-                assert False, "Got output from the formatter of an unexpected type"
+                if isinstance(formatted, six.text_type):
+                    self.stream.write(formatted.encode("utf-8", "replace"))
+                elif isinstance(formatted, str):
+                    self.stream.write(formatted)
+                else:
+                    assert False, "Got output from the formatter of an unexpected type"
 
             self.stream.flush()

testing/mozbase/mozlog/mozlog/unstructured/logger.py

@@ -150,7 +150,7 @@ class MozFormatter(Formatter):
         # this protected member is used to define the format
         # used by the base Formatter's method
         self._fmt = fmt
-        return Formatter.format(self, record)
+        return Formatter(fmt=fmt).format(record)
 
 
 def getLogger(name, handler=None):

testing/mozbase/mozlog/mozlog/unstructured/loglistener.py

@@ -32,7 +32,7 @@ class LogMessageHandler(socketserver.BaseRequestHandler):
                 data = self.request.recv(1024)
                 if not data:
                     return
-                self.process_message(data)
+                self.process_message(data.decode())
             except socket.timeout:
                 return
 

testing/mozbase/mozlog/setup.cfg

@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1

testing/mozbase/mozlog/setup.py

@@ -7,7 +7,7 @@ from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 PACKAGE_NAME = 'mozlog'
-PACKAGE_VERSION = '3.10'
+PACKAGE_VERSION = '4.0'
 DEPS = [
     'blessings>=1.3',
     'mozterm',
@@ -34,7 +34,7 @@ setup(name=PACKAGE_NAME,
                    'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)',
                    'Operating System :: OS Independent',
                    'Programming Language :: Python :: 2.7',
-                   'Programming Language :: Python :: 3',
+                   'Programming Language :: Python :: 3.5',
                    'Topic :: Software Development :: Libraries :: Python Modules'],
       package_data={"mozlog": ["formatters/html/main.js",
                                "formatters/html/style.css"]},

testing/mozbase/mozlog/tests/manifest.ini

@@ -1,9 +1,6 @@
 [DEFAULT]
 subsuite = mozbase
 [test_logger.py]
-skip-if = python == 3
 [test_logtypes.py]
 [test_formatters.py]
-skip-if = python == 3
 [test_structured.py]
-skip-if = python == 3

testing/mozbase/mozlog/tests/test_formatters.py

@@ -6,7 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import mozunit
 import pytest
-from io import BytesIO
+from six import BytesIO
 
 from mozlog.structuredlog import StructuredLogger
 from mozlog.formatters import (
@@ -21,7 +21,7 @@ formatters = {
 FORMATS = {
     # A list of tuples consisting of (name, options, expected string).
     'PASS': [
-        ('mach', {}, """
+        ('mach', {}, b"""
  0:00.00 SUITE_START: running 3 tests
  0:00.00 TEST_START: test_foo
  0:00.00 TEST_END: OK
@@ -33,11 +33,11 @@ FORMATS = {
 
 suite 1
 ~~~~~~~
-Ran 4 checks (3 tests, 1 subtests)
+Ran 4 checks (1 subtests, 3 tests)
 Expected results: 4
 OK
-""".lstrip('\n')),
+""".lstrip(b'\n')),
-        ('mach', {'verbose': True}, """
+        ('mach', {'verbose': True}, b"""
  0:00.00 SUITE_START: running 3 tests
  0:00.00 TEST_START: test_foo
  0:00.00 TEST_END: OK
@@ -50,14 +50,14 @@ OK
 
 suite 1
 ~~~~~~~
-Ran 4 checks (3 tests, 1 subtests)
+Ran 4 checks (1 subtests, 3 tests)
 Expected results: 4
 OK
-""".lstrip('\n')),
+""".lstrip(b'\n')),
     ],
 
     'FAIL': [
-        ('mach', {}, """
+        ('mach', {}, b"""
  0:00.00 SUITE_START: running 3 tests
  0:00.00 TEST_START: test_foo
  0:00.00 TEST_END: FAIL, expected PASS - expected 0 got 1
@@ -73,7 +73,7 @@ TIMEOUT another subtest
 
 suite 1
 ~~~~~~~
-Ran 5 checks (3 tests, 2 subtests)
+Ran 5 checks (2 subtests, 3 tests)
 Expected results: 1
 Unexpected results: 4
   test: 2 (1 fail, 1 pass)
@@ -90,8 +90,8 @@ test_bar
   TIMEOUT another subtest
 test_baz
   UNEXPECTED-PASS test_baz
-""".lstrip('\n')),
+""".lstrip(b'\n')),
-        ('mach', {'verbose': True}, """
+        ('mach', {'verbose': True}, b"""
  0:00.00 SUITE_START: running 3 tests
  0:00.00 TEST_START: test_foo
  0:00.00 TEST_END: FAIL, expected PASS - expected 0 got 1
@@ -107,7 +107,7 @@ test_baz
 
 suite 1
 ~~~~~~~
-Ran 5 checks (3 tests, 2 subtests)
+Ran 5 checks (2 subtests, 3 tests)
 Expected results: 1
 Unexpected results: 4
   test: 2 (1 fail, 1 pass)
@@ -124,7 +124,7 @@ test_bar
   TIMEOUT another subtest
 test_baz
   UNEXPECTED-PASS test_baz
-""".lstrip('\n')),
+""".lstrip(b'\n')),
     ],
 }
 

testing/mozbase/mozlog/tests/test_logger.py

@@ -210,15 +210,15 @@ class TestStructuredLogging(unittest.TestCase):
 
         # Sleeps prevent listener from receiving entire message in a single call
         # to recv in order to test reconstruction of partial messages.
-        sock.sendall(message_string[:8])
+        sock.sendall(message_string[:8].encode())
         time.sleep(.01)
-        sock.sendall(message_string[8:32])
+        sock.sendall(message_string[8:32].encode())
         time.sleep(.01)
-        sock.sendall(message_string[32:64])
+        sock.sendall(message_string[32:64].encode())
         time.sleep(.01)
-        sock.sendall(message_string[64:128])
+        sock.sendall(message_string[64:128].encode())
         time.sleep(.01)
-        sock.sendall(message_string[128:])
+        sock.sendall(message_string[128:].encode())
 
         server_thread.join()
 

testing/mozbase/mozlog/tests/test_structured.py

@@ -488,8 +488,14 @@ class TestTypeConversions(BaseStructuredTest):
 
     def test_tuple(self):
         self.logger.suite_start([])
-        self.logger.test_start(("\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90\x8d\x83\xf0\x90\x8d\x84",
+        if six.PY3:
-                                42, u"\u16a4"))
+            self.logger.test_start((b"\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90"
+                                    b"\x8d\x83\xf0\x90\x8d\x84".decode(),
+                                    42, u"\u16a4"))
+        else:
+            self.logger.test_start(("\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90"
+                                    "\x8d\x83\xf0\x90\x8d\x84",
+                                    42, u"\u16a4"))
         self.assert_log_equals({"action": "test_start",
                                 "test": (u'\U00010344\U00010334\U00010343\U00010344',
                                          u"42", u"\u16a4")})
@@ -502,9 +508,15 @@ class TestTypeConversions(BaseStructuredTest):
                                 "message": "1",
                                 "level": "INFO"})
         self.logger.info([1, (2, '3'), "s", "s" + chr(255)])
-        self.assert_log_equals({"action": "log",
+        if six.PY3:
-                                "message": "[1, (2, '3'), 's', 's\\xff']",
+            self.assert_log_equals({"action": "log",
-                                "level": "INFO"})
+                                    "message": "[1, (2, '3'), 's', 's\xff']",
+                                    "level": "INFO"})
+        else:
+            self.assert_log_equals({"action": "log",
+                                    "message": "[1, (2, '3'), 's', 's\\xff']",
+                                    "level": "INFO"})
+
         self.logger.suite_end()
 
     def test_utf8str_write(self):
@@ -516,7 +528,10 @@ class TestTypeConversions(BaseStructuredTest):
             self.logger.info("☺")
             logfile.seek(0)
             data = logfile.readlines()[-1].strip()
-            self.assertEquals(data, "☺")
+            if six.PY3:
+                self.assertEquals(data.decode(), "☺")
+            else:
+                self.assertEquals(data, "☺")
             self.logger.suite_end()
             self.logger.remove_handler(_handler)
 
@@ -799,7 +814,7 @@ Unexpected results: 2
         self.set_position()
         self.logger.suite_end()
 
-        self.assertIn("Ran 5 checks (2 tests, 3 subtests)", self.loglines)
+        self.assertIn("Ran 5 checks (3 subtests, 2 tests)", self.loglines)
         self.assertIn("Expected results: 2", self.loglines)
         self.assertIn("""
 Unexpected results: 3
@@ -962,8 +977,8 @@ class TestCommandline(unittest.TestCase):
         logger.debug("DEBUG message")
         logger.error("ERROR message")
         # The debug level is not logged by default.
-        self.assertEqual(["INFO message",
+        self.assertEqual([b"INFO message",
-                          "ERROR message"],
+                          b"ERROR message"],
                          self.loglines)
 
     def test_logging_errorlevel(self):
@@ -977,7 +992,7 @@ class TestCommandline(unittest.TestCase):
         logger.error("ERROR message")
 
         # Only the error level and above were requested.
-        self.assertEqual(["ERROR message"],
+        self.assertEqual([b"ERROR message"],
                          self.loglines)
 
     def test_logging_debuglevel(self):
@@ -990,9 +1005,9 @@ class TestCommandline(unittest.TestCase):
         logger.debug("DEBUG message")
         logger.error("ERROR message")
         # Requesting a lower log level than default works as expected.
-        self.assertEqual(["INFO message",
+        self.assertEqual([b"INFO message",
-                          "DEBUG message",
+                          b"DEBUG message",
-                          "ERROR message"],
+                          b"ERROR message"],
                          self.loglines)
 
     def test_unused_options(self):