From 3631b957960a482df51f3002dffddaf79c84a82c Mon Sep 17 00:00:00 2001 From: Pavel Slepushkin Date: Thu, 31 Jan 2019 12:49:42 +0000 Subject: [PATCH] Bug 1471648 - [mozlog] Add support for Python 3; r=raphael Differential Revision: https://phabricator.services.mozilla.com/D18069 --HG-- extra : moz-landing-system : lando --- .../mozlog/mozlog/formatters/html/html.py | 7 +++- .../mozlog/mozlog/formatters/machformatter.py | 4 +- .../mozbase/mozlog/mozlog/handlers/base.py | 37 +++++++++++------ .../mozlog/mozlog/unstructured/logger.py | 2 +- .../mozlog/mozlog/unstructured/loglistener.py | 2 +- testing/mozbase/mozlog/setup.cfg | 2 + testing/mozbase/mozlog/setup.py | 4 +- testing/mozbase/mozlog/tests/manifest.ini | 3 -- .../mozbase/mozlog/tests/test_formatters.py | 26 ++++++------ testing/mozbase/mozlog/tests/test_logger.py | 10 ++--- .../mozbase/mozlog/tests/test_structured.py | 41 +++++++++++++------ 11 files changed, 84 insertions(+), 54 deletions(-) create mode 100644 testing/mozbase/mozlog/setup.cfg diff --git a/testing/mozbase/mozlog/mozlog/formatters/html/html.py b/testing/mozbase/mozlog/mozlog/formatters/html/html.py index f6873a546eab..b172d26540ca 100755 --- a/testing/mozbase/mozlog/mozlog/formatters/html/html.py +++ b/testing/mozbase/mozlog/mozlog/formatters/html/html.py @@ -161,8 +161,11 @@ class HTMLFormatter(base.BaseFormatter): # Encode base64 to avoid that some browsers (such as Firefox, Opera) # treats '#' as the start of another link if it is contained in the data URL. # Use 'charset=utf-8' to show special characters like Chinese. - utf_encoded = six.text_type(content).encode('utf-8', 'xmlcharrefreplace') - href = 'data:text/html;charset=utf-8;base64,%s' % base64.b64encode(utf_encoded) + utf8_encoded_bytes = six.text_type(content).encode('utf-8', + 'xmlcharrefreplace') + b64_encoded_bytes = base64.b64encode(utf8_encoded_bytes) + b64_encoded_str = b64_encoded_bytes.decode() + href = "data:text/html;charset=utf-8;base64,{0}".format(b64_encoded_str) links_html.append(html.a( name.title(), diff --git a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py index ae8decb949f6..8019552841dd 100644 --- a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py +++ b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py @@ -122,7 +122,7 @@ class MachFormatter(base.BaseFormatter): # Format check counts checks = self.summary.aggregate('count', count) rv.append("Ran {} checks ({})".format(sum(checks.values()), - ', '.join(['{} {}s'.format(v, k) for k, v in checks.items() if v]))) + ', '.join(['{} {}s'.format(v, k) for k, v in sorted(checks.items()) if v]))) # Format expected counts checks = self.summary.aggregate('expected', count, include_skip=False) @@ -146,7 +146,7 @@ class MachFormatter(base.BaseFormatter): if not count[key]['unexpected']: continue status_str = ", ".join(["{} {}".format(n, s) - for s, n in count[key]['unexpected'].items()]) + for s, n in sorted(count[key]['unexpected'].items())]) rv.append(" {}: {} ({})".format( key, sum(count[key]['unexpected'].values()), status_str)) diff --git a/testing/mozbase/mozlog/mozlog/handlers/base.py b/testing/mozbase/mozlog/mozlog/handlers/base.py index 372fe591df9b..654a07382cda 100644 --- a/testing/mozbase/mozlog/mozlog/handlers/base.py +++ b/testing/mozbase/mozlog/mozlog/handlers/base.py @@ -78,14 +78,15 @@ class StreamHandler(BaseHandler): def __init__(self, stream, formatter): BaseHandler.__init__(self, formatter) assert stream is not None - # This is a hack to deal with the case where we are passed a - # StreamWriter (e.g. by mach for stdout). A StreamWriter requires - # the code to handle unicode in exactly the opposite way compared - # to a normal stream i.e. you always have to pass in a Unicode - # object rather than a string object. Cope with that by extracting - # the underlying raw stream. - if isinstance(stream, codecs.StreamWriter): - stream = stream.stream + if six.PY2: + # This is a hack to deal with the case where we are passed a + # StreamWriter (e.g. by mach for stdout). A StreamWriter requires + # the code to handle unicode in exactly the opposite way compared + # to a normal stream i.e. you always have to pass in a Unicode + # object rather than a string object. Cope with that by extracting + # the underlying raw stream. + if isinstance(stream, codecs.StreamWriter): + stream = stream.stream self.formatter = formatter self.stream = stream @@ -98,11 +99,23 @@ class StreamHandler(BaseHandler): if not formatted: return with self._lock: - if isinstance(formatted, six.text_type): - self.stream.write(formatted.encode("utf-8", "replace")) - elif isinstance(formatted, str): + if six.PY3: + import io + import mozfile + if isinstance(self.stream, io.StringIO) and isinstance(formatted, bytes): + formatted = formatted.decode() + elif ( + isinstance(self.stream, io.BytesIO) + or isinstance(self.stream, mozfile.NamedTemporaryFile) + ) and isinstance(formatted, str): + formatted = formatted.encode() self.stream.write(formatted) else: - assert False, "Got output from the formatter of an unexpected type" + if isinstance(formatted, six.text_type): + self.stream.write(formatted.encode("utf-8", "replace")) + elif isinstance(formatted, str): + self.stream.write(formatted) + else: + assert False, "Got output from the formatter of an unexpected type" self.stream.flush() diff --git a/testing/mozbase/mozlog/mozlog/unstructured/logger.py b/testing/mozbase/mozlog/mozlog/unstructured/logger.py index 19703369fc8e..c0e307670dd6 100644 --- a/testing/mozbase/mozlog/mozlog/unstructured/logger.py +++ b/testing/mozbase/mozlog/mozlog/unstructured/logger.py @@ -150,7 +150,7 @@ class MozFormatter(Formatter): # this protected member is used to define the format # used by the base Formatter's method self._fmt = fmt - return Formatter.format(self, record) + return Formatter(fmt=fmt).format(record) def getLogger(name, handler=None): diff --git a/testing/mozbase/mozlog/mozlog/unstructured/loglistener.py b/testing/mozbase/mozlog/mozlog/unstructured/loglistener.py index abc0da505d79..93ec19c90ff0 100644 --- a/testing/mozbase/mozlog/mozlog/unstructured/loglistener.py +++ b/testing/mozbase/mozlog/mozlog/unstructured/loglistener.py @@ -32,7 +32,7 @@ class LogMessageHandler(socketserver.BaseRequestHandler): data = self.request.recv(1024) if not data: return - self.process_message(data) + self.process_message(data.decode()) except socket.timeout: return diff --git a/testing/mozbase/mozlog/setup.cfg b/testing/mozbase/mozlog/setup.cfg new file mode 100644 index 000000000000..3c6e79cf31da --- /dev/null +++ b/testing/mozbase/mozlog/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/testing/mozbase/mozlog/setup.py b/testing/mozbase/mozlog/setup.py index ca38194feeea..91a03930e0d7 100644 --- a/testing/mozbase/mozlog/setup.py +++ b/testing/mozbase/mozlog/setup.py @@ -7,7 +7,7 @@ from __future__ import absolute_import from setuptools import setup, find_packages PACKAGE_NAME = 'mozlog' -PACKAGE_VERSION = '3.10' +PACKAGE_VERSION = '4.0' DEPS = [ 'blessings>=1.3', 'mozterm', @@ -34,7 +34,7 @@ setup(name=PACKAGE_NAME, 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)', 'Operating System :: OS Independent', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', 'Topic :: Software Development :: Libraries :: Python Modules'], package_data={"mozlog": ["formatters/html/main.js", "formatters/html/style.css"]}, diff --git a/testing/mozbase/mozlog/tests/manifest.ini b/testing/mozbase/mozlog/tests/manifest.ini index 55b41eb518fa..ebbf5ca55bd0 100644 --- a/testing/mozbase/mozlog/tests/manifest.ini +++ b/testing/mozbase/mozlog/tests/manifest.ini @@ -1,9 +1,6 @@ [DEFAULT] subsuite = mozbase [test_logger.py] -skip-if = python == 3 [test_logtypes.py] [test_formatters.py] -skip-if = python == 3 [test_structured.py] -skip-if = python == 3 diff --git a/testing/mozbase/mozlog/tests/test_formatters.py b/testing/mozbase/mozlog/tests/test_formatters.py index a556fe2cd2d6..f70507dc4151 100644 --- a/testing/mozbase/mozlog/tests/test_formatters.py +++ b/testing/mozbase/mozlog/tests/test_formatters.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals import mozunit import pytest -from io import BytesIO +from six import BytesIO from mozlog.structuredlog import StructuredLogger from mozlog.formatters import ( @@ -21,7 +21,7 @@ formatters = { FORMATS = { # A list of tuples consisting of (name, options, expected string). 'PASS': [ - ('mach', {}, """ + ('mach', {}, b""" 0:00.00 SUITE_START: running 3 tests 0:00.00 TEST_START: test_foo 0:00.00 TEST_END: OK @@ -33,11 +33,11 @@ FORMATS = { suite 1 ~~~~~~~ -Ran 4 checks (3 tests, 1 subtests) +Ran 4 checks (1 subtests, 3 tests) Expected results: 4 OK -""".lstrip('\n')), - ('mach', {'verbose': True}, """ +""".lstrip(b'\n')), + ('mach', {'verbose': True}, b""" 0:00.00 SUITE_START: running 3 tests 0:00.00 TEST_START: test_foo 0:00.00 TEST_END: OK @@ -50,14 +50,14 @@ OK suite 1 ~~~~~~~ -Ran 4 checks (3 tests, 1 subtests) +Ran 4 checks (1 subtests, 3 tests) Expected results: 4 OK -""".lstrip('\n')), +""".lstrip(b'\n')), ], 'FAIL': [ - ('mach', {}, """ + ('mach', {}, b""" 0:00.00 SUITE_START: running 3 tests 0:00.00 TEST_START: test_foo 0:00.00 TEST_END: FAIL, expected PASS - expected 0 got 1 @@ -73,7 +73,7 @@ TIMEOUT another subtest suite 1 ~~~~~~~ -Ran 5 checks (3 tests, 2 subtests) +Ran 5 checks (2 subtests, 3 tests) Expected results: 1 Unexpected results: 4 test: 2 (1 fail, 1 pass) @@ -90,8 +90,8 @@ test_bar TIMEOUT another subtest test_baz UNEXPECTED-PASS test_baz -""".lstrip('\n')), - ('mach', {'verbose': True}, """ +""".lstrip(b'\n')), + ('mach', {'verbose': True}, b""" 0:00.00 SUITE_START: running 3 tests 0:00.00 TEST_START: test_foo 0:00.00 TEST_END: FAIL, expected PASS - expected 0 got 1 @@ -107,7 +107,7 @@ test_baz suite 1 ~~~~~~~ -Ran 5 checks (3 tests, 2 subtests) +Ran 5 checks (2 subtests, 3 tests) Expected results: 1 Unexpected results: 4 test: 2 (1 fail, 1 pass) @@ -124,7 +124,7 @@ test_bar TIMEOUT another subtest test_baz UNEXPECTED-PASS test_baz -""".lstrip('\n')), +""".lstrip(b'\n')), ], } diff --git a/testing/mozbase/mozlog/tests/test_logger.py b/testing/mozbase/mozlog/tests/test_logger.py index 247939786b67..964117761cec 100644 --- a/testing/mozbase/mozlog/tests/test_logger.py +++ b/testing/mozbase/mozlog/tests/test_logger.py @@ -210,15 +210,15 @@ class TestStructuredLogging(unittest.TestCase): # Sleeps prevent listener from receiving entire message in a single call # to recv in order to test reconstruction of partial messages. - sock.sendall(message_string[:8]) + sock.sendall(message_string[:8].encode()) time.sleep(.01) - sock.sendall(message_string[8:32]) + sock.sendall(message_string[8:32].encode()) time.sleep(.01) - sock.sendall(message_string[32:64]) + sock.sendall(message_string[32:64].encode()) time.sleep(.01) - sock.sendall(message_string[64:128]) + sock.sendall(message_string[64:128].encode()) time.sleep(.01) - sock.sendall(message_string[128:]) + sock.sendall(message_string[128:].encode()) server_thread.join() diff --git a/testing/mozbase/mozlog/tests/test_structured.py b/testing/mozbase/mozlog/tests/test_structured.py index 6e3815e24250..de7d6270f494 100644 --- a/testing/mozbase/mozlog/tests/test_structured.py +++ b/testing/mozbase/mozlog/tests/test_structured.py @@ -488,8 +488,14 @@ class TestTypeConversions(BaseStructuredTest): def test_tuple(self): self.logger.suite_start([]) - self.logger.test_start(("\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90\x8d\x83\xf0\x90\x8d\x84", - 42, u"\u16a4")) + if six.PY3: + self.logger.test_start((b"\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90" + b"\x8d\x83\xf0\x90\x8d\x84".decode(), + 42, u"\u16a4")) + else: + self.logger.test_start(("\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90" + "\x8d\x83\xf0\x90\x8d\x84", + 42, u"\u16a4")) self.assert_log_equals({"action": "test_start", "test": (u'\U00010344\U00010334\U00010343\U00010344', u"42", u"\u16a4")}) @@ -502,9 +508,15 @@ class TestTypeConversions(BaseStructuredTest): "message": "1", "level": "INFO"}) self.logger.info([1, (2, '3'), "s", "s" + chr(255)]) - self.assert_log_equals({"action": "log", - "message": "[1, (2, '3'), 's', 's\\xff']", - "level": "INFO"}) + if six.PY3: + self.assert_log_equals({"action": "log", + "message": "[1, (2, '3'), 's', 's\xff']", + "level": "INFO"}) + else: + self.assert_log_equals({"action": "log", + "message": "[1, (2, '3'), 's', 's\\xff']", + "level": "INFO"}) + self.logger.suite_end() def test_utf8str_write(self): @@ -516,7 +528,10 @@ class TestTypeConversions(BaseStructuredTest): self.logger.info("☺") logfile.seek(0) data = logfile.readlines()[-1].strip() - self.assertEquals(data, "☺") + if six.PY3: + self.assertEquals(data.decode(), "☺") + else: + self.assertEquals(data, "☺") self.logger.suite_end() self.logger.remove_handler(_handler) @@ -799,7 +814,7 @@ Unexpected results: 2 self.set_position() self.logger.suite_end() - self.assertIn("Ran 5 checks (2 tests, 3 subtests)", self.loglines) + self.assertIn("Ran 5 checks (3 subtests, 2 tests)", self.loglines) self.assertIn("Expected results: 2", self.loglines) self.assertIn(""" Unexpected results: 3 @@ -962,8 +977,8 @@ class TestCommandline(unittest.TestCase): logger.debug("DEBUG message") logger.error("ERROR message") # The debug level is not logged by default. - self.assertEqual(["INFO message", - "ERROR message"], + self.assertEqual([b"INFO message", + b"ERROR message"], self.loglines) def test_logging_errorlevel(self): @@ -977,7 +992,7 @@ class TestCommandline(unittest.TestCase): logger.error("ERROR message") # Only the error level and above were requested. - self.assertEqual(["ERROR message"], + self.assertEqual([b"ERROR message"], self.loglines) def test_logging_debuglevel(self): @@ -990,9 +1005,9 @@ class TestCommandline(unittest.TestCase): logger.debug("DEBUG message") logger.error("ERROR message") # Requesting a lower log level than default works as expected. - self.assertEqual(["INFO message", - "DEBUG message", - "ERROR message"], + self.assertEqual([b"INFO message", + b"DEBUG message", + b"ERROR message"], self.loglines) def test_unused_options(self):