From 4d7f8fd97218ebf824c75fd2dd3316293647a936 Mon Sep 17 00:00:00 2001
From: Dave Hunt <dhunt@mozilla.com>
Date: Thu, 31 May 2018 13:35:19 +0100
Subject: [PATCH] Bug 1428362 - Run modernize against mozlog to support Python
 3; r=ahal

MozReview-Commit-ID: 6BxTt4eGmfc

--HG--
extra : rebase_source : 2185715e8664288a9847dd81695fdb1640ee32eb
---
 testing/mozbase/mozlog/mozlog/commandline.py  | 19 +++++++--------
 .../mozlog/mozlog/formatters/html/html.py     |  5 ++--
 .../mozlog/mozlog/formatters/machformatter.py |  6 +++--
 .../mozlog/mozlog/formatters/process.py       |  1 +
 .../mozlog/mozlog/formatters/tbplformatter.py |  6 +++--
 .../mozbase/mozlog/mozlog/formatters/xunit.py |  5 ++--
 .../mozbase/mozlog/mozlog/handlers/base.py    |  3 ++-
 .../mozlog/mozlog/handlers/summaryhandler.py  |  3 ++-
 testing/mozbase/mozlog/mozlog/logtypes.py     | 17 ++++++++------
 .../mozlog/mozlog/pytest_mozlog/plugin.py     |  7 +++---
 .../mozbase/mozlog/mozlog/scripts/__init__.py |  3 ++-
 .../mozbase/mozlog/mozlog/scripts/format.py   |  2 +-
 .../mozbase/mozlog/mozlog/scripts/unstable.py | 23 ++++++++++---------
 .../mozbase/mozlog/mozlog/structuredlog.py    |  3 ++-
 testing/mozbase/mozlog/tests/test_logger.py   |  3 ++-
 .../mozbase/mozlog/tests/test_structured.py   |  6 +++--
 16 files changed, 66 insertions(+), 46 deletions(-)

diff --git a/testing/mozbase/mozlog/mozlog/commandline.py b/testing/mozbase/mozlog/mozlog/commandline.py
index 456692ec794c..25a40e0396df 100644
--- a/testing/mozbase/mozlog/mozlog/commandline.py
+++ b/testing/mozbase/mozlog/mozlog/commandline.py
@@ -13,6 +13,7 @@ from collections import defaultdict
 from . import handlers
 from . import formatters
 from .structuredlog import StructuredLogger, set_default_logger
+import six
 
 log_formatters = {
     'raw': (formatters.JSONFormatter, "Raw structured log messages "
@@ -150,12 +151,12 @@ def add_logging_group(parser, include_formatters=None):
         opt_log_type = log_file
         group_add = group.add_argument
 
-    for name, (cls, help_str) in log_formatters.iteritems():
+    for name, (cls, help_str) in six.iteritems(log_formatters):
         if name in include_formatters:
             group_add("--log-" + name, action="append", type=opt_log_type,
                       help=help_str)
 
-    for optname, (cls, help_str, formatters_, action) in fmt_options.iteritems():
+    for optname, (cls, help_str, formatters_, action) in six.iteritems(fmt_options):
         for fmt in formatters_:
             # make sure fmt is in log_formatters and is accepted
             if fmt in log_formatters and fmt in include_formatters:
@@ -179,12 +180,12 @@ def setup_handlers(logger, formatters, formatter_options, allow_unused_options=F
                list(unused_options))
         raise ValueError(msg)
 
-    for fmt, streams in formatters.iteritems():
+    for fmt, streams in six.iteritems(formatters):
         formatter_cls = log_formatters[fmt][0]
         formatter = formatter_cls()
         handler_wrappers_and_options = []
 
-        for option, value in formatter_options[fmt].iteritems():
+        for option, value in six.iteritems(formatter_options[fmt]):
             wrapper, wrapper_args = None, ()
             if option == "valgrind":
                 wrapper = valgrind_handler_wrapper
@@ -249,7 +250,7 @@ def setup_logging(logger, args, defaults=None, formatter_defaults=None,
         else:
             defaults = {"raw": sys.stdout}
 
-    for name, values in args.iteritems():
+    for name, values in six.iteritems(args):
         parts = name.split('_')
         if len(parts) > 3:
             continue
@@ -263,7 +264,7 @@ def setup_logging(logger, args, defaults=None, formatter_defaults=None,
                 _, formatter = parts
                 for value in values:
                     found = True
-                    if isinstance(value, basestring):
+                    if isinstance(value, six.string_types):
                         value = log_file(value)
                     if value == sys.stdout:
                         found_stdout_logger = True
@@ -277,11 +278,11 @@ def setup_logging(logger, args, defaults=None, formatter_defaults=None,
 
     # If there is no user-specified logging, go with the default options
     if not found:
-        for name, value in defaults.iteritems():
+        for name, value in six.iteritems(defaults):
             formatters[name].append(value)
 
-    elif not found_stdout_logger and sys.stdout in defaults.values():
-        for name, value in defaults.iteritems():
+    elif not found_stdout_logger and sys.stdout in list(defaults.values()):
+        for name, value in six.iteritems(defaults):
             if value == sys.stdout:
                 formatters[name].append(value)
 
diff --git a/testing/mozbase/mozlog/mozlog/formatters/html/html.py b/testing/mozbase/mozlog/mozlog/formatters/html/html.py
index aa95ad22e390..f6873a546eab 100755
--- a/testing/mozbase/mozlog/mozlog/formatters/html/html.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/html/html.py
@@ -13,6 +13,7 @@ import os
 from .. import base
 
 from collections import defaultdict
+import six
 
 html = None
 raw = None
@@ -160,7 +161,7 @@ class HTMLFormatter(base.BaseFormatter):
                     # Encode base64 to avoid that some browsers (such as Firefox, Opera)
                     # treats '#' as the start of another link if it is contained in the data URL.
                     # Use 'charset=utf-8' to show special characters like Chinese.
-                    utf_encoded = unicode(content).encode('utf-8', 'xmlcharrefreplace')
+                    utf_encoded = six.text_type(content).encode('utf-8', 'xmlcharrefreplace')
                     href = 'data:text/html;charset=utf-8;base64,%s' % base64.b64encode(utf_encoded)
 
                 links_html.append(html.a(
@@ -210,7 +211,7 @@ class HTMLFormatter(base.BaseFormatter):
                         id='environment'),
 
                     html.h2('Summary'),
-                    html.p('%i tests ran in %.1f seconds.' % (sum(self.test_count.itervalues()),
+                    html.p('%i tests ran in %.1f seconds.' % (sum(six.itervalues(self.test_count)),
                                                               (self.suite_times["end"] -
                                                                self.suite_times["start"]) / 1000.),
                            html.br(),
diff --git a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
index e23029faad42..9fe4280b4f20 100644
--- a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
@@ -12,6 +12,8 @@ from mozterm import Terminal
 from . import base
 from .process import strstatus
 from ..handlers import SummaryHandler
+import six
+from functools import reduce
 
 
 def format_seconds(total):
@@ -60,7 +62,7 @@ class MachFormatter(base.BaseFormatter):
         return test_id
 
     def _get_file_name(self, test_id):
-        if isinstance(test_id, (str, unicode)):
+        if isinstance(test_id, (str, six.text_type)):
             return test_id
 
         if isinstance(test_id, tuple):
@@ -69,7 +71,7 @@ class MachFormatter(base.BaseFormatter):
         assert False, "unexpected test_id"
 
     def suite_start(self, data):
-        num_tests = reduce(lambda x, y: x + len(y), data['tests'].itervalues(), 0)
+        num_tests = reduce(lambda x, y: x + len(y), six.itervalues(data['tests']), 0)
         action = self.term.yellow(data['action'].upper())
         name = ""
         if 'name' in data:
diff --git a/testing/mozbase/mozlog/mozlog/formatters/process.py b/testing/mozbase/mozlog/mozlog/formatters/process.py
index b3664c8da5a9..7855e74b519a 100644
--- a/testing/mozbase/mozlog/mozlog/formatters/process.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/process.py
@@ -6,6 +6,7 @@ from __future__ import absolute_import
 
 import os
 import signal
+from six.moves import range
 
 # a dict cache of signal number -> signal name
 _SIG_NAME = None
diff --git a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
index aae0ecdd5351..d80e96c2811b 100644
--- a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
@@ -10,6 +10,8 @@ from collections import deque
 from .base import BaseFormatter
 from .process import strstatus
 from ..handlers import SummaryHandler
+import six
+from functools import reduce
 
 
 def output_subtests(func):
@@ -129,7 +131,7 @@ class TbplFormatter(BaseFormatter):
 
     def suite_start(self, data):
         self.suite_start_time = data["time"]
-        num_tests = reduce(lambda x, y: x + len(y), data['tests'].itervalues(), 0)
+        num_tests = reduce(lambda x, y: x + len(y), six.itervalues(data['tests']), 0)
         return "SUITE-START | Running %i tests\n" % num_tests
 
     def test_start(self, data):
@@ -256,7 +258,7 @@ class TbplFormatter(BaseFormatter):
         return "SUITE-END | took %is\n" % time
 
     def test_id(self, test_id):
-        if isinstance(test_id, (str, unicode)):
+        if isinstance(test_id, (str, six.text_type)):
             return test_id
         else:
             return tuple(test_id)
diff --git a/testing/mozbase/mozlog/mozlog/formatters/xunit.py b/testing/mozbase/mozlog/mozlog/formatters/xunit.py
index c4f8a57a9391..b4c98d75b5b2 100644
--- a/testing/mozbase/mozlog/mozlog/formatters/xunit.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/xunit.py
@@ -1,15 +1,16 @@
 from __future__ import absolute_import
 
-import types
 from xml.etree import ElementTree
 
+import six
+
 from . import base
 
 
 def format_test_id(test_id):
     """Take a test id and return something that looks a bit like
     a class path"""
-    if type(test_id) not in types.StringTypes:
+    if not isinstance(test_id, six.string_types):
         # Not sure how to deal with reftests yet
         raise NotImplementedError
 
diff --git a/testing/mozbase/mozlog/mozlog/handlers/base.py b/testing/mozbase/mozlog/mozlog/handlers/base.py
index 1e453f08e976..372fe591df9b 100644
--- a/testing/mozbase/mozlog/mozlog/handlers/base.py
+++ b/testing/mozbase/mozlog/mozlog/handlers/base.py
@@ -8,6 +8,7 @@ from threading import Lock
 import codecs
 
 from mozlog.structuredlog import log_levels
+import six
 
 
 class BaseHandler(object):
@@ -97,7 +98,7 @@ class StreamHandler(BaseHandler):
         if not formatted:
             return
         with self._lock:
-            if isinstance(formatted, unicode):
+            if isinstance(formatted, six.text_type):
                 self.stream.write(formatted.encode("utf-8", "replace"))
             elif isinstance(formatted, str):
                 self.stream.write(formatted)
diff --git a/testing/mozbase/mozlog/mozlog/handlers/summaryhandler.py b/testing/mozbase/mozlog/mozlog/handlers/summaryhandler.py
index 9e47101a6cb4..3fa09f38aec2 100644
--- a/testing/mozbase/mozlog/mozlog/handlers/summaryhandler.py
+++ b/testing/mozbase/mozlog/mozlog/handlers/summaryhandler.py
@@ -10,6 +10,7 @@ from collections import (
 )
 
 from ..reader import LogHandler
+import six
 
 
 class SummaryHandler(LogHandler):
@@ -68,7 +69,7 @@ class SummaryHandler(LogHandler):
         Yields a tuple of (suite, summary). The summary returned is
         the same format as returned by 'get'.
         """
-        for suite, data in self.summary.iteritems():
+        for suite, data in six.iteritems(self.summary):
             yield suite, data
 
     @classmethod
diff --git a/testing/mozbase/mozlog/mozlog/logtypes.py b/testing/mozbase/mozlog/mozlog/logtypes.py
index 5ab5b38d9ca0..86453bee8b4b 100644
--- a/testing/mozbase/mozlog/mozlog/logtypes.py
+++ b/testing/mozbase/mozlog/mozlog/logtypes.py
@@ -5,6 +5,9 @@
 from __future__ import absolute_import
 
 import inspect
+import six
+from six.moves import range
+from six.moves import zip
 
 convertor_registry = {}
 missing = object()
@@ -87,7 +90,7 @@ class log_action(object):
             if name not in values:
                 values[name] = self.args[name].default
 
-        for key, value in values.iteritems():
+        for key, value in six.iteritems(values):
             if key in self.args:
                 out_value = self.args[key](value)
                 if out_value is not missing:
@@ -98,7 +101,7 @@ class log_action(object):
         return data
 
     def convert_known(self, **kwargs):
-        known_kwargs = {name: value for name, value in kwargs.iteritems()
+        known_kwargs = {name: value for name, value in six.iteritems(kwargs)
                         if name in self.args}
         return self.convert(**known_kwargs)
 
@@ -155,17 +158,17 @@ class ContainerType(DataType):
 class Unicode(DataType):
 
     def convert(self, data):
-        if isinstance(data, unicode):
+        if isinstance(data, six.text_type):
             return data
         if isinstance(data, str):
             return data.decode("utf8", "replace")
-        return unicode(data)
+        return six.text_type(data)
 
 
 class TestId(DataType):
 
     def convert(self, data):
-        if isinstance(data, unicode):
+        if isinstance(data, six.text_type):
             return data
         elif isinstance(data, bytes):
             return data.decode("utf-8", "replace")
@@ -200,7 +203,7 @@ class Dict(ContainerType):
         if isinstance(item_type, dict):
             if len(item_type) != 1:
                 raise ValueError("Dict item type specifier must contain a single entry.")
-            key_type, value_type = item_type.items()[0]
+            key_type, value_type = list(item_type.items())[0]
             return superfmt(key_type), superfmt(value_type)
         return Any(None), superfmt(item_type)
 
@@ -214,7 +217,7 @@ class List(ContainerType):
     def convert(self, data):
         # while dicts and strings _can_ be cast to lists,
         # doing so is likely not intentional behaviour
-        if isinstance(data, (basestring, dict)):
+        if isinstance(data, (six.string_types, dict)):
             raise ValueError("Expected list but got %s" % type(data))
         return [self.item_type.convert(item) for item in data]
 
diff --git a/testing/mozbase/mozlog/mozlog/pytest_mozlog/plugin.py b/testing/mozbase/mozlog/mozlog/pytest_mozlog/plugin.py
index d1f8938b98a0..ad24a490ca2d 100644
--- a/testing/mozbase/mozlog/mozlog/pytest_mozlog/plugin.py
+++ b/testing/mozbase/mozlog/mozlog/pytest_mozlog/plugin.py
@@ -8,6 +8,7 @@ import mozlog
 import time
 
 import pytest
+import six
 
 
 def pytest_addoption(parser):
@@ -15,10 +16,10 @@ def pytest_addoption(parser):
     # Pytest's parser doesn't have the add_argument_group method Mozlog expects.
     group = parser.getgroup('mozlog')
 
-    for name, (_class, _help) in mozlog.commandline.log_formatters.iteritems():
+    for name, (_class, _help) in six.iteritems(mozlog.commandline.log_formatters):
         group.addoption('--log-{0}'.format(name), action='append', help=_help)
 
-    formatter_options = mozlog.commandline.fmt_options.iteritems()
+    formatter_options = six.iteritems(mozlog.commandline.fmt_options)
     for name, (_class, _help, formatters, action) in formatter_options:
         for formatter in formatters:
             if formatter in mozlog.commandline.log_formatters:
@@ -89,7 +90,7 @@ class MozLog(object):
             status = 'SKIP' if not hasattr(report, 'wasxfail') else 'FAIL'
         if report.longrepr is not None:
             longrepr = report.longrepr
-            if isinstance(longrepr, basestring):
+            if isinstance(longrepr, six.string_types):
                 # When using pytest-xdist, longrepr is serialised as a str
                 message = stack = longrepr
                 if longrepr.startswith('[XPASS(strict)]'):
diff --git a/testing/mozbase/mozlog/mozlog/scripts/__init__.py b/testing/mozbase/mozlog/mozlog/scripts/__init__.py
index 62505afaa3c3..fcb4cf87b777 100644
--- a/testing/mozbase/mozlog/mozlog/scripts/__init__.py
+++ b/testing/mozbase/mozlog/mozlog/scripts/__init__.py
@@ -6,6 +6,7 @@ import argparse
 import unstable
 import format as formatlog
 import logmerge
+import six
 
 
 def get_parser():
@@ -18,7 +19,7 @@ def get_parser():
 
     sub_parser = parser.add_subparsers(title='Subcommands')
 
-    for command, (parser_func, main_func) in commands.iteritems():
+    for command, (parser_func, main_func) in six.iteritems(commands):
         parent = parser_func(False)
         command_parser = sub_parser.add_parser(command,
                                                description=parent.description,
diff --git a/testing/mozbase/mozlog/mozlog/scripts/format.py b/testing/mozbase/mozlog/mozlog/scripts/format.py
index f4e618d7763e..d82d9e9de2f8 100644
--- a/testing/mozbase/mozlog/mozlog/scripts/format.py
+++ b/testing/mozbase/mozlog/mozlog/scripts/format.py
@@ -14,7 +14,7 @@ def get_parser(add_help=True):
                         help="Filename to read from, defaults to stdin")
     parser.add_argument("--output", action="store", default=None,
                         help="Filename to write to, defaults to stdout")
-    parser.add_argument("format", choices=commandline.log_formatters.keys(),
+    parser.add_argument("format", choices=list(commandline.log_formatters.keys()),
                         help="Format to use")
     return parser
 
diff --git a/testing/mozbase/mozlog/mozlog/scripts/unstable.py b/testing/mozbase/mozlog/mozlog/scripts/unstable.py
index df86bf8e6fb5..1858a3399e0f 100644
--- a/testing/mozbase/mozlog/mozlog/scripts/unstable.py
+++ b/testing/mozbase/mozlog/mozlog/scripts/unstable.py
@@ -5,6 +5,7 @@ from collections import defaultdict
 import json
 
 from mozlog import reader
+import six
 
 
 class StatusHandler(reader.LogHandler):
@@ -15,7 +16,7 @@ class StatusHandler(reader.LogHandler):
             lambda: defaultdict(lambda: defaultdict(int))))
 
     def test_id(self, test):
-        if type(test) in (str, unicode):
+        if type(test) in (str, six.text_type):
             return test
         else:
             return tuple(test)
@@ -48,9 +49,9 @@ def _filter(results_cmp):
     def inner(statuses):
         rv = defaultdict(lambda: defaultdict(dict))
 
-        for run_info, tests in statuses.iteritems():
-            for test, subtests in tests.iteritems():
-                for name, results in subtests.iteritems():
+        for run_info, tests in six.iteritems(statuses):
+            for test, subtests in six.iteritems(tests):
+                for name, results in six.iteritems(subtests):
                     if results_cmp(results):
                         rv[run_info][test][name] = results
 
@@ -65,16 +66,16 @@ filter_stable = _filter(lambda x: len(x) == 1)
 def group_results(data):
     rv = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
 
-    for run_info, tests in data.iteritems():
-        for test, subtests in tests.iteritems():
-            for name, results in subtests.iteritems():
-                for status, number in results.iteritems():
+    for run_info, tests in six.iteritems(data):
+        for test, subtests in six.iteritems(tests):
+            for name, results in six.iteritems(subtests):
+                for status, number in six.iteritems(results):
                     rv[test][name][status] += number
     return rv
 
 
 def print_results(data):
-    for run_info, tests in data.iteritems():
+    for run_info, tests in six.iteritems(data):
         run_str = " ".join("%s:%s" % (k, v) for k, v in run_info) if run_info else "No Run Info"
         print(run_str)
         print("=" * len(run_str))
@@ -85,9 +86,9 @@ def print_run(tests):
     for test, subtests in sorted(tests.items()):
         print("\n" + str(test))
         print("-" * len(test))
-        for name, results in subtests.iteritems():
+        for name, results in six.iteritems(subtests):
             print("[%s]: %s" % (name if name is not None else "",
-                                " ".join("%s (%i)" % (k, v) for k, v in results.iteritems())))
+                                " ".join("%s (%i)" % (k, v) for k, v in six.iteritems(results))))
 
 
 def get_parser(add_help=True):
diff --git a/testing/mozbase/mozlog/mozlog/structuredlog.py b/testing/mozbase/mozlog/mozlog/structuredlog.py
index 74a7ef55e3f4..56f1b72f145a 100644
--- a/testing/mozbase/mozlog/mozlog/structuredlog.py
+++ b/testing/mozbase/mozlog/mozlog/structuredlog.py
@@ -13,6 +13,7 @@ import traceback
 
 from .logtypes import Unicode, TestId, TestList, Status, SubStatus, Dict, List, Int, Any, Tuple
 from .logtypes import log_action, convertor_registry
+import six
 
 """Structured Logging for recording test results.
 
@@ -207,7 +208,7 @@ class StructuredLogger(object):
 
         action = raw_data["action"]
         converted_data = convertor_registry[action].convert_known(**raw_data)
-        for k, v in raw_data.iteritems():
+        for k, v in six.iteritems(raw_data):
             if k not in converted_data:
                 converted_data[k] = v
 
diff --git a/testing/mozbase/mozlog/tests/test_logger.py b/testing/mozbase/mozlog/tests/test_logger.py
index 330c6eccfbb7..247939786b67 100644
--- a/testing/mozbase/mozlog/tests/test_logger.py
+++ b/testing/mozbase/mozlog/tests/test_logger.py
@@ -15,6 +15,7 @@ import mozunit
 
 import mozfile
 import mozlog.unstructured as mozlog
+import six
 
 
 class ListHandler(mozlog.Handler):
@@ -83,7 +84,7 @@ class TestStructuredLogging(unittest.TestCase):
         The actual message should contain no fields other than the timestamp
         field and those present in expected."""
 
-        self.assertTrue(isinstance(actual['_time'], (int, long)))
+        self.assertTrue(isinstance(actual['_time'], six.integer_types))
 
         for k, v in expected.items():
             self.assertEqual(v, actual[k])
diff --git a/testing/mozbase/mozlog/tests/test_structured.py b/testing/mozbase/mozlog/tests/test_structured.py
index 82bcbbe8d21f..6e3815e24250 100644
--- a/testing/mozbase/mozlog/tests/test_structured.py
+++ b/testing/mozbase/mozlog/tests/test_structured.py
@@ -23,6 +23,8 @@ from mozlog import (
     handlers,
     formatters,
 )
+import six
+from six import unichr
 
 
 class TestHandler(object):
@@ -62,7 +64,7 @@ class BaseStructuredTest(unittest.TestCase):
         specials = set(["time"])
 
         all_expected.update(expected)
-        for key, value in all_expected.iteritems():
+        for key, value in six.iteritems(all_expected):
             self.assertEqual(actual[key], value)
 
         self.assertEquals(set(all_expected.keys()) |
@@ -1013,7 +1015,7 @@ class TestBuffer(BaseStructuredTest):
         specials = set(["time"])
 
         all_expected.update(expected)
-        for key, value in all_expected.iteritems():
+        for key, value in six.iteritems(all_expected):
             self.assertEqual(actual[key], value)
 
         self.assertEquals(set(all_expected.keys()) |