Bug 1367745 - Make sure 'process_output' messages are logged at the proper mozharness level, r=jgraham

In addition to saving the log level of 'process_output' messages, this will also start passing 'log'
messages through the error lists. This means mozharness will start using 'log' errors when determining
the tbpl_status and worst_log_level.

MozReview-Commit-ID: CZnH6aI1Wo0

--HG--
extra : rebase_source : 55c74bfa2afdf6d7b510b351ad657ecd615d4347

testing/mozharness/mozharness/base/log.py

@@ -297,6 +297,7 @@ class OutputParser(LogMixin):
         Args:
             line (str): command line output to parse.
         """
+        log_level = INFO
         for error_check in self.error_list:
             # TODO buffer for context_lines.
             match = False
@@ -310,7 +311,7 @@ class OutputParser(LogMixin):
                 self.warning("error_list: 'substr' and 'regex' not in %s" %
                              error_check)
             if match:
-                log_level = error_check.get('level', INFO)
+                log_level = error_check.get('level', log_level)
                 if self.log_output:
                     message = ' %s' % line
                     if error_check.get('explanation'):
@@ -328,7 +329,9 @@ class OutputParser(LogMixin):
                 break
         else:
             if self.log_output:
-                self.info(' %s' % line)
+                self.log(' %s' % line, level=log_level)
+
+        return log_level
 
     def add_lines(self, output):
         """ process a string or list of strings, decode them to utf-8,strip

testing/mozharness/mozharness/mozilla/structuredlog.py

@@ -49,7 +49,7 @@ class StructuredOutputParser(OutputParser):
 
     def _handle_unstructured_output(self, line, log_output=True):
         self.log_output = log_output
-        super(StructuredOutputParser, self).parse_single_line(line)
+        return super(StructuredOutputParser, self).parse_single_line(line)
 
     def parse_single_line(self, line):
         """Parses a line of log output from the child process and passes
@@ -80,13 +80,19 @@ class StructuredOutputParser(OutputParser):
         self.handler(data)
 
         action = data["action"]
-        if action == "process_output":
-            # Run process output through the error lists, but make sure the super parser
-            # doesn't print them to stdout (they should go through the log formatter).
-            self._handle_unstructured_output(data['data'], log_output=False)
+        if action in ('log', 'process_output'):
+            # Run log and process_output actions through the error lists, but make sure
+            # the super parser doesn't print them to stdout (they should go through the
+            # log formatter).
+            if 'message' in data:
+                message = data['message']
+            else:
+                message = data['data']
+            error_level = self._handle_unstructured_output(message, log_output=False)
 
-        if action == "log":
-            level = getattr(log, data["level"].upper())
+            if 'level' in data:
+                level = getattr(log, data['level'].upper())
+            level = self.worst_level(error_level, level)
 
         log_data = self.formatter(data)
         if log_data is not None: