Bug 1640758 - Use multiple errorsummary and raw logs for test-verify; r=jmaher

In all mozharness scripts running test-verify (or per-test coverage), use distinct
file names for errorsummary and raw logs. This usually means that the command needs
to be built inside the per-test loop.

Differential Revision: https://phabricator.services.mozilla.com/D77746

testing/mozharness/mozharness/mozilla/testing/per_test_base.py

@@ -28,6 +28,7 @@ class SingleTestMixin(object):
         # Map from full test path on the test machine to a relative path in the source checkout.
         # Use self._map_test_path_to_source(test_machine_path, source_path) to add a mapping.
         self.test_src_path = {}
+        self.per_test_log_index = 1
 
     def _map_test_path_to_source(self, test_machine_path, source_path):
         test_machine_path = test_machine_path.replace(os.sep, posixpath.sep)
@@ -396,3 +397,16 @@ class SingleTestMixin(object):
             test_name = test_name.rstrip(os.path.sep)
         self.log("TinderboxPrint: Per-test run of %s<br/>: %s" %
                  (test_name, tbpl_status), level=log_level)
+
+    def get_indexed_logs(self, dir, test_suite):
+        """
+           Per-test tasks need distinct file names for the raw and errorsummary logs
+           on each run.
+        """
+        index = ''
+        if self.verify_enabled or self.per_test_coverage:
+            index = '-test%d' % self.per_test_log_index
+            self.per_test_log_index += 1
+        raw_log_file = os.path.join(dir, '%s%s_raw.log' % (test_suite, index))
+        error_summary_file = os.path.join(dir, '%s%s_errorsummary.log' % (test_suite, index))
+        return raw_log_file, error_summary_file

testing/mozharness/scripts/android_emulator_unittest.py

@@ -223,11 +223,9 @@ class AndroidEmulatorTest(TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMi
             ),
         ]
 
-        raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                    '%s_raw.log' % self.test_suite)
+        raw_log_file, error_summary_file = self.get_indexed_logs(dirs['abs_blob_upload_dir'],
+                                                                 self.test_suite)
 
-        error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                          '%s_errorsummary.log' % self.test_suite)
         str_format_values = {
             'device_serial': self.device_serial,
             # IP address of the host as seen from the emulator
@@ -404,8 +402,6 @@ class AndroidEmulatorTest(TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMi
         for (per_test_suite, suite) in suites:
             self.test_suite = suite
 
-            cmd = self._build_command()
-
             try:
                 cwd = self._query_tests_dir(self.test_suite)
             except Exception:
@@ -432,6 +428,7 @@ class AndroidEmulatorTest(TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMi
                     # suite categories loops also.
                     return
 
+                cmd = self._build_command()
                 final_cmd = copy.copy(cmd)
                 if len(per_test_args) > 0:
                     # in per-test mode, remove any chunk arguments from command

testing/mozharness/scripts/android_hardware_unittest.py

@@ -193,11 +193,9 @@ class AndroidHardwareTest(TestingMixin, BaseScript, MozbaseMixin,
             ),
         ]
 
-        raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                    '%s_raw.log' % self.test_suite)
+        raw_log_file, error_summary_file = self.get_indexed_logs(dirs['abs_blob_upload_dir'],
+                                                                 self.test_suite)
 
-        error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                          '%s_errorsummary.log' % self.test_suite)
         str_format_values = {
             'device_serial': self.device_serial,
             'remote_webserver': c['remote_webserver'],
@@ -363,8 +361,6 @@ class AndroidHardwareTest(TestingMixin, BaseScript, MozbaseMixin,
         for (per_test_suite, suite) in suites:
             self.test_suite = suite
 
-            cmd = self._build_command()
-
             try:
                 cwd = self._query_tests_dir()
             except Exception:
@@ -388,6 +384,7 @@ class AndroidHardwareTest(TestingMixin, BaseScript, MozbaseMixin,
                     # suite categories loops also.
                     return
 
+                cmd = self._build_command()
                 final_cmd = copy.copy(cmd)
                 if len(per_test_args) > 0:
                     # in per-test mode, remove any chunk arguments from command

testing/mozharness/scripts/desktop_unittest.py

@@ -381,11 +381,9 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
             abs_app_dir = self.query_abs_app_dir()
             abs_res_dir = self.query_abs_res_dir()
 
-            raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                        '%s_raw.log' % suite)
+            raw_log_file, error_summary_file = self.get_indexed_logs(
+                dirs['abs_blob_upload_dir'], suite)
 
-            error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                              '%s_errorsummary.log' % suite)
             str_format_values = {
                 'binary_path': self.binary_path,
                 'symbols_path': self._query_symbols_url(),
@@ -817,8 +815,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
                 if executed_too_many_tests and not self.per_test_coverage:
                     return False
 
-                abs_base_cmd = self._query_abs_base_cmd(suite_category, suite)
-                cmd = abs_base_cmd[:]
                 replace_dict = {
                     'abs_app_dir': abs_app_dir,
 
@@ -846,13 +842,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
                 flavor = self._query_try_flavor(suite_category, suite)
                 try_options, try_tests = self.try_args(flavor)
 
-                cmd.extend(self.query_options(options_list,
-                                              try_options,
-                                              str_format_values=replace_dict))
-                cmd.extend(self.query_tests_args(tests_list,
-                                                 try_tests,
-                                                 str_format_values=replace_dict))
-
                 suite_name = suite_category + '-' + suite
                 tbpl_status, log_level = None, None
                 error_list = BaseErrorList + HarnessErrorList
@@ -918,6 +907,15 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
 
                         executed_tests = executed_tests + 1
 
+                    abs_base_cmd = self._query_abs_base_cmd(suite_category, suite)
+                    cmd = abs_base_cmd[:]
+                    cmd.extend(self.query_options(options_list,
+                                                  try_options,
+                                                  str_format_values=replace_dict))
+                    cmd.extend(self.query_tests_args(tests_list,
+                                                     try_tests,
+                                                     str_format_values=replace_dict))
+
                     final_cmd = copy.copy(cmd)
                     final_cmd.extend(per_test_args)
 

testing/mozharness/scripts/web_platform_tests.py

@@ -227,13 +227,14 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidM
 
         mozinfo.find_and_update_from_json(dirs['abs_test_install_dir'])
 
+        raw_log_file, error_summary_file = self.get_indexed_logs(dirs['abs_blob_upload_dir'],
+                                                                 'wpt')
+
         cmd += ["--log-raw=-",
-                "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
-                                              "wpt_raw.log"),
+                "--log-raw=%s" % raw_log_file,
                 "--log-wptreport=%s" % os.path.join(dirs["abs_blob_upload_dir"],
                                                     "wptreport.json"),
-                "--log-errorsummary=%s" % os.path.join(dirs["abs_blob_upload_dir"],
-                                                       "wpt_errorsummary.log"),
+                "--log-errorsummary=%s" % error_summary_file,
                 "--binary=%s" % self.binary_path,
                 "--symbols-path=%s" % self.symbols_path,
                 "--stackwalk-binary=%s" % self.query_minidump_stackwalk(),