From b638ed69e190d119da3df2559e83059b69093377 Mon Sep 17 00:00:00 2001
From: Andrew Halberstadt <ahalberstadt@mozilla.com>
Date: Wed, 22 Feb 2017 14:36:31 -0500
Subject: [PATCH] Bug 1340551 - Log tests by manifest from suite_start in
 mochitest harness, r=jgraham

MozReview-Commit-ID: 1lcw62fmofa

--HG--
extra : rebase_source : 930dd542be6fdace8c6bec916d2246329afffad6
---
 testing/mochitest/runrobocop.py |  8 +++++++-
 testing/mochitest/runtests.py   | 10 ++++++----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/testing/mochitest/runrobocop.py b/testing/mochitest/runrobocop.py
index d263bbc50f6f..1d555fd88d6d 100644
--- a/testing/mochitest/runrobocop.py
+++ b/testing/mochitest/runrobocop.py
@@ -7,6 +7,7 @@ import os
 import sys
 import tempfile
 import traceback
+from collections import defaultdict
 
 sys.path.insert(
     0, os.path.abspath(
@@ -511,7 +512,12 @@ class RobocopTestRunner(MochitestDesktop):
                               (test['name'], test['disabled']))
                 continue
             active_tests.append(test)
-        self.log.suite_start([t['name'] for t in active_tests])
+
+        tests_by_manifest = defaultdict(list)
+        for test in active_tests:
+            tests_by_manifest[test['manifest']].append(test['name'])
+        self.log.suite_start(tests_by_manifest)
+
         worstTestResult = None
         for test in active_tests:
             result = self.runSingleTest(test)
diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py
index a06d9e2fe884..be875ab056ce 100644
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -13,6 +13,7 @@ SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 sys.path.insert(0, SCRIPT_DIR)
 
 from argparse import Namespace
+from collections import defaultdict
 import ctypes
 import glob
 import json
@@ -798,6 +799,7 @@ class MochitestDesktop(object):
         self.wsserver = None
         self.websocketProcessBridge = None
         self.sslTunnel = None
+        self.tests_by_manifest = defaultdict(list)
         self._active_tests = None
         self._locations = None
 
@@ -1305,7 +1307,7 @@ toolbar#nav-bar {
     def logPreamble(self, tests):
         """Logs a suite_start message and test_start/test_end at the beginning of a run.
         """
-        self.log.suite_start([t['path'] for t in tests])
+        self.log.suite_start(self.tests_by_manifest)
         for test in tests:
             if 'disabled' in test:
                 self.log.test_start(test['path'])
@@ -1440,7 +1442,6 @@ toolbar#nav-bar {
         # When running mochitest locally the manifest is based on topsrcdir,
         # but when running in automation it is based on the test root.
         manifest_root = build_obj.topsrcdir if build_obj else self.testRootAbs
-        manifests = set()
         for test in tests:
             if len(tests) == 1 and 'disabled' in test:
                 del test['disabled']
@@ -1455,7 +1456,8 @@ toolbar#nav-bar {
                     (test['name'], test['manifest']))
                 continue
 
-            manifests.add(os.path.relpath(test['manifest'], manifest_root))
+            manifest_relpath = os.path.relpath(test['manifest'], manifest_root)
+            self.tests_by_manifest[manifest_relpath].append(tp)
 
             testob = {'path': tp}
             if 'disabled' in test:
@@ -1491,7 +1493,7 @@ toolbar#nav-bar {
         if 'MOZ_UPLOAD_DIR' in os.environ:
             artifact = os.path.join(os.environ['MOZ_UPLOAD_DIR'], 'manifests.list')
             with open(artifact, 'a') as fh:
-                fh.write('\n'.join(sorted(manifests)))
+                fh.write('\n'.join(sorted(self.tests_by_manifest.keys())))
 
         self._active_tests = paths
         return self._active_tests