Bug 1819736 - Make test-manifest-alpha more helpful in its feedback. r=ahal

This patch makes it so that test-manifest-alpha emits a diff between unsorted
and sorted order to make it easier for people to reorder these files (since it's
currently kinda difficult to reorder the files automatically).

Differential Revision: https://phabricator.services.mozilla.com/D171393

tools/lint/test-manifest-alpha/__init__.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import difflib
 import os
 import sys
 
@@ -56,6 +57,10 @@ def lint(paths, config, fix=None, **lintargs):
             ):
                 level = "error"
 
+            diff_instance = difflib.Differ()
+            diff_result = diff_instance.compare(test_names, sorted_test_names)
+            diff_list = list(diff_result)
+
             res = {
                 "path": rel_file_path,
                 "lineno": 0,
@@ -65,6 +70,7 @@ def lint(paths, config, fix=None, **lintargs):
                     "Expected ordering: \n\n%s\n\n" % "\n".join(sorted_test_names)
                 ),
                 "level": level,
+                "diff": "\n".join(diff_list),
             }
             results.append(result.from_config(config, **res))
 

tools/lint/test/test_manifest_alpha.py

@@ -9,6 +9,7 @@ LINTER = "test-manifest-alpha"
 def test_very_out_of_order(lint, paths):
     results = lint(paths("mochitest-very-out-of-order.ini"))
     assert len(results) == 1
+    assert results[0].diff
 
 
 def test_in_order(lint, paths):
@@ -19,6 +20,7 @@ def test_in_order(lint, paths):
 def test_mostly_in_order(lint, paths):
     results = lint(paths("mochitest-mostly-in-order.ini"))
     assert len(results) == 1
+    assert results[0].diff
 
 
 def test_other_ini_very_out_of_order(lint, paths):