Bug 1739426 - Add a diff feature to perfdocs verification. r=perftest-reviewers,AlexandruIonescu

This patch adds some code to produce a diff of the files that are problematic.

Differential Revision: https://phabricator.services.mozilla.com/D130380

tools/lint/perfdocs/utils.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 from __future__ import absolute_import
 
+import difflib
 import filecmp
 import os
 import yaml
@@ -65,6 +66,13 @@ def are_dirs_equal(dir_1, dir_2):
 
     dirs_cmp = filecmp.dircmp(dir_1, dir_2)
     if dirs_cmp.left_only or dirs_cmp.right_only or dirs_cmp.funny_files:
+        logger.log("Some files are missing or are funny.")
+        for file in dirs_cmp.left_only:
+            logger.log(f"Missing in existing docs: {file}")
+        for file in dirs_cmp.right_only:
+            logger.log(f"Missing in new docs: {file}")
+        for file in dirs_cmp.funny_files:
+            logger.log(f"The following file is funny: {file}")
         return False
 
     _, mismatch, errors = filecmp.cmpfiles(
@@ -72,6 +80,18 @@ def are_dirs_equal(dir_1, dir_2):
     )
 
     if mismatch or errors:
+        logger.log(f"Found mismatches: {mismatch}")
+        for entry in mismatch:
+            logger.log(f"Mismatch found on {entry}")
+            with open(os.path.join(dir_1, entry)) as f:
+                newlines = f.readlines()
+            with open(os.path.join(dir_2, entry)) as f:
+                baselines = f.readlines()
+            for line in difflib.unified_diff(
+                baselines, newlines, fromfile="base", tofile="new"
+            ):
+                logger.log(line)
+            logger.log(f"Completed diff on {entry}")
         return False
 
     for common_dir in dirs_cmp.common_dirs:

tools/lint/test/test_perfdocs_generation.py

@@ -36,10 +36,12 @@ def setup_sample_logger(logger, structured_logger, top_dir):
     import perfdocs.verifier as vf
     import perfdocs.gatherer as gt
     import perfdocs.generator as gn
+    import perfdocs.utils as utils
 
     gt.logger = logger
     vf.logger = logger
     gn.logger = logger
+    utils.logger = logger
 
 
 @mock.patch("perfdocs.logger.PerfDocLogger")
@@ -123,6 +125,17 @@ def test_perfdocs_generator_needed_update(logger, structured_logger, perfdocs_sa
     assert logger.warning.call_count == 1
     assert args[0] == expected
 
+    # Check to ensure a diff was produced
+    assert logger.log.call_count == 6
+
+    logs = [v[0][0] for v in logger.log.call_args_list]
+    for failure_log in (
+        "Some files are missing or are funny.",
+        "Missing in existing docs: index.rst",
+        "Missing in existing docs: mozperftest.rst",
+    ):
+        assert failure_log in logs
+
 
 @mock.patch("perfdocs.logger.PerfDocLogger")
 def test_perfdocs_generator_created_perfdocs(