Bug 1814442: Improve ./mach vendor's output of long file lists r=jewilde

Differential Revision: https://phabricator.services.mozilla.com/D169708

python/mozbuild/mozbuild/test/python.ini

@@ -58,3 +58,4 @@ subsuite = mozbuild
 [test_vendor.py]
 skip-if = true # Bug 1765416
 requirements = python/mozbuild/mozbuild/test/vendor_requirements.txt
+[test_vendor_tools.py]

python/mozbuild/mozbuild/test/test_vendor_tools.py

@@ -0,0 +1,90 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozunit
+
+from mozbuild.vendor.vendor_manifest import list_of_paths_to_readable_string
+
+
+def test_list_of_paths_to_readable_string():
+    paths = ["/tmp/a", "/tmp/b"]
+    s = list_of_paths_to_readable_string(paths)
+    assert not s.endswith(", ]")
+    assert s.endswith("]")
+    assert "/tmp/a" in s
+    assert "/tmp/b" in s
+
+    paths = ["/tmp/a", "/tmp/b", "/tmp/c", "/tmp/d"]
+    s = list_of_paths_to_readable_string(paths)
+    assert not s.endswith(", ")
+    assert s.endswith("]")
+    assert "/tmp/a" not in s
+    assert "/tmp/b" not in s
+    assert "4 items in /tmp" in s
+
+    paths = [
+        "/tmp/a",
+        "/tmp/b",
+        "/tmp/c",
+        "/tmp/d",
+        "/tmp/d",
+        "/tmp/d",
+        "/tmp/d",
+        "/tmp/d",
+        "/tmp/d",
+        "/tmp/d",
+    ]
+    s = list_of_paths_to_readable_string(paths)
+    assert not s.endswith(", ")
+    assert s.endswith("]")
+    assert "/tmp/a" not in s
+    assert " a" not in s
+    assert "/tmp/b" not in s
+    assert "10 (omitted) items in /tmp" in s
+
+    paths = ["/tmp", "/foo"]
+    s = list_of_paths_to_readable_string(paths)
+    assert not s.endswith(", ")
+    assert s.endswith("]")
+    assert "/tmp" in s
+    assert "/foo" in s
+
+    paths = [
+        "/tmp/a",
+        "/tmp/b",
+        "/tmp/c",
+        "/tmp/d",
+        "/tmp/d",
+        "/tmp/d",
+        "/tmp/d",
+        "/tmp/d",
+        "/tmp/d",
+        "/tmp/d",
+    ]
+    paths.extend(["/foo/w", "/foo/x", "/foo/y", "/foo/z"])
+    paths.extend(["/bar/m", "/bar/n"])
+    paths.extend(["/etc"])
+    s = list_of_paths_to_readable_string(paths)
+    assert not s.endswith(", ")
+    assert s.endswith("]")
+    assert "/tmp/a" not in s
+    assert " d" not in s
+    assert "/tmp/b" not in s
+    assert "10 (omitted) items in /tmp" in s
+
+    assert "/foo/w" not in s
+    assert "/foo/x" not in s
+    assert "4 items in /foo" in s
+    assert " w" in s
+
+    assert "/bar/m" in s
+    assert "/bar/n" in s
+
+    assert "/etc" in s
+
+    assert len(s) < len(str(paths))
+
+
+if __name__ == "__main__":
+    mozunit.main()

python/mozbuild/mozbuild/vendor/vendor_manifest.py

@@ -11,6 +11,7 @@ import shutil
 import sys
 import tarfile
 import tempfile
+from collections import defaultdict
 
 import mozfile
 import mozpack.path as mozpath
@@ -51,6 +52,38 @@ def _replace_in_file(file, pattern, replacement, regex=False):
         f.write(newcontents)
 
 
+def list_of_paths_to_readable_string(paths):
+    # From https://stackoverflow.com/a/41578071
+    dic = defaultdict(list)
+    for item in paths:
+        if os.path.isdir(item):  # To check path is a directory
+            _ = dic[item]  # will set default value as empty list
+        else:
+            path, file = os.path.split(item)
+            dic[path].append(file)
+
+    final_string = "["
+    for key, val in dic.items():
+        if len(val) == 0:
+            final_string += key + ", "
+        elif len(val) < 3:
+            final_string += ", ".join([os.path.join(key, v) for v in val]) + ", "
+        elif len(val) < 10:
+            final_string += "%s items in %s: %s and %s, " % (
+                len(val),
+                key,
+                ", ".join(val[0:-1]),
+                val[-1],
+            )
+        else:
+            final_string += "%s (omitted) items in %s, " % (len(val), key)
+
+    if final_string[-2:] == ", ":
+        final_string = final_string[:-2]
+
+    return final_string + "]"
+
+
 class VendorManifest(MozbuildObject):
     def should_perform_step(self, step):
         return step not in self.manifest["vendoring"].get("skip-vendoring-steps", [])
@@ -415,7 +448,10 @@ class VendorManifest(MozbuildObject):
 
                 to_exclude = list(set(to_exclude) - set(to_include))
                 if to_exclude:
-                    self.logInfo({"files": str(to_exclude)}, "Removing: {files}")
+                    self.logInfo(
+                        {"files": list_of_paths_to_readable_string(to_exclude)},
+                        "Removing: {files}",
+                    )
                     for exclusion in to_exclude:
                         mozfile.remove(exclusion)