Bug 1832974 - Changing unclear output message when too many tests are selected. r=perftest-reviewers,afinder

When too many tests are run(defined as rebuilds * tests selected > 300) we say too many tests are selected without letting the user know how far over the 300 limit they have gone, this patches makes it clear that you are over the 300 maximum and displays the math clearly so that they can alter the needed things to make their runs be under the threshold

Differential Revision: https://phabricator.services.mozilla.com/D183781

tools/tryselect/selectors/perf.py

@@ -1158,11 +1158,14 @@ class PerfParser(CompareParser):
             print("No tasks selected")
             return None
 
-        if (len(selected_tasks) * rebuild) > MAX_PERF_TASKS:
+        total_task_count = len(selected_tasks) * rebuild
+        if total_task_count > MAX_PERF_TASKS:
             print(
-                "That's a lot of tests selected (%s)!\n"
-                "These tests won't be triggered. If this was unexpected, "
-                "please file a bug in Testing :: Performance." % MAX_PERF_TASKS
+                "\n\n----------------------------------------------------------------------------------------------\n"
+                f"You have selected {total_task_count} total test runs! (selected tasks({len(selected_tasks)}) * rebuild"
+                f" count({rebuild}) \nThese tests won't be triggered as the current maximum for a single ./mach try "
+                f"perf run is {MAX_PERF_TASKS}. \nIf this was unexpected, please file a bug in Testing :: Performance."
+                "\n----------------------------------------------------------------------------------------------\n\n"
             )
             return None
 

tools/tryselect/test/test_perf.py

@@ -1170,9 +1170,11 @@ def test_save_revision_treeherder(args, call_counts, exists_cache_file):
             {},
             [1, 0, 0, 1],
             (
-                "That's a lot of tests selected (300)!\n"
-                "These tests won't be triggered. If this was unexpected, "
-                "please file a bug in Testing :: Performance."
+                "\n\n----------------------------------------------------------------------------------------------\n"
+                f"You have selected {MAX_PERF_TASKS+1} total test runs! (selected tasks({MAX_PERF_TASKS+1}) * rebuild"
+                f" count(1) \nThese tests won't be triggered as the current maximum for a single ./mach try "
+                f"perf run is {MAX_PERF_TASKS}. \nIf this was unexpected, please file a bug in Testing :: Performance."
+                "\n----------------------------------------------------------------------------------------------\n\n"
             ),
             True,
         ),
@@ -1191,9 +1193,12 @@ def test_save_revision_treeherder(args, call_counts, exists_cache_file):
             {"show_all": True, "try_config": {"rebuild": 2}},
             [1, 0, 0, 1],
             (
-                "That's a lot of tests selected (300)!\n"
-                "These tests won't be triggered. If this was unexpected, "
-                "please file a bug in Testing :: Performance."
+                "\n\n----------------------------------------------------------------------------------------------\n"
+                f"You have selected {int((MAX_PERF_TASKS + 2) / 2) * 2} total test runs! (selected tasks("
+                f"{int((MAX_PERF_TASKS + 2) / 2)}) * rebuild"
+                f" count(2) \nThese tests won't be triggered as the current maximum for a single ./mach try "
+                f"perf run is {MAX_PERF_TASKS}. \nIf this was unexpected, please file a bug in Testing :: Performance."
+                "\n----------------------------------------------------------------------------------------------\n\n"
             ),
             True,
         ),