Bug 1577777 - perf-reftest-singletons (perf_reftest_singletons) harness should not alert on the overall number, just on the subtests r=perftest-reviewers,rwood

Differential Revision: https://phabricator.services.mozilla.com/D53961

--HG--
extra : moz-landing-system : lando

testing/talos/talos/output.py

@@ -47,7 +47,8 @@ class Output(object):
                 suite = {
                     'name': test.name(),
                     'extraOptions': self.results.extra_options or [],
-                    'subtests': subtests
+                    'subtests': subtests,
+                    'shouldAlert': test.test_config.get('suite_should_alert')
                 }
 
                 suites.append(suite)
@@ -182,7 +183,8 @@ class Output(object):
             if counter_subtests:
                 suites.append({'name': test.name(),
                                'extraOptions': self.results.extra_options or [],
-                               'subtests': counter_subtests})
+                               'subtests': counter_subtests,
+                               'shouldAlert': test.test_config.get('suite_should_alert')})
         return test_results
 
     def output(self, results, results_url):

testing/talos/talos/test.py

@@ -270,6 +270,7 @@ class PageloaderTest(Test):
     tpcycles = 1  # number of time to run each page
     cycles = None
     timeout = None
+    suite_should_alert = True
 
     keys = ['tpmanifest', 'tpcycles', 'tppagecycles', 'tprender', 'tpchrome',
             'tpmozafterpaint', 'fnbpaint', 'tphero', 'tploadnocache', 'firstpaint',
@@ -280,7 +281,8 @@ class PageloaderTest(Test):
             'profile_path', 'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
             'format_pagename', 'filters', 'preferences', 'extensions', 'setup', 'cleanup',
             'lower_is_better', 'alert_threshold', 'unit', 'webextensions', 'profile',
-            'subtest_alerts', 'perfherder_framework', 'pdfpaint', 'webextensions_folder']
+            'suite_should_alert', 'subtest_alerts', 'perfherder_framework', 'pdfpaint',
+            'webextensions_folder']
 
 
 class QuantumPageloadTest(PageloaderTest):
@@ -933,6 +935,7 @@ class perf_reftest_singletons(PageloaderTest):
     lower_is_better = True
     alert_threshold = 5.0
     subtest_alerts = True
+    suite_should_alert = False
 
 
 @register_test()