From 3217de63504f517a09e3755426300b1874a1cbea Mon Sep 17 00:00:00 2001 From: Joel Maher Date: Wed, 3 Feb 2021 17:55:21 -0500 Subject: [PATCH] Bug 1689610 - replace passFailRatio with totalFailures/totalJobs (#7001) * Bug 1684931 - include manifest name in result_groups * Bug 1689610 - replace passFailRatio with totalFailures/totalJobs --- tests/ui/mock/push_health.json | 171 +++++++++++++++-------- treeherder/push_health/classification.py | 4 +- treeherder/push_health/tests.py | 15 +- 3 files changed, 124 insertions(+), 66 deletions(-) diff --git a/tests/ui/mock/push_health.json b/tests/ui/mock/push_health.json index ab7ab0d49..4ed882620 100644 --- a/tests/ui/mock/push_health.json +++ b/tests/ui/mock/push_health.json @@ -754,7 +754,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -771,7 +772,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -788,7 +790,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -805,7 +808,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -822,7 +826,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -839,7 +844,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -856,7 +862,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -873,7 +880,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -890,7 +898,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -907,7 +916,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -924,7 +934,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -941,7 +952,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -958,7 +970,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -975,7 +988,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -992,7 +1006,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -1009,7 +1024,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false }, { @@ -1026,7 +1042,8 @@ "suggestedClassification": "New Failure", "confidence": 0, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": false } ], @@ -1045,7 +1062,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0.75, + "totalFailures": 3, + "totalJobs": 4, "isClassifiedIntermittent": true }, { @@ -1062,7 +1080,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1079,7 +1098,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1096,7 +1116,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1113,7 +1134,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1130,7 +1152,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1147,7 +1170,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0.75, + "totalFailures": 3, + "totalJobs": 4, "isClassifiedIntermittent": true }, { @@ -1164,7 +1188,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0.75, + "totalFailures": 3, + "totalJobs": 4, "isClassifiedIntermittent": true }, { @@ -1181,7 +1206,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0.5, + "totalFailures": 2, + "totalJobs": 4, "isClassifiedIntermittent": true }, { @@ -1198,7 +1224,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1215,7 +1242,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1232,7 +1260,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1249,7 +1278,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1266,7 +1296,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1283,7 +1314,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1300,7 +1332,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1317,7 +1350,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1334,7 +1368,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1351,7 +1386,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1368,7 +1404,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1385,7 +1422,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1402,7 +1440,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1419,7 +1458,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1436,7 +1476,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1453,7 +1494,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1470,7 +1512,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0.8333333333333334, + "totalFailures": 8, + "totalJobs": 10, "isClassifiedIntermittent": true }, { @@ -1487,7 +1530,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1504,7 +1548,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1521,7 +1566,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1538,7 +1584,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1555,7 +1602,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1572,7 +1620,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1589,7 +1638,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1606,7 +1656,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1623,7 +1674,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1640,7 +1692,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0.75, + "totalFailures": 3, + "totalJobs": 4, "isClassifiedIntermittent": true }, { @@ -1657,7 +1710,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 2, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true }, { @@ -1674,7 +1728,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0.6666666666666666, + "totalFailures": 2, + "totalJobs": 3, "isClassifiedIntermittent": true }, { @@ -1691,7 +1746,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0.6666666666666666, + "totalFailures": 2, + "totalJobs": 3, "isClassifiedIntermittent": true }, { @@ -1708,7 +1764,8 @@ "suggestedClassification": "intermittent", "confidence": 100, "tier": 1, - "passFailRatio": 0, + "totalFailures": 1, + "totalJobs": 1, "isClassifiedIntermittent": true } ] diff --git a/treeherder/push_health/classification.py b/treeherder/push_health/classification.py index 924433e27..e603ee2c1 100644 --- a/treeherder/push_health/classification.py +++ b/treeherder/push_health/classification.py @@ -76,7 +76,9 @@ def get_grouped(failures): for failure in failures: is_intermittent = failure['suggestedClassification'] == 'intermittent' - if (is_intermittent and failure['confidence'] == 100) or failure['passFailRatio'] > 0.5: + if (is_intermittent and failure['confidence'] == 100) or failure['totalFailures'] / failure[ + 'totalJobs' + ] <= 0.5: classified[KNOWN_ISSUES].append(failure) else: classified[NEED_INVESTIGATION].append(failure) diff --git a/treeherder/push_health/tests.py b/treeherder/push_health/tests.py index a60304fde..1cb1ff879 100644 --- a/treeherder/push_health/tests.py +++ b/treeherder/push_health/tests.py @@ -109,13 +109,6 @@ def get_current_test_failures(push, option_map, jobs, investigatedTests=None): test_key = re.sub( r'\W+', '', 't{}{}{}{}{}'.format(test_name, config, platform, job_name, job_group) ) - countPassed = len(list(filter(lambda x: x['result'] == 'success', jobs[job_name]))) - passFailRatio = ( - countPassed / countPassed - + len(list(filter(lambda x: x['result'] == 'testfailed', jobs[job_name]))) - if countPassed - else 0 - ) isClassifiedIntermittent = any( job['failure_classification_id'] == 4 for job in jobs[job_name] ) @@ -146,13 +139,19 @@ def get_current_test_failures(push, option_map, jobs, investigatedTests=None): 'suggestedClassification': 'New Failure', 'confidence': 0, 'tier': job.tier, + 'totalFailures': 0, + 'totalJobs': 0, 'failedInParent': False, - 'passFailRatio': passFailRatio, 'isClassifiedIntermittent': isClassifiedIntermittent, 'isInvestigated': isInvestigated, 'investigatedTestId': investigatedTestId, } tests[test_key] = line + countJobs = len( + list(filter(lambda x: x['result'] in ['success', 'testfailed'], jobs[job_name])) + ) + tests[test_key]['totalFailures'] += 1 + tests[test_key]['totalJobs'] = countJobs # Each line of the sorted list that is returned here represents one test file per platform/ # config. Each line will have at least one failing job, but may have several