mozilla
/
treeherder
зеркало из https://github.com/mozilla/treeherder.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1689610 - replace passFailRatio with totalFailures/totalJobs (#7001) 

* Bug 1684931 - include manifest name in result_groups

* Bug 1689610 - replace passFailRatio with totalFailures/totalJobs
This commit is contained in:
Joel Maher 2021-02-03 17:55:21 -05:00 коммит произвёл GitHub
Родитель d0834c5a75
Коммит 3217de6350
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
3 изменённых файлов: 124 добавлений и 66 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

171
tests/ui/mock/push_health.json
Просмотреть файл

@ -754,7 +754,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -771,7 +772,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -788,7 +790,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -805,7 +808,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -822,7 +826,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -839,7 +844,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -856,7 +862,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -873,7 +880,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -890,7 +898,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -907,7 +916,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -924,7 +934,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -941,7 +952,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -958,7 +970,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -975,7 +988,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -992,7 +1006,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -1009,7 +1024,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
}, },
{ {
@ -1026,7 +1042,8 @@
"suggestedClassification": "New Failure", "suggestedClassification": "New Failure",
"confidence": 0, "confidence": 0,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": false "isClassifiedIntermittent": false
} }
], ],
@ -1045,7 +1062,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0.75, "totalFailures": 3,
"totalJobs": 4,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1062,7 +1080,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1079,7 +1098,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1096,7 +1116,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1113,7 +1134,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1130,7 +1152,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1147,7 +1170,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0.75, "totalFailures": 3,
"totalJobs": 4,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1164,7 +1188,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0.75, "totalFailures": 3,
"totalJobs": 4,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1181,7 +1206,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0.5, "totalFailures": 2,
"totalJobs": 4,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1198,7 +1224,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1215,7 +1242,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1232,7 +1260,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1249,7 +1278,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1266,7 +1296,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1283,7 +1314,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1300,7 +1332,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1317,7 +1350,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1334,7 +1368,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1351,7 +1386,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1368,7 +1404,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1385,7 +1422,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1402,7 +1440,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1419,7 +1458,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1436,7 +1476,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1453,7 +1494,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1470,7 +1512,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0.8333333333333334, "totalFailures": 8,
"totalJobs": 10,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1487,7 +1530,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1504,7 +1548,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1521,7 +1566,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1538,7 +1584,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1555,7 +1602,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1572,7 +1620,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1589,7 +1638,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1606,7 +1656,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1623,7 +1674,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1640,7 +1692,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0.75, "totalFailures": 3,
"totalJobs": 4,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1657,7 +1710,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 2, "tier": 2,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1674,7 +1728,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0.6666666666666666, "totalFailures": 2,
"totalJobs": 3,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1691,7 +1746,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0.6666666666666666, "totalFailures": 2,
"totalJobs": 3,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
}, },
{ {
@ -1708,7 +1764,8 @@
"suggestedClassification": "intermittent", "suggestedClassification": "intermittent",
"confidence": 100, "confidence": 100,
"tier": 1, "tier": 1,
"passFailRatio": 0, "totalFailures": 1,
"totalJobs": 1,
"isClassifiedIntermittent": true "isClassifiedIntermittent": true
} }
] ]

4
treeherder/push_health/classification.py
Просмотреть файл

@ -76,7 +76,9 @@ def get_grouped(failures):
for failure in failures: for failure in failures:
is_intermittent = failure['suggestedClassification'] == 'intermittent' is_intermittent = failure['suggestedClassification'] == 'intermittent'
if (is_intermittent and failure['confidence'] == 100) or failure['passFailRatio'] > 0.5: if (is_intermittent and failure['confidence'] == 100) or failure['totalFailures'] / failure[
'totalJobs'
] <= 0.5:
classified[KNOWN_ISSUES].append(failure) classified[KNOWN_ISSUES].append(failure)
else: else:
classified[NEED_INVESTIGATION].append(failure) classified[NEED_INVESTIGATION].append(failure)

15
treeherder/push_health/tests.py
Просмотреть файл

@ -109,13 +109,6 @@ def get_current_test_failures(push, option_map, jobs, investigatedTests=None):
test_key = re.sub( test_key = re.sub(
r'\W+', '', 't{}{}{}{}{}'.format(test_name, config, platform, job_name, job_group) r'\W+', '', 't{}{}{}{}{}'.format(test_name, config, platform, job_name, job_group)
) )
countPassed = len(list(filter(lambda x: x['result'] == 'success', jobs[job_name])))
passFailRatio = (
countPassed / countPassed
+ len(list(filter(lambda x: x['result'] == 'testfailed', jobs[job_name])))
if countPassed
else 0
)
isClassifiedIntermittent = any( isClassifiedIntermittent = any(
job['failure_classification_id'] == 4 for job in jobs[job_name] job['failure_classification_id'] == 4 for job in jobs[job_name]
) )
@ -146,13 +139,19 @@ def get_current_test_failures(push, option_map, jobs, investigatedTests=None):
'suggestedClassification': 'New Failure', 'suggestedClassification': 'New Failure',
'confidence': 0, 'confidence': 0,
'tier': job.tier, 'tier': job.tier,
'totalFailures': 0,
'totalJobs': 0,
'failedInParent': False, 'failedInParent': False,
'passFailRatio': passFailRatio,
'isClassifiedIntermittent': isClassifiedIntermittent, 'isClassifiedIntermittent': isClassifiedIntermittent,
'isInvestigated': isInvestigated, 'isInvestigated': isInvestigated,
'investigatedTestId': investigatedTestId, 'investigatedTestId': investigatedTestId,
} }
tests[test_key] = line tests[test_key] = line
countJobs = len(
list(filter(lambda x: x['result'] in ['success', 'testfailed'], jobs[job_name]))
)
tests[test_key]['totalFailures'] += 1
tests[test_key]['totalJobs'] = countJobs
# Each line of the sorted list that is returned here represents one test file per platform/ # Each line of the sorted list that is returned here represents one test file per platform/
# config. Each line will have at least one failing job, but may have several # config. Each line will have at least one failing job, but may have several