Bug 1521157 - Have Push Health prototype return live data (#4787)

This has some of the logic for detecting intermittents.  But it is
still a work in progress.  It only does some of the detection
so far.

tests/push_health/test_classification.py

@@ -0,0 +1,46 @@
+import pytest
+
+from treeherder.push_health.classification import set_classifications
+
+
+def test_intermittent_win7_reftest():
+    """test that a failed test is classified as infra"""
+    failures = [
+        {
+            'testName': 'foo',
+            'failureLines': [],
+            'jobName': 'Foodebug-reftest',
+            'platform': 'windows7-32',
+            'suggestedClassification': 'New Failure',
+            'config': 'foo',
+        }
+    ]
+    set_classifications(failures, {}, {})
+
+    assert failures[0]['suggestedClassification'] == 'Intermittent'
+
+
+@pytest.mark.parametrize(('history', 'confidence', 'classification'), [
+    ({'foo': {'bing': {'baz': 2}}}, 100, 'Intermittent'),
+    ({'foo': {'bing': {'bee': 2}}}, 75, 'Intermittent'),
+    ({'foo': {'bee': {'bee': 2}}}, 50, 'Intermittent'),
+    ({'fee': {'bee': {'bee': 2}}}, 0, 'New Failure'),
+])
+def test_intermittent_confidence(history, confidence, classification):
+    """test that a failed test is classified as intermittent, confidence 100"""
+    failures = [
+        {
+            'testName': 'foo',
+            'failureLines': [],
+            'jobName': 'bar',
+            'platform': 'bing',
+            'suggestedClassification': 'New Failure',
+            'config': 'baz',
+            'confidence': 0,
+        }
+    ]
+
+    set_classifications(failures, history, {})
+
+    assert failures[0]['suggestedClassification'] == classification
+    assert failures[0]['confidence'] == confidence

treeherder/config/settings.py

@@ -211,7 +211,7 @@ LOGGING = {
         'kombu': {
             'handlers': ['console'],
             'level': 'WARNING',
-        }
+        },
     }
 }
 

treeherder/push_health/classification.py

@@ -0,0 +1,56 @@
+def set_classifications(failures, intermittent_history, fixed_by_commit_history):
+    for failure in failures:
+        if set_intermittent(failure, intermittent_history):
+            continue
+        if set_previous_regression(failure, fixed_by_commit_history):
+            continue
+
+
+def set_previous_regression(failure, fixed_by_commit_history):
+    # Not perfect, could have intermittent that is cause of fbc
+    if failure['testName'] in fixed_by_commit_history.keys():
+        failure['suggestedClassification'] = 'previousregression'
+        return True
+    return False
+
+
+def set_intermittent(failure, previous_failures):
+    # Not clear if we need these TODO items or not:
+    # TODO: if there is >1 failure for platforms/config, increase pct
+    # TODO: if >1 failures in the same dir or platform, increase pct
+
+    name = failure['testName']
+    platform = failure['platform']
+    config = failure['config']
+    job_name = failure['jobName']
+
+    confidence = 0
+    if name in previous_failures:
+        confidence = 50
+        if platform in previous_failures[name]:
+            confidence = 75
+            if config in previous_failures[name][platform]:
+                confidence = 100
+
+    # TODO: how many unique regression in win7*reftest*
+    # Marking all win7 reftest failures as int, too many font issues
+    if confidence == 0 and platform == 'windows7-32' and (
+        'opt-reftest' in job_name or 'debug-reftest' in job_name
+    ):
+        confidence = 50
+
+    if confidence:
+        failure['confidence'] = confidence
+        failure['suggestedClassification'] = 'Intermittent'
+        return True
+    return False
+
+
+def get_log_lines(failure):
+    messages = []
+    for line in failure['logLines']:
+        line = line.encode('ascii', 'ignore')
+        parts = line.split(b'|')
+        if len(parts) == 3:
+            messages.append(parts[2].strip())
+    return messages

treeherder/push_health/filter.py

@@ -0,0 +1,19 @@
+def filter_failure(failure):
+    # TODO: Add multiple filters, as needed
+    filters = [
+        filter_job_type_names
+    ]
+
+    for test_filter in filters:
+        if not test_filter(failure):
+            return False
+    return True
+
+
+def filter_job_type_names(failure):
+    name = failure['jobName']
+
+    return (
+        not name.startswith(('build', 'repackage', 'hazard', 'valgrind', 'spidermonkey'))
+        and 'test-verify' not in name
+    )

treeherder/push_health/push_health.py

@@ -0,0 +1,112 @@
+import datetime
+from collections import defaultdict
+
+from treeherder.model.models import (FailureLine,
+                                     OptionCollection,
+                                     Repository)
+from treeherder.push_health.classification import set_classifications
+from treeherder.push_health.filter import filter_failure
+from treeherder.push_health.utils import clean_test
+
+intermittent_history_days = 14
+
+
+def get_intermittent_history(prior_day, days, option_map):
+    start_date = datetime.datetime.now() - datetime.timedelta(days=days)
+    repos = Repository.objects.filter(name__in=['mozilla-inbound', 'autoland', 'mozilla-central'])
+    failure_lines = FailureLine.objects.filter(
+        job_log__job__result='testfailed',
+        job_log__job__tier=1,
+        job_log__job__failure_classification_id=4,
+        job_log__job__push__repository__in=repos,
+        job_log__job__push__time__gt=start_date,
+        job_log__job__push__time__lt=prior_day,
+    ).exclude(
+        test=None
+    ).select_related(
+        'job_log__job__machine_platform', 'job_log__job__push'
+    ).values(
+        'test',
+        'job_log__job__machine_platform__platform',
+        'job_log__job__option_collection_hash'
+    ).distinct()
+
+    previous_failures = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
+    for line in failure_lines:
+        previous_failures[
+            clean_test(line['test'])
+        ][
+            line['job_log__job__machine_platform__platform']
+        ][
+            option_map[line['job_log__job__option_collection_hash']]
+        ] += 1
+
+    return previous_failures
+
+
+def get_push_failures(push, option_map):
+    # Using .distinct(<fields>) here would help by removing duplicate FailureLines
+    # for the same job (with different sub-tests), but it's only supported by
+    # postgres.  Just using .distinct() has no effect.
+    new_failure_lines = FailureLine.objects.filter(
+        action='test_result',
+        job_log__job__push=push,
+        job_log__job__result='testfailed',
+        job_log__job__tier=1
+    ).exclude(
+        test=None
+    ).select_related(
+        'job_log__job__job_type', 'job_log__job__machine_platform'
+    ).prefetch_related(
+        'job_log__job__text_log_step__errors'
+    )
+
+    # using a dict here to avoid duplicates due to multiple failure_lines for
+    # each job.
+    tests = {}
+    for failure_line in new_failure_lines:
+        test_name = clean_test(failure_line.test)
+        test_key = '{}{}'.format(test_name, failure_line.job_guid)
+        if test_name and test_key not in tests:
+            job = failure_line.job_log.job
+            config = option_map[job.option_collection_hash]
+            errors = []
+            for step in failure_line.job_log.job.text_log_step.all():
+                for error in step.errors.all():
+                    if len(errors) < 5:
+                        errors.append(error.line)
+            line = {
+                'testName': test_name,
+                'logLines': errors,
+                'jobName': job.job_type.name,
+                'jobId': job.id,
+                'jobClassificationId': job.failure_classification_id,
+                'platform': job.machine_platform.platform,
+                'suggestedClassification': 'New Failure',
+                'config': config,
+                'key': test_key,
+                'confidence': 0,
+            }
+            tests[test_key] = line
+    return sorted(tests.values(), key=lambda k: k['testName'])
+
+
+def get_push_health_test_failures(push):
+    # query for jobs for the last two weeks excluding today
+    # find tests that have failed in the last 14 days
+    # this is very cache-able for reuse on other pushes.
+    option_map = OptionCollection.objects.get_option_collection_map()
+    start_date = push.time.date() - datetime.timedelta(days=1)
+    intermittent_history = get_intermittent_history(start_date, intermittent_history_days, option_map)
+    push_failures = get_push_failures(push, option_map)
+    # push_failures = []
+    filtered_push_failures = [
+        failure for failure in push_failures if filter_failure(failure)
+    ]
+
+    set_classifications(
+        filtered_push_failures,
+        intermittent_history,
+        {},  # TODO: Use fbc history
+    )
+    return filtered_push_failures

treeherder/push_health/utils.py

@@ -0,0 +1,60 @@
+def clean_test(test_name):
+    try:
+        clean_name = str(test_name)
+    except UnicodeEncodeError:
+        return ''
+
+    if clean_name.startswith('pid:'):
+        return None
+
+    if ' == ' in clean_name or ' != ' in clean_name:
+        if ' != ' in clean_name:
+            left, right = clean_name.split(' != ')
+        elif ' == ' in clean_name:
+            left, right = clean_name.split(' == ')
+
+        if 'tests/layout/' in left and 'tests/layout/' in right:
+            left = 'layout%s' % left.split('tests/layout')[1]
+            right = 'layout%s' % right.split('tests/layout')[1]
+        elif 'build/tests/reftest/tests/' in left and \
+             'build/tests/reftest/tests/' in right:
+            left = '%s' % left.split('build/tests/reftest/tests/')[1]
+            right = '%s' % right.split('build/tests/reftest/tests/')[1]
+        elif clean_name.startswith('http://10.0'):
+            left = '/tests/'.join(left.split('/tests/')[1:])
+            right = '/tests/'.join(right.split('/tests/')[1:])
+        clean_name = "%s == %s" % (left, right)
+
+    if 'build/tests/reftest/tests/' in clean_name:
+        clean_name = clean_name.split('build/tests/reftest/tests/')[1]
+
+    if 'jsreftest.html' in clean_name:
+        clean_name = clean_name.split('test=')[1]
+
+    if clean_name.startswith('http://10.0'):
+        clean_name = '/tests/'.join(clean_name.split('/tests/')[1:])
+
+    # http://localhost:50462/1545303666006/4/41276-1.html
+    if clean_name.startswith('http://localhost:'):
+        parts = clean_name.split('/')
+        clean_name = parts[-1]
+
+    if " (finished)" in clean_name:
+        clean_name = clean_name.split(" (finished)")[0]
+
+    # TODO: does this affect anything?
+    if clean_name in ['Main app process exited normally',
+                      None,
+                      'Last test finished',
+                      '(SimpleTest/TestRunner.js)']:
+        return None
+
+    clean_name = clean_name.strip()
+    clean_name = clean_name.replace('\\', '/')
+    clean_name = clean_name.lstrip('/')
+    return clean_name
+
+
+def is_valid_failure_line(line):
+    skip_lines = ['Return code:', 'unexpected status', 'unexpected crashes', 'exit status', 'Finished in']
+    return not any(skip_line in line for skip_line in skip_lines)

treeherder/webapp/api/push.py

@@ -9,6 +9,7 @@ from rest_framework.status import (HTTP_400_BAD_REQUEST,
 
 from treeherder.model.models import (Push,
                                      Repository)
+from treeherder.push_health.push_health import get_push_health_test_failures
 from treeherder.webapp.api.serializers import PushSerializer
 from treeherder.webapp.api.utils import (to_datetime,
                                          to_timestamp)
@@ -196,8 +197,6 @@ class PushViewSet(viewsets.ViewSet):
     def health(self, request, project):
         """
         Return a calculated assessment of the health of this push.
-
-        TODO: Replace this static dummy data with real data.
         """
         revision = request.query_params.get('revision')
 
@@ -206,6 +205,8 @@ class PushViewSet(viewsets.ViewSet):
         except Push.DoesNotExist:
             return Response("No push with revision: {0}".format(revision),
                             status=HTTP_404_NOT_FOUND)
+        push_health_test_failures = get_push_health_test_failures(push)
+
         return Response({
             'revision': revision,
             'id': push.id,
@@ -227,31 +228,8 @@ class PushViewSet(viewsets.ViewSet):
                     'name': 'Tests',
                     'result': 'fail',
                     'value': 2,
-                    'failures': [
-                        {
-                            'testName': 'dom/tests/mochitest/fetch/test_fetch_cors_sw_reroute.html',
-                            'jobName': 'test-linux32/opt-mochitest-browser-chrome-e10s-4',
-                            'jobId': 223458405,
-                            'classification': 'intermittent',
-                            'failureLine':
-                                'REFTEST TEST-UNEXPECTED-FAIL | file:///builds/worker/workspace/build/tests/reftest/tests/layout/reftests/border-dotted/border-dashed-no-radius.html == file:///builds/worker/workspace/build/tests/reftest/tests/layout/reftests/border-dotted/masked.html | image comparison, max difference: 255, number of differing pixels: 54468',
-                            'confidence': 3,
-                        },
-                        {
-                            'testName':
-                                'browser/components/extensions/test/browser/test-oop-extensions/browser_ext_pageAction_context.js',
-                            'jobName': 'test-linux64/debug-mochitest-plain-headless-e10s-8',
-                            'jobId': 223458405,
-                            'classification': 'intermittent',
-                            'failureLine':
-                                "raptor-main TEST-UNEXPECTED-FAIL: test 'raptor-tp6-bing-firefox' timed out loading test page: https://www.bing.com/search?q=barack+obama",
-                            'confidence': 4,
-                        },
-                    ],
-                    'details': [
-                        'Ran some tests that did not go so well',
-                        'See [foo.bar.baz/mongo/rational/fee]',
-                    ],
+                    'failures': push_health_test_failures,
+                    'details': [],
                 },
                 {
                     'name': 'Coverage',

ui/push-health/Metric.jsx

@@ -66,7 +66,7 @@ export default class Metric extends React.PureComponent {
                   {failures &&
                     failures.map(failure => (
                       <TestFailure
-                        key={failure.testName}
+                        key={failure.key}
                         failure={failure}
                         repo={repo}
                         revision={revision}

ui/push-health/Navigation.jsx

@@ -16,7 +16,7 @@ export default class Navigation extends React.PureComponent {
           title="This data is for UI prototyping purposes only"
           className="text-white"
         >
-          [---FAKE-DATA---]
+          [---PROTOTYPE---]
         </span>
         <Login user={user} setUser={setUser} />
       </Navbar>

ui/push-health/TestFailure.jsx

@@ -14,8 +14,9 @@ export default class TestFailure extends React.PureComponent {
       jobName,
       jobId,
       classification,
-      failureLine,
+      logLines,
       confidence,
+      config,
     } = failure;
 
     return (
@@ -31,17 +32,25 @@ export default class TestFailure extends React.PureComponent {
         </Row>
         <div className="small">
           <a
-            className="text-dark ml-3"
+            className="text-dark ml-3 px-1 border border-secondary rounded"
             href={getJobsUrl({ selectedJob: jobId, repo, revision })}
           >
             {jobName}
           </a>
+          <span className="ml-1">{config},</span>
           <span className="ml-1">
-            <FontAwesomeIcon icon={faStar} />
-            {classification}
+            {classification !== 'not classified' && (
+              <FontAwesomeIcon icon={faStar} />
+            )}
+            <span className="ml-1">{classification}</span>
           </span>
         </div>
-        <Row className="small text-monospace mt-2 ml-3">{failureLine}</Row>
+        {!!logLines.length &&
+          logLines.map(logLine => (
+            <Row className="small text-monospace mt-2 ml-3" key={logLine}>
+              {logLine}
+            </Row>
+          ))}
       </Col>
     );
   }