Bug 1312575 - Duplicate autoclassification information across TextLogError and FailureLine (#2178)

This adds a FailureLine foreign key to TextLogError and duplicates the
best_classification and best_is_verified columns. Classifications are
still set on FailureLine and copied to the corresponding column on
TextLogError.

This is a prelude to future work which will remove classifications from
FailureLine altogether, so that all autoclassification data can be
retrieved from TextLogError.

tests/autoclassify/test_classify_failures.py

@@ -13,7 +13,7 @@ from treeherder.model.models import (BugJobMap,
                                      FailureMatch,
                                      JobNote,
                                      TextLogError,
-                                     TextLogStep)
+                                     TextLogErrorMetadata)
 
 from .utils import (crash_line,
                     create_failure_lines,
@@ -37,45 +37,69 @@ def do_autoclassify(job, test_failure_lines, matchers, status="testfailed"):
         item.refresh_from_db()
 
 
-def test_classify_test_failure(failure_lines, classified_failures,
+def create_lines(test_job, lines):
+    error_lines = create_text_log_errors(test_job, lines)
+    failure_lines = create_failure_lines(test_job, lines)
+
+    for error_line, failure_line in zip(error_lines, failure_lines):
+        TextLogErrorMetadata.objects.create(text_log_error=error_line,
+                                            failure_line=failure_line)
+    return error_lines, failure_lines
+
+
+def test_classify_test_failure(text_log_errors_failure_lines,
+                               classified_failures,
                                test_job_2):
-    test_failure_lines = create_failure_lines(test_job_2,
-                                              [(test_line, {}),
-                                               (test_line, {"subtest": "subtest2"}),
-                                               (test_line, {"status": "TIMEOUT"}),
-                                               (test_line, {"expected": "ERROR"}),
-                                               (test_line, {"message": "message2"})])
+    # Ensure that running autoclassify on a new job classifies lines that
+    # exactly match previous classifications
+
+    # The first two lines match classified failures created in teh fixtures
+    lines = [(test_line, {}),
+             (test_line, {"subtest": "subtest2"}),
+             (test_line, {"status": "TIMEOUT"}),
+             (test_line, {"expected": "ERROR"}),
+             (test_line, {"message": "message2"})]
+    test_error_lines, test_failure_lines = create_lines(test_job_2, lines)
 
     do_autoclassify(test_job_2, test_failure_lines, [PreciseTestMatcher])
 
-    expected_classified = test_failure_lines[:2]
-    expected_unclassified = test_failure_lines[2:]
+    expected_classified = test_error_lines[:2], test_failure_lines[:2]
+    expected_unclassified = test_error_lines[2:], test_failure_lines[2:]
 
-    for actual, expected in zip(expected_classified, classified_failures):
-        assert [item.id for item in actual.classified_failures.all()] == [expected.id]
+    for (error_line, failure_line), expected in zip(zip(*expected_classified),
+                                                    classified_failures):
+        assert list(error_line.classified_failures.values_list('id', flat=True)) == [expected.id]
+        assert list(failure_line.classified_failures.values_list('id', flat=True)) == [expected.id]
 
-    for item in expected_unclassified:
-        assert item.classified_failures.count() == 0
+    for error_line, failure_line in zip(*expected_unclassified):
+        assert error_line.classified_failures.count() == 0
+        assert failure_line.classified_failures.count() == 0
 
 
-def test_no_autoclassify_job_success(failure_lines, classified_failures, test_job_2):
-    test_failure_lines = create_failure_lines(test_job_2,
-                                              [(test_line, {}),
-                                               (test_line, {"subtest": "subtest2"}),
-                                               (test_line, {"status": "TIMEOUT"}),
-                                               (test_line, {"expected": "ERROR"}),
-                                               (test_line, {"message": "message2"})])
+def test_no_autoclassify_job_success(text_log_errors_failure_lines,
+                                     classified_failures,
+                                     test_job_2):
+    # Ensure autoclassification doesn't occur for successful jobs
+    lines = [(test_line, {}),
+             (test_line, {"subtest": "subtest2"}),
+             (test_line, {"status": "TIMEOUT"}),
+             (test_line, {"expected": "ERROR"}),
+             (test_line, {"message": "message2"})]
+    test_error_lines, test_failure_lines = create_lines(test_job_2, lines)
 
     do_autoclassify(test_job_2, test_failure_lines, [PreciseTestMatcher], status="success")
 
-    expected_classified = []
-    expected_unclassified = test_failure_lines
+    expected_classified = [], []
+    expected_unclassified = test_error_lines, test_failure_lines
 
-    for actual, expected in zip(expected_classified, classified_failures):
-        assert [item.id for item in actual.classified_failures.all()] == [expected.id]
+    for (error_line, failure_line), expected in zip(zip(*expected_classified),
+                                                    classified_failures):
+        assert list(error_line.classified_failures.values_list('id', flat=True)) == [expected.id]
+        assert list(failure_line.classified_failures.values_list('id', flat=True)) == [expected.id]
 
-    for item in expected_unclassified:
-        assert item.classified_failures.count() == 0
+    for error_line, failure_line in zip(*expected_unclassified):
+        assert error_line.classified_failures.count() == 0
+        assert failure_line.classified_failures.count() == 0
 
 
 def test_autoclassify_update_job_classification(failure_lines, classified_failures,
@@ -84,10 +108,9 @@ def test_autoclassify_update_job_classification(failure_lines, classified_failur
         item.bug_number = "1234%i" % i
         item.save()
 
-    test_failure_lines = create_failure_lines(test_job_2,
-                                              [(test_line, {})])
+    lines = [(test_line, {})]
+    test_error_lines, test_failure_lines = create_lines(test_job_2, lines)
 
-    create_text_log_errors(test_job_2, [(test_line, {})])
     do_autoclassify(test_job_2, test_failure_lines, [PreciseTestMatcher])
 
     assert JobNote.objects.filter(job=test_job_2).count() == 1
@@ -100,20 +123,11 @@ def test_autoclassify_no_update_job_classification(test_job, test_job_2,
                                                    failure_lines,
                                                    classified_failures):
 
-    test_failure_lines = create_failure_lines(test_job, [(test_line, {})])
-    step = TextLogStep.objects.create(job=test_job_2,
-                                      name='unnamed step',
-                                      started_line_number=1,
-                                      finished_line_number=10,
-                                      result=TextLogStep.TEST_FAILED)
-    TextLogError.objects.create(step=step,
-                                line='TEST-UNEXPECTED-FAIL | test1 | message1',
-                                line_number=1)
-    TextLogError.objects.create(step=step,
+    lines = [(test_line, {})]
+    test_error_lines, test_failure_lines = create_lines(test_job_2, lines)
+    TextLogError.objects.create(step=test_error_lines[0].step,
                                 line="Some error that isn't in the structured logs",
                                 line_number=2)
-    test_failure_lines = create_failure_lines(test_job_2,
-                                              [(test_line, {})])
 
     do_autoclassify(test_job_2, test_failure_lines, [PreciseTestMatcher])
 
@@ -124,19 +138,24 @@ def test_autoclassified_after_manual_classification(test_user, test_job_2,
                                                     failure_lines, failure_classifications):
     register_detectors(ManualDetector, TestFailureDetector)
 
-    create_text_log_errors(test_job_2, [(test_line, {})])
-    test_failure_lines = create_failure_lines(test_job_2, [(test_line, {})])
+    lines = [(test_line, {})]
+    test_error_lines, test_failure_lines = create_lines(test_job_2, lines)
 
     JobNote.objects.create(job=test_job_2,
                            failure_classification_id=4,
                            user=test_user,
                            text="")
 
-    for item in test_failure_lines:
-        item.refresh_from_db()
+    for error_line, failure_line in zip(test_error_lines, test_failure_lines):
+        error_line.refresh_from_db()
+        error_line.metadata.refresh_from_db()
+        failure_line.refresh_from_db()
 
+    assert len(test_error_lines[0].matches.all()) == 1
     assert len(test_failure_lines[0].matches.all()) == 1
+    assert test_error_lines[0].metadata.best_classification == test_error_lines[0].classified_failures.all()[0]
     assert test_failure_lines[0].best_classification == test_failure_lines[0].classified_failures.all()[0]
+    assert test_error_lines[0].metadata.best_is_verified
     assert test_failure_lines[0].best_is_verified
 
 
@@ -146,16 +165,19 @@ def test_autoclassified_no_update_after_manual_classification_1(test_job_2,
     register_detectors(ManualDetector, TestFailureDetector)
 
     # Line type won't be detected by the detectors we have registered
-    test_failure_lines = create_failure_lines(test_job_2, [(log_line, {})])
+    lines = [(log_line, {})]
+    test_error_lines, test_failure_lines = create_lines(test_job_2, lines)
 
     JobNote.objects.create(job=test_job_2,
                            failure_classification_id=4,
                            user=test_user,
                            text="")
 
-    for item in test_failure_lines:
-        item.refresh_from_db()
+    for error_line, failure_line in zip(test_error_lines, test_failure_lines):
+        error_line.refresh_from_db()
+        failure_line.refresh_from_db()
 
+    assert len(test_error_lines[0].matches.all()) == 0
     assert len(test_failure_lines[0].matches.all()) == 0
 
 
@@ -179,9 +201,15 @@ def test_autoclassified_no_update_after_manual_classification_2(test_user, test_
     assert len(test_failure_lines[0].matches.all()) == 0
 
 
-def test_classify_skip_ignore(test_job_2, failure_lines,
+def test_classify_skip_ignore(test_job_2,
+                              text_log_errors_failure_lines,
                               classified_failures):
 
+    text_log_errors, failure_lines = text_log_errors_failure_lines
+    text_log_errors[1].metadata.best_is_verified = True
+    text_log_errors[1].metadata.best_classification = None
+    text_log_errors[1].metadata.save()
+
     failure_lines[1].best_is_verified = True
     failure_lines[1].best_classification = None
     failure_lines[1].save()
@@ -236,12 +264,12 @@ def test_classify_multiple(test_job_2, failure_lines, classified_failures):
                                                      ElasticSearchTestMatcher])
 
     for actual, expected in zip(expected_classified_precise, classified_failures):
-        assert [item.id for item in actual.classified_failures.all()] == [expected.id]
-        assert [item.matcher.id == 1 for item in item.matches.all()]
+        assert list(actual.classified_failures.values_list('id', flat=True)) == [expected.id]
+        assert [item.matcher.id == 1 for item in actual.matches.all()]
 
     for actual, expected in zip(expected_classified_fuzzy, classified_failures):
-        assert [item.id for item in actual.classified_failures.all()] == [expected.id]
-        assert [item.matcher.id == 2 for item in item.matches.all()]
+        assert list(actual.classified_failures.values_list('id', flat=True)) == [expected.id]
+        assert [item.matcher.id == 2 for item in actual.matches.all()]
 
 
 def test_classify_crash(test_repository, test_job, test_job_2, test_matcher):
@@ -265,7 +293,7 @@ def test_classify_crash(test_repository, test_job, test_job_2, test_matcher):
     expected_unclassified = failure_lines[2:]
 
     for actual in expected_classified:
-        assert [item.id for item in actual.classified_failures.all()] == [classified_failure.id]
+        assert list(actual.classified_failures.values_list('id', flat=True)) == [classified_failure.id]
 
     for item in expected_unclassified:
         assert item.classified_failures.count() == 0

tests/autoclassify/utils.py

@@ -44,7 +44,7 @@ def get_data(base_data, updates):
         else:
             data["action"] = "test_end"
     elif data["action"] == "log":
-        if data["level"] not in ("error", "critical"):
+        if data["level"] not in ("ERROR", "CRITICAL"):
             return
     else:
         return

tests/conftest.py

@@ -16,7 +16,8 @@ from treeherder.config.wsgi import application
 from treeherder.etl.jobs import store_job_data
 from treeherder.etl.resultset import store_result_set_data
 from treeherder.model.models import (JobNote,
-                                     Push)
+                                     Push,
+                                     TextLogErrorMetadata)
 
 
 def pytest_addoption(parser):
@@ -345,6 +346,22 @@ def failure_classifications(transactional_db):
         FailureClassification(name=name).save()
 
 
+@pytest.fixture
+def text_log_errors_failure_lines(test_job, failure_lines):
+    from tests.autoclassify.utils import test_line, create_text_log_errors
+
+    lines = [(test_line, {}),
+             (test_line, {"subtest": "subtest2"})]
+
+    text_log_errors = create_text_log_errors(test_job, lines)
+
+    for error_line, failure_line in zip(text_log_errors, failure_lines):
+        TextLogErrorMetadata.objects.create(text_log_error=error_line,
+                                            failure_line=failure_line)
+
+    return text_log_errors, failure_lines
+
+
 @pytest.fixture
 def test_matcher(request):
     from treeherder.autoclassify import detectors
@@ -367,11 +384,13 @@ def test_matcher(request):
 
 
 @pytest.fixture
-def classified_failures(test_job, failure_lines, test_matcher,
+def classified_failures(test_job, text_log_errors_failure_lines, test_matcher,
                         failure_classifications):
     from treeherder.model.models import ClassifiedFailure
     from treeherder.model.search import refresh_all
 
+    _, failure_lines = text_log_errors_failure_lines
+
     classified_failures = []
 
     for failure_line in failure_lines:

tests/e2e/test_client_job_ingestion.py

@@ -376,11 +376,12 @@ def test_post_job_artifacts_by_add_artifact(
     }
 
     assert TextLogError.objects.count() == 1
-    assert model_to_dict(TextLogError.objects.get(step__job__guid=job_guid)) == {
+    text_log_error = TextLogError.objects.get(step__job__guid=job_guid)
+    assert model_to_dict(text_log_error) == {
         'id': 1,
         'line': 'TEST_UNEXPECTED_FAIL | /sdcard/tests/autophone/s1s2test/nytimes.com/index.html | Failed to get uncached measurement.',
         'line_number': 64435,
-        'step': 1
+        'step': 1,
     }
 
     # assert that some bug suggestions got generated

tests/log_parser/test_crossreference_error_lines.py

@@ -1,6 +1,7 @@
 from django.core.management import call_command
 
 from treeherder.model.models import (FailureLine,
+                                     TextLogError,
                                      TextLogSummary,
                                      TextLogSummaryLine)
 
@@ -21,6 +22,7 @@ def test_crossreference_error_lines(test_job):
 
     call_command('crossreference_error_lines', str(test_job.id))
 
+    error_lines = TextLogError.objects.filter(step__job=test_job).all()
     summary = TextLogSummary.objects.all()
     assert len(summary) == 1
     summary = summary[0]
@@ -34,12 +36,16 @@ def test_crossreference_error_lines(test_job):
     failure_lines = FailureLine.objects.all()
     assert len(failure_lines) == len(lines)
 
-    for i, (failure_line, summary_line) in enumerate(zip(failure_lines, summary_lines)):
+    for i, (failure_line, error_line, summary_line) in enumerate(
+            zip(failure_lines, error_lines, summary_lines)):
         assert summary_line.summary == summary
         assert summary_line.line_number == i
         assert summary_line.failure_line == failure_line
         assert summary_line.verified is False
         assert summary_line.bug_number is None
+        assert error_line.metadata.failure_line == failure_line
+        assert error_line.metadata.best_is_verified is False
+        assert error_line.metadata.best_classification is None
 
 
 def test_crossreference_error_lines_truncated(test_job):
@@ -75,43 +81,26 @@ def test_crossreference_error_lines_missing(test_job):
     call_command('crossreference_error_lines', str(test_job.id))
 
     failure_lines = FailureLine.objects.all()
+    error_lines = TextLogError.objects.filter(step__job=test_job).all()
     summary_lines = TextLogSummaryLine.objects.all()
     summary = TextLogSummary.objects.all()[0]
     assert len(summary_lines) == len(failure_lines) + 1
 
     summary_line = summary_lines[0]
+    error_line = error_lines[0]
     assert summary_line.summary == summary
     assert summary_line.line_number == 0
     assert summary_line.failure_line is None
     assert summary_line.verified is False
     assert summary_line.bug_number is None
 
-    for i, (failure_line, summary_line) in enumerate(zip(failure_lines, summary_lines[1:])):
+    for i, (failure_line, error_line, summary_line) in enumerate(
+            zip(failure_lines, error_lines[1:], summary_lines[1:])):
         assert summary_line.summary == summary
         assert summary_line.line_number == i + 1
         assert summary_line.failure_line == failure_line
         assert summary_line.verified is False
         assert summary_line.bug_number is None
-
-
-def test_crossreference_error_lines_repeat(test_job):
-
-    lines = [(test_line, {})]
-
-    create_failure_lines(test_job, lines)
-    create_text_log_errors(test_job, lines)
-
-    call_command('crossreference_error_lines', str(test_job.id))
-
-    failure_lines = FailureLine.objects.all()
-    summary_lines = TextLogSummaryLine.objects.all()
-    summary = TextLogSummary.objects.all()[0]
-    assert len(summary_lines) == 1
-
-    summary_line = summary_lines[0]
-    assert summary_line.summary == summary
-    assert summary_line.failure_line == failure_lines[0]
-    assert summary_line.verified is False
-    assert summary_line.bug_number is None
-
-    call_command('crossreference_error_lines', str(test_job.id))
+        assert error_line.metadata.failure_line == failure_line
+        assert error_line.metadata.best_is_verified is False
+        assert error_line.metadata.best_classification is None

tests/webapp/api/test_failureline.py

@@ -2,6 +2,7 @@ from django.core.urlresolvers import reverse
 from rest_framework.test import APIClient
 
 from tests.autoclassify.utils import (create_failure_lines,
+                                      create_text_log_errors,
                                       test_line)
 from treeherder.autoclassify.detectors import ManualDetector
 from treeherder.model.models import (BugJobMap,
@@ -9,7 +10,9 @@ from treeherder.model.models import (BugJobMap,
                                      Job,
                                      JobNote,
                                      Matcher,
-                                     MatcherManager)
+                                     MatcherManager,
+                                     TextLogError,
+                                     TextLogErrorMetadata)
 from treeherder.model.search import TestFailureLine
 
 
@@ -34,15 +37,22 @@ def test_get_failure_line(webapp, failure_lines):
     assert set(failure_line.keys()) == set(exp_failure_keys)
 
 
-def test_update_failure_line_verify(test_repository, failure_lines, classified_failures,
+def test_update_failure_line_verify(test_repository,
+                                    text_log_errors_failure_lines,
+                                    classified_failures,
                                     test_user):
 
+    text_log_errors, failure_lines = text_log_errors_failure_lines
     client = APIClient()
     client.force_authenticate(user=test_user)
 
     failure_line = failure_lines[0]
+    error_line = text_log_errors[0]
     assert failure_line.best_classification == classified_failures[0]
     assert failure_line.best_is_verified is False
+    assert error_line.metadata.failure_line == failure_line
+    assert error_line.metadata.best_classification == classified_failures[0]
+    assert error_line.metadata.best_is_verified is False
 
     body = {"project": test_repository.name,
             "best_classification": classified_failures[0].id}
@@ -54,26 +64,37 @@ def test_update_failure_line_verify(test_repository, failure_lines, classified_f
     assert resp.status_code == 200
 
     failure_line.refresh_from_db()
+    error_line.metadata.refresh_from_db()
+    error_line.refresh_from_db()
 
     assert failure_line.best_classification == classified_failures[0]
     assert failure_line.best_is_verified
+    assert error_line.metadata.best_classification == classified_failures[0]
+    assert error_line.metadata.best_is_verified
 
     es_line = TestFailureLine.get(failure_line.id, routing=failure_line.test)
     assert es_line.best_classification == classified_failures[0].id
     assert es_line.best_is_verified
 
 
-def test_update_failure_line_replace(test_repository, failure_lines,
-                                     classified_failures, test_user):
+def test_update_failure_line_replace(test_repository,
+                                     text_log_errors_failure_lines,
+                                     classified_failures,
+                                     test_user):
 
     MatcherManager.register_detector(ManualDetector)
 
     client = APIClient()
     client.force_authenticate(user=test_user)
 
+    text_log_errors, failure_lines = text_log_errors_failure_lines
     failure_line = failure_lines[0]
+    error_line = text_log_errors[0]
     assert failure_line.best_classification == classified_failures[0]
     assert failure_line.best_is_verified is False
+    assert error_line.metadata.failure_line == failure_line
+    assert error_line.metadata.best_classification == classified_failures[0]
+    assert error_line.metadata.best_is_verified is False
 
     body = {"project": test_repository.name,
             "best_classification": classified_failures[1].id}
@@ -85,20 +106,28 @@ def test_update_failure_line_replace(test_repository, failure_lines,
     assert resp.status_code == 200
 
     failure_line.refresh_from_db()
+    error_line.metadata.refresh_from_db()
+    error_line.refresh_from_db()
 
     assert failure_line.best_classification == classified_failures[1]
     assert failure_line.best_is_verified
     assert len(failure_line.classified_failures.all()) == 2
+    assert error_line.metadata.failure_line == failure_line
+    assert error_line.metadata.best_classification == classified_failures[1]
+    assert error_line.metadata.best_is_verified
 
     expected_matcher = Matcher.objects.get(name="ManualDetector")
     assert failure_line.matches.get(classified_failure_id=classified_failures[1].id).matcher == expected_matcher
+    assert error_line.matches.get(classified_failure_id=classified_failures[1].id).matcher == expected_matcher
 
 
 def test_update_failure_line_mark_job(test_repository, test_job,
-                                      failure_lines,
+                                      text_log_errors_failure_lines,
                                       classified_failures,
                                       test_user):
 
+    text_log_errors, failure_lines = text_log_errors_failure_lines
+
     MatcherManager.register_detector(ManualDetector)
 
     client = APIClient()
@@ -107,9 +136,10 @@ def test_update_failure_line_mark_job(test_repository, test_job,
     classified_failures[1].bug_number = 1234
     classified_failures[1].save()
 
-    for failure_line in failure_lines:
+    for text_log_error, failure_line in zip(text_log_errors, failure_lines):
 
         assert failure_line.best_is_verified is False
+        assert text_log_error.metadata.best_is_verified is False
 
         body = {"best_classification": classified_failures[1].id}
 
@@ -119,9 +149,13 @@ def test_update_failure_line_mark_job(test_repository, test_job,
         assert resp.status_code == 200
 
         failure_line.refresh_from_db()
+        text_log_error.refresh_from_db()
+        text_log_error.metadata.refresh_from_db()
 
         assert failure_line.best_classification == classified_failures[1]
         assert failure_line.best_is_verified
+        assert text_log_error.metadata.best_classification == classified_failures[1]
+        assert text_log_error.metadata.best_is_verified
 
     assert test_job.is_fully_verified()
 
@@ -135,9 +169,10 @@ def test_update_failure_line_mark_job(test_repository, test_job,
 
 
 def test_update_failure_line_mark_job_with_human_note(test_job,
-                                                      failure_lines,
+                                                      text_log_errors_failure_lines,
                                                       classified_failures, test_user):
 
+    text_log_errors, failure_lines = text_log_errors_failure_lines
     MatcherManager.register_detector(ManualDetector)
 
     client = APIClient()
@@ -166,11 +201,13 @@ def test_update_failure_line_mark_job_with_human_note(test_job,
 
 
 def test_update_failure_line_mark_job_with_auto_note(test_job,
-                                                     mock_autoclassify_jobs_true, test_repository,
-                                                     failure_lines,
+                                                     mock_autoclassify_jobs_true,
+                                                     test_repository,
+                                                     text_log_errors_failure_lines,
                                                      classified_failures,
                                                      test_user):
 
+    text_log_errors, failure_lines = text_log_errors_failure_lines
     MatcherManager.register_detector(ManualDetector)
 
     client = APIClient()
@@ -181,7 +218,6 @@ def test_update_failure_line_mark_job_with_auto_note(test_job,
                            text="note")
 
     for failure_line in failure_lines:
-
         body = {"best_classification": classified_failures[1].id}
 
         resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
@@ -204,7 +240,9 @@ def test_update_failure_line_mark_job_with_auto_note(test_job,
 
 
 def test_update_failure_lines(mock_autoclassify_jobs_true,
-                              test_repository, classified_failures,
+                              test_repository,
+                              text_log_errors_failure_lines,
+                              classified_failures,
                               eleven_jobs_stored,
                               test_user):
 
@@ -215,14 +253,23 @@ def test_update_failure_lines(mock_autoclassify_jobs_true,
     client = APIClient()
     client.force_authenticate(user=test_user)
 
-    create_failure_lines(jobs[1],
-                         [(test_line, {}),
-                          (test_line, {"subtest": "subtest2"})])
+    lines = [(test_line, {}),
+             (test_line, {"subtest": "subtest2"})]
+    new_failure_lines = create_failure_lines(jobs[1], lines)
+    new_text_log_errors = create_text_log_errors(jobs[1], lines)
+
+    for text_log_error, failure_line in zip(new_text_log_errors,
+                                            new_failure_lines):
+        TextLogErrorMetadata.objects.create(text_log_error=text_log_error,
+                                            failure_line=failure_line)
 
     failure_lines = FailureLine.objects.filter(
         job_guid__in=[job.guid for job in jobs]).all()
+    text_log_errors = TextLogError.objects.filter(
+        step__job__in=jobs).all()
 
-    for failure_line in failure_lines:
+    for text_log_error, failure_line in zip(text_log_errors, failure_lines):
+        assert text_log_error.metadata.best_is_verified is False
         assert failure_line.best_is_verified is False
 
     body = [{"id": failure_line.id,
@@ -232,10 +279,14 @@ def test_update_failure_lines(mock_autoclassify_jobs_true,
 
     assert resp.status_code == 200
 
-    for failure_line in failure_lines:
+    for text_log_error, failure_line in zip(text_log_errors, failure_lines):
+        text_log_error.refresh_from_db()
+        text_log_error.metadata.refresh_from_db()
         failure_line.refresh_from_db()
         assert failure_line.best_classification == classified_failures[1]
         assert failure_line.best_is_verified
+        assert text_log_error.metadata.best_classification == classified_failures[1]
+        assert text_log_error.metadata.best_is_verified
 
     for job in jobs:
         assert job.is_fully_verified()
@@ -246,15 +297,21 @@ def test_update_failure_lines(mock_autoclassify_jobs_true,
         assert note.user == test_user
 
 
-def test_update_failure_line_ignore(test_job, test_repository, failure_lines,
+def test_update_failure_line_ignore(test_job,
+                                    test_repository,
+                                    text_log_errors_failure_lines,
                                     classified_failures, test_user):
 
+    text_log_errors, failure_lines = text_log_errors_failure_lines
     client = APIClient()
     client.force_authenticate(user=test_user)
 
     MatcherManager.register_detector(ManualDetector)
 
+    text_log_error = text_log_errors[0]
     failure_line = failure_lines[0]
+    assert text_log_error.metadata.best_classification == classified_failures[0]
+    assert text_log_error.metadata.best_is_verified is False
     assert failure_line.best_classification == classified_failures[0]
     assert failure_line.best_is_verified is False
 
@@ -268,17 +325,22 @@ def test_update_failure_line_ignore(test_job, test_repository, failure_lines,
     assert resp.status_code == 200
 
     failure_line.refresh_from_db()
+    text_log_error.refresh_from_db()
+    text_log_error.metadata.refresh_from_db()
 
     assert failure_line.best_classification is None
     assert failure_line.best_is_verified
+    assert text_log_error.metadata.best_classification is None
+    assert text_log_error.metadata.best_is_verified
 
 
 def test_update_failure_line_all_ignore_mark_job(test_job,
                                                  mock_autoclassify_jobs_true,
-                                                 failure_lines,
+                                                 text_log_errors_failure_lines,
                                                  classified_failures,
                                                  test_user):
 
+    text_log_errors, failure_lines = text_log_errors_failure_lines
     MatcherManager.register_detector(ManualDetector)
 
     client = APIClient()
@@ -286,15 +348,20 @@ def test_update_failure_line_all_ignore_mark_job(test_job,
 
     job_failure_lines = [line for line in failure_lines if
                          line.job_guid == test_job.guid]
+    job_text_log_errors = [error for error in text_log_errors if
+                           error.step.job == test_job]
 
-    for failure_line in job_failure_lines:
+    for error_line, failure_line in zip(job_text_log_errors, job_failure_lines):
+        error_line.metadata.best_is_verified = False
+        error_line.metadata.best_classification = None
         failure_line.best_is_verified = False
         failure_line.best_classification = None
 
     assert JobNote.objects.count() == 0
 
-    for failure_line in job_failure_lines:
+    for error_line, failure_line in zip(job_text_log_errors, job_failure_lines):
 
+        assert error_line.metadata.best_is_verified is False
         assert failure_line.best_is_verified is False
 
         body = {"best_classification": None}
@@ -304,10 +371,14 @@ def test_update_failure_line_all_ignore_mark_job(test_job,
 
         assert resp.status_code == 200
 
+        error_line.refresh_from_db()
+        error_line.metadata.refresh_from_db()
         failure_line.refresh_from_db()
 
         assert failure_line.best_classification is None
         assert failure_line.best_is_verified
+        assert error_line.metadata.best_classification is None
+        assert error_line.metadata.best_is_verified
 
     assert test_job.is_fully_verified()
 
@@ -316,17 +387,18 @@ def test_update_failure_line_all_ignore_mark_job(test_job,
 
 def test_update_failure_line_partial_ignore_mark_job(test_job,
                                                      mock_autoclassify_jobs_true,
-                                                     failure_lines,
+                                                     text_log_errors_failure_lines,
                                                      classified_failures,
                                                      test_user):
 
+    text_log_errors, failure_lines = text_log_errors_failure_lines
     MatcherManager.register_detector(ManualDetector)
 
     client = APIClient()
     client.force_authenticate(user=test_user)
 
-    for i, failure_line in enumerate(failure_lines):
-
+    for i, (error_line, failure_line) in enumerate(zip(text_log_errors, failure_lines)):
+        assert error_line.metadata.best_is_verified is False
         assert failure_line.best_is_verified is False
 
         body = {"best_classification": None if i == 0 else classified_failures[0].id}
@@ -336,12 +408,16 @@ def test_update_failure_line_partial_ignore_mark_job(test_job,
 
         assert resp.status_code == 200
 
+        error_line.refresh_from_db()
+        error_line.metadata.refresh_from_db()
         failure_line.refresh_from_db()
 
         if i == 0:
             assert failure_line.best_classification is None
+            assert error_line.metadata.best_classification is None
         else:
             assert failure_line.best_classification == classified_failures[0]
+            assert error_line.metadata.best_classification == classified_failures[0]
         assert failure_line.best_is_verified
 
     assert test_job.is_fully_verified()

tests/webapp/api/test_jobs_api.py

@@ -432,7 +432,8 @@ def test_text_log_steps_and_errors(webapp, test_job):
                         'search': 'failure 1',
                         'search_terms': ['failure 1'],
                         'bugs': {'open_recent': [], 'all_others': []}
-                    }
+                    },
+                    'metadata': None
                 },
                 {
                     'id': 2,
@@ -442,7 +443,8 @@ def test_text_log_steps_and_errors(webapp, test_job):
                         'search': 'failure 2',
                         'search_terms': ['failure 2'],
                         'bugs': {'open_recent': [], 'all_others': []}
-                    }
+                    },
+                    'metadata': None
                 }
             ],
             'finished': '1970-01-01T00:03:20',
@@ -491,7 +493,8 @@ def test_text_log_errors(webapp, test_job):
                 'search': 'failure 1',
                 'search_terms': ['failure 1'],
                 'bugs': {'open_recent': [], 'all_others': []}
-            }
+            },
+            'metadata': None
         },
         {
             'id': 2,
@@ -501,7 +504,8 @@ def test_text_log_errors(webapp, test_job):
                 'search': 'failure 2',
                 'search_terms': ['failure 2'],
                 'bugs': {'open_recent': [], 'all_others': []}
-            }
+            },
+            'metadata': None
         }
     ]
 

treeherder/autoclassify/autoclassify.py

@@ -6,7 +6,8 @@ from django.db.utils import IntegrityError
 from treeherder.model.models import (FailureLine,
                                      FailureMatch,
                                      JobNote,
-                                     Matcher)
+                                     Matcher,
+                                     TextLogErrorMatch)
 
 logger = logging.getLogger(__name__)
 
@@ -63,6 +64,12 @@ def update_db(job, matches, all_matched):
                     matcher=matcher,
                     classified_failure=match.classified_failure,
                     failure_line=failure_line)
+                if failure_line.error:
+                    TextLogErrorMatch.objects.create(
+                        score=match.score,
+                        matcher=matcher,
+                        classified_failure=match.classified_failure,
+                        text_log_error=failure_line.error)
             except IntegrityError:
                 logger.warning(
                     "Tried to create duplicate match for failure line %i with matcher %i and classified_failure %i" %
@@ -70,7 +77,10 @@ def update_db(job, matches, all_matched):
         best_match = failure_line.best_automatic_match(AUTOCLASSIFY_CUTOFF_RATIO)
         if best_match:
             failure_line.best_classification = best_match.classified_failure
-            failure_line.save()
+            failure_line.save(update_fields=['best_classification'])
+            if failure_line.error:
+                failure_line.error.metadata.best_classification = best_match.classified_failure
+                failure_line.error.metadata.save(update_fields=['best_classification'])
 
     if all_matched:
         if job.is_fully_autoclassified():

treeherder/log_parser/crossreference.py

@@ -7,12 +7,14 @@ from mozlog.formatters.tbplformatter import TbplFormatter
 
 from treeherder.model.models import (FailureLine,
                                      TextLogError,
+                                     TextLogErrorMetadata,
                                      TextLogSummary,
                                      TextLogSummaryLine)
 
 logger = logging.getLogger(__name__)
 
 
+@transaction.atomic
 def crossreference_job(job):
     """Populate the TextLogSummary and TextLogSummaryLine tables for a
     job. Specifically this function tries to match the
@@ -63,6 +65,8 @@ def _crossreference(job):
                 summary=summary,
                 line_number=error.line_number,
                 failure_line=failure_line))
+            TextLogErrorMetadata.objects.create(text_log_error=error,
+                                                failure_line=failure_line)
             failure_line, regexp = match_iter.next()
         else:
             logger.debug("Failed to match '%s'" % (error.line,))

treeherder/model/migrations/0004_duplicate_failure_classifications.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-02-23 16:50
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('model', '0003_taskcluster_taskid_not_unique'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TextLogErrorMatch',
+            fields=[
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
+                ('score', models.DecimalField(blank=True, decimal_places=2, max_digits=3, null=True)),
+                ('classified_failure', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='error_matches', to='model.ClassifiedFailure')),
+                ('matcher', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='model.Matcher')),
+            ],
+            options={
+                'db_table': 'text_log_error_match',
+                'verbose_name_plural': 'text log error matches',
+            },
+        ),
+        migrations.CreateModel(
+            name='TextLogErrorMetadata',
+            fields=[
+                ('text_log_error', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='_metadata', serialize=False, to='model.TextLogError')),
+                ('best_is_verified', models.BooleanField(default=False)),
+                ('best_classification', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='best_for_errors', to='model.ClassifiedFailure')),
+                ('failure_line', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='text_log_error_metadata', to='model.FailureLine')),
+            ],
+            options={
+                'db_table': 'text_log_error_metadata',
+            },
+        ),
+        migrations.AddField(
+            model_name='textlogerrormatch',
+            name='text_log_error',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='matches', to='model.TextLogError'),
+        ),
+        migrations.AddField(
+            model_name='classifiedfailure',
+            name='text_log_errors',
+            field=models.ManyToManyField(related_name='classified_failures', through='model.TextLogErrorMatch', to='model.TextLogError'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='textlogerrormatch',
+            unique_together=set([('text_log_error', 'classified_failure', 'matcher')]),
+        ),
+    ]

treeherder/model/models.py

@@ -651,6 +651,19 @@ class Job(models.Model):
     objects = JobManager()
 
     id = models.BigAutoField(primary_key=True)
+
+    PENDING = 0
+    CROSSREFERENCED = 1
+    AUTOCLASSIFIED = 2
+    SKIPPED = 3
+    FAILED = 255
+
+    AUTOCLASSIFY_STATUSES = ((PENDING, 'pending'),
+                             (CROSSREFERENCED, 'crossreferenced'),
+                             (AUTOCLASSIFIED, 'autoclassified'),
+                             (SKIPPED, 'skipped'),
+                             (FAILED, 'failed'))
+
     repository = models.ForeignKey(Repository)
     guid = models.CharField(max_length=50, unique=True)
     project_specific_id = models.PositiveIntegerField(null=True)
@@ -1132,6 +1145,17 @@ class FailureLine(models.Model):
             ('job_log',  'line')
         )
 
+    def __str__(self):
+        return "{0} {1}".format(self.id, Job.objects.get(guid=self.job_guid).id)
+
+    @property
+    def error(self):
+        # Return the related text-log-error or None if there is no related field.
+        try:
+            return self.text_log_error_metadata.text_log_error
+        except TextLogErrorMetadata.DoesNotExist:
+            return None
+
     def best_automatic_match(self, min_score=0):
         return FailureMatch.objects.filter(
             failure_line_id=self.id,
@@ -1160,6 +1184,17 @@ class FailureLine(models.Model):
             if mark_best:
                 self.best_classification = classification
                 self.save(update_fields=['best_classification'])
+
+            if self.error:
+                TextLogErrorMatch.objects.create(
+                    text_log_error=self.error,
+                    classified_failure=classification,
+                    matcher=matcher,
+                    score=1)
+                if mark_best:
+                    self.error.metadata.best_classification = classification
+                    self.error.metadata.save(update_fields=['best_classification'])
+
             self.elastic_search_insert()
         return classification, new_link
 
@@ -1171,6 +1206,10 @@ class FailureLine(models.Model):
         self.best_classification = classification
         self.best_is_verified = True
         self.save()
+        if self.error:
+            self.error.metadata.best_classification = classification
+            self.error.metadata.best_is_verified = True
+            self.error.metadata.save(update_fields=["best_classification", "best_is_verified"])
         self.elastic_search_insert()
 
     def _serialized_components(self):
@@ -1227,12 +1266,17 @@ class ClassifiedFailure(models.Model):
     id = models.BigAutoField(primary_key=True)
     failure_lines = models.ManyToManyField(FailureLine, through='FailureMatch',
                                            related_name='classified_failures')
+    text_log_errors = models.ManyToManyField("TextLogError", through='TextLogErrorMatch',
+                                             related_name='classified_failures')
     # Note that we use a bug number of 0 as a sentinel value to indicate lines that
     # are not actually symptomatic of a real bug, but are still possible to autoclassify
     bug_number = models.PositiveIntegerField(blank=True, null=True, unique=True)
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
 
+    def __str__(self):
+        return "{0} {1}".format(self.id, self.bug_number)
+
     def bug(self):
         # Putting this here forces one query per object; there should be a way
         # to make things more efficient
@@ -1396,6 +1440,10 @@ class FailureMatch(models.Model):
             ('failure_line', 'classified_failure', 'matcher')
         )
 
+    def __str__(self):
+        return "{0} {1}".format(
+            self.failure_line.id, self.classified_failure.id)
+
 
 @python_2_unicode_compatible
 class RunnableJob(models.Model):
@@ -1476,11 +1524,74 @@ class TextLogError(models.Model):
         db_table = "text_log_error"
         unique_together = ('step', 'line_number')
 
+    def __str__(self):
+        return "{0} {1}".format(self.id, self.step.job.id)
+
+    @property
+    def metadata(self):
+        try:
+            return self._metadata
+        except TextLogErrorMetadata.DoesNotExist:
+            return None
+
     def bug_suggestions(self):
         from treeherder.model import error_summary
         return error_summary.bug_suggestions_line(self)
 
 
+class TextLogErrorMetadata(models.Model):
+    """Optional, mutable, data that can be associated with a TextLogError."""
+
+    text_log_error = models.OneToOneField(TextLogError,
+                                          primary_key=True,
+                                          related_name="_metadata",
+                                          on_delete=models.CASCADE)
+
+    failure_line = models.OneToOneField(FailureLine,
+                                        related_name="text_log_error_metadata",
+                                        null=True)
+
+    # Note that the case of best_classification = None and best_is_verified = True
+    # has the special semantic that the line is ignored and should not be considered
+    # for future autoclassifications.
+    best_classification = models.ForeignKey(ClassifiedFailure,
+                                            related_name="best_for_errors",
+                                            null=True,
+                                            on_delete=models.SET_NULL)
+    best_is_verified = models.BooleanField(default=False)
+
+    class Meta:
+        db_table = "text_log_error_metadata"
+
+
+class TextLogErrorMatch(models.Model):
+    """Association table between TextLogError and ClassifiedFailure, containing
+    additional data about the association including the matcher that was used
+    to create it and a score in the range 0-1 for the goodness of match."""
+
+    id = models.BigAutoField(primary_key=True)
+    text_log_error = models.ForeignKey(TextLogError,
+                                       related_name="matches",
+                                       on_delete=models.CASCADE)
+    classified_failure = models.ForeignKey(ClassifiedFailure,
+                                           related_name="error_matches",
+                                           on_delete=models.CASCADE)
+
+    matcher = models.ForeignKey(Matcher)
+    score = models.DecimalField(max_digits=3, decimal_places=2, blank=True, null=True)
+
+    class Meta:
+        db_table = 'text_log_error_match'
+        verbose_name_plural = 'text log error matches'
+        unique_together = (
+            ('text_log_error', 'classified_failure', 'matcher')
+        )
+
+    def __str__(self):
+        return "{0} {1}".format(
+            self.text_log_error.id, self.classified_failure.id)
+
+
 class TextLogSummary(models.Model):
     """
     An intermediate class correlating artifact + text log data with

treeherder/webapp/api/serializers.py

@@ -193,7 +193,7 @@ class ClassifiedFailureSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = models.ClassifiedFailure
-        exclude = ['failure_lines', 'created', 'modified']
+        exclude = ['failure_lines', 'created', 'modified', 'text_log_errors']
 
 
 class FailureMatchSerializer(serializers.ModelSerializer):
@@ -215,8 +215,14 @@ class FailureLineNoStackSerializer(serializers.ModelSerializer):
                    'stackwalk_stderr']
 
 
+class TextLogErrorMetadataSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.TextLogErrorMetadata
+
+
 class TextLogErrorSerializer(serializers.ModelSerializer):
     bug_suggestions = NoOpSerializer(read_only=True)
+    metadata = TextLogErrorMetadataSerializer(read_only=True)
 
     class Meta:
         model = models.TextLogError