From 3fd8b1b5b3a55270a5d02cef20dc8dd7fbed38c7 Mon Sep 17 00:00:00 2001 From: Joel Maher Date: Wed, 23 Oct 2024 13:41:41 -0700 Subject: [PATCH] WIP - add new_failure field to TextLogError and update when we have a new failure. (#8254) --- tests/etl/test_load_artifacts.py | 12 +++++++++++ tests/webapp/api/test_jobs_api.py | 2 ++ treeherder/etl/artifact.py | 11 +++++++++- .../0033_textlogerror_new_failure.py | 21 +++++++++++++++++++ treeherder/model/models.py | 1 + 5 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 treeherder/model/migrations/0033_textlogerror_new_failure.py diff --git a/tests/etl/test_load_artifacts.py b/tests/etl/test_load_artifacts.py index 4c70fed95..2579a503c 100644 --- a/tests/etl/test_load_artifacts.py +++ b/tests/etl/test_load_artifacts.py @@ -2,6 +2,7 @@ import json from treeherder.etl.artifact import store_job_artifacts from treeherder.model.models import TextLogError +from treeherder.model.error_summary import get_error_summary def test_load_textlog_summary_twice(test_repository, test_job): @@ -50,8 +51,19 @@ def test_load_non_ascii_textlog_errors(test_job): "job_guid": test_job.guid, } + # ensure a result='failed' to treat failure as a NEW_failure + test_job.result = "testfailed" + test_job.save() + store_job_artifacts([text_log_summary_artifact]) + # ensure bug_suggestions data is stored and retrieved properly + tle_all = TextLogError.objects.all() + bug_suggestions = get_error_summary(test_job) + for suggestion in bug_suggestions: + tle = next(t for t in tle_all if t.line_number == suggestion["line_number"]) + assert suggestion["failure_new_in_rev"] == tle.new_failure + assert TextLogError.objects.count() == 2 assert TextLogError.objects.get(line_number=1587).line == "07:51:28 WARNING - \U000000c3" assert TextLogError.objects.get(line_number=1588).line == "07:51:29 WARNING - " diff --git a/tests/webapp/api/test_jobs_api.py b/tests/webapp/api/test_jobs_api.py index 7b1fb6e3b..90ba1ac8b 100644 --- a/tests/webapp/api/test_jobs_api.py +++ b/tests/webapp/api/test_jobs_api.py @@ -222,12 +222,14 @@ def test_text_log_errors(client, test_job): "job": 1, "line": "failure 1", "line_number": 101, + "new_failure": False, }, { "id": 2, "job": 1, "line": "failure 2", "line_number": 102, + "new_failure": False, }, ] diff --git a/treeherder/etl/artifact.py b/treeherder/etl/artifact.py index 7facdd002..2d0b46a7a 100644 --- a/treeherder/etl/artifact.py +++ b/treeherder/etl/artifact.py @@ -30,6 +30,9 @@ def store_text_log_summary_artifact(job, text_log_summary_artifact): ignore_conflicts=True, ) + # Bulk create doesn't return .id field, so query to get them. + log_errors = TextLogError.objects.filter(job=job) + # get error summary immediately (to warm the cache) # Conflicts may have occured during the insert, but we pass the queryset for performance bugs = error_summary.get_error_summary(job, queryset=log_errors) @@ -42,7 +45,13 @@ def store_text_log_summary_artifact(job, text_log_summary_artifact): ]: # classify job as `new failure` - for filtering, etc. job.failure_classification_id = 6 - job.save() + job.save(update_fields=["failure_classification_id"]) + # for every log_errors (TLE object) there is a corresponding bugs/suggestion + for tle in log_errors: + if tle.line_number == suggestion["line_number"]: + tle.new_failure = True + tle.save(update_fields=["new_failure"]) + break def store_job_artifacts(artifact_data): diff --git a/treeherder/model/migrations/0033_textlogerror_new_failure.py b/treeherder/model/migrations/0033_textlogerror_new_failure.py new file mode 100644 index 000000000..fcf45af48 --- /dev/null +++ b/treeherder/model/migrations/0033_textlogerror_new_failure.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.13 on 2024-10-21 13:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "model", + "0032_rename_failureline_job_guid_repository_failure_lin_job_gui_b67c6d_idx_and_more", + ), + ] + + operations = [ + migrations.AddField( + model_name="textlogerror", + name="new_failure", + field=models.BooleanField(default=False), + ), + ] diff --git a/treeherder/model/models.py b/treeherder/model/models.py index 3a59e5636..49e77d6a8 100644 --- a/treeherder/model/models.py +++ b/treeherder/model/models.py @@ -1219,6 +1219,7 @@ class TextLogError(models.Model): job = models.ForeignKey(Job, on_delete=models.CASCADE, related_name="text_log_error", null=True) line = models.TextField() line_number = models.PositiveIntegerField() + new_failure = models.BooleanField(default=False) # TODO delete this field and unique_together once backfill of jobs in TextLogError table has been completed step = models.ForeignKey(