Bug 1323110 - Don't have a uniqueness constraint on task id column

We can have multiple jobs with the same task id in the case of retries

tests/etl/test_job_loader.py

@@ -59,7 +59,7 @@ def test_ingest_pulse_jobs(pulse_jobs, test_repository, result_set_stored,
     jl.process_job_list(pulse_jobs)
 
     jobs = Job.objects.all()
-    assert len(jobs) == 4
+    assert len(jobs) == 5
 
     assert [job.taskcluster_metadata for job in jobs]
     assert set(TaskclusterMetadata.objects.values_list(
@@ -118,7 +118,7 @@ def test_ingest_pulse_jobs_with_revision_hash(pulse_jobs, test_repository,
 
     jl.process_job_list(pulse_jobs)
 
-    assert Job.objects.count() == 4
+    assert Job.objects.count() == 5
 
 
 def test_ingest_pulse_jobs_with_missing_resultset(pulse_jobs):

tests/sample_data/pulse_consumer/job_data.json

@@ -108,6 +108,48 @@
     "version": 1
   },
 
+  {
+    "taskId": "5c909b6d-143a-4a9f-942f-594bfadc399e/1",
+    "retryId": 1,
+    "origin": {
+      "kind": "hg.mozilla.org",
+      "project": "set by test",
+      "revision": "1234567890123456789012345678901234567890"
+    },
+    "buildSystem": "taskcluster",
+    "display": {
+      "jobSymbol": "P",
+      "chunkId": 2,
+      "jobName": "test-android-4.3-arm7-api-15/debug-mochitest-media-1",
+      "groupSymbol": "?",
+      "groupName": "unknown"
+    },
+    "state": "running",
+    "jobKind": "test",
+    "runMachine": {
+      "name": "bld-linux64-ec2-104",
+      "platform": "linux64",
+      "os": "linux",
+      "architecture": "x86_64"
+    },
+    "buildMachine": {
+      "name": "bld-linux64-ec2-104",
+      "platform": "linux64",
+      "os": "linux",
+      "architecture": "x86_64"
+    },
+    "owner": "thedude@lebowski.com",
+    "reason": "scheduler",
+    "productName": "Oscillation Overthruster",
+    "timeScheduled": "2014-12-19T16:39:57-08:00",
+    "timeStarted": "2014-12-19T17:39:57-08:00",
+    "timeCompleted": "2014-12-19T18:39:57-08:00",
+    "labels": [
+      "debug"
+    ],
+    "version": 1
+  },
+
   {
     "taskId": "66c4b325-0bb7-43ba-b631-f7a120103329/0",
     "retryId": 0,

tests/sample_data/pulse_consumer/transformed_job_data.json

@@ -110,6 +110,45 @@
     "revision": "45f8637cb9f78f19cb8463ff174e81756805d8cf",
     "coalesced": []
   },
+  {
+    "job": {
+      "taskcluster_task_id": "XJCbbRQ6Sp-UL1lL-tw5ng",
+      "taskcluster_retry_id": 1,
+      "build_platform": {
+        "platform": "linux64",
+        "os_name": "linux",
+        "architecture": "x86_64"
+      },
+      "submit_timestamp": 1419035997.0,
+      "start_timestamp": 1419039597.0,
+      "name": "test-android-4.3-arm7-api-15/debug-mochitest-media-1",
+      "option_collection": {
+        "debug": true
+      },
+      "artifacts": [],
+      "machine_platform": {
+        "platform": "linux64",
+        "os_name": "linux",
+        "architecture": "x86_64"
+      },
+      "build_system_type": "taskcluster",
+      "who": "thedude@lebowski.com",
+      "group_symbol": "?",
+      "reason": "scheduler",
+      "group_name": "unknown",
+      "machine": "bld-linux64-ec2-104",
+      "state": "running",
+      "result": "unknown",
+      "log_references": [],
+      "tier": 1,
+      "job_symbol": "P2",
+      "job_guid": "5c909b6d-143a-4a9f-942f-594bfadc399e/1",
+      "product_name": "Oscillation Overthruster",
+      "end_timestamp": 1419043197.0
+    },
+    "revision": "45f8637cb9f78f19cb8463ff174e81756805d8cf",
+    "coalesced": []
+  },
   {
     "job": {
       "taskcluster_task_id": "ZsSzJQu3Q7q2MfehIBAzKQ",

treeherder/etl/jobs.py

@@ -6,6 +6,7 @@ from hashlib import sha1
 import newrelic.agent
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
+from django.db.utils import IntegrityError
 
 from treeherder.etl.artifact import (serialize_artifact_json_blobs,
                                      store_job_artifacts)
@@ -311,11 +312,13 @@ def _load_job(repository, job_datum, push_id, lower_tier_signatures):
         push_id=push_id)
 
     if all([k in job_datum for k in ['taskcluster_task_id', 'taskcluster_retry_id']]):
-        TaskclusterMetadata.objects.get_or_create(
+        try:
+            TaskclusterMetadata.objects.create(
                 job=job,
                 task_id=job_datum['taskcluster_task_id'],
                 retry_id=job_datum['taskcluster_retry_id'])
-
+        except IntegrityError:
+            pass
     artifacts = job_datum.get('artifacts', [])
 
     has_text_log_summary = any(x for x in artifacts

treeherder/model/migrations/0003_taskcluster_taskid_not_unique.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-02-24 15:23
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('model', '0002_add_taskcluster_job_metadata'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='taskclustermetadata',
+            name='task_id',
+            field=models.CharField(max_length=22, validators=[django.core.validators.MinLengthValidator(22)]),
+        ),
+    ]

treeherder/model/models.py

@@ -830,8 +830,7 @@ class TaskclusterMetadata(models.Model):
         related_name='taskcluster_metadata'
     )
 
-    task_id = models.CharField(unique=True,
-                               max_length=22,
+    task_id = models.CharField(max_length=22,
                                validators=[MinLengthValidator(22)])
     retry_id = models.PositiveIntegerField()