Bug 1277955 - Add support for revision_hash to pulse jobs

Some repos are longer-lived and do not yet have the Task Cluster
code that allows them to submit tasks with a revision.  They only
have the older code to submit revision_hash.  This prevents the
jobs from being ingested via Pulse.  This commit adds support
for revision_hash until a time when it's no longer needed.

schemas/pulse-job.yml

@@ -33,6 +33,9 @@ properties:
   origin:
     oneOf:
       - type: "object"
+        description: |
+          PREFERRED: An HG job that only has a revision.  This is for all
+          jobs going forward.
         properties:
           kind:
             type: "string"
@@ -51,6 +54,29 @@ properties:
             type: "integer"
         required: [kind, project, revision]
 
+      - type: "object"
+        description: |
+          BACKWARD COMPATABILITY: An HG job that only has a revision_hash.
+          Some repos like mozilla-beta have not yet merged in the code that
+          allows them access to the revision.
+        properties:
+          kind:
+            type: "string"
+            enum: ['hg.mozilla.org']
+          project:
+            type: "string"
+            pattern: "^[\\w-]+$"
+            minLength: 1
+            maxLength: 50
+          revision_hash:
+            type: "string"
+            pattern: "^[0-9a-f]+$"
+            minLength: 40
+            maxLength: 40
+          pushLogID:
+            type: "integer"
+        required: [kind, project, revision_hash]
+
       - type: "object"
         properties:
           kind:

tests/etl/test_job_loader.py

@@ -4,7 +4,8 @@ import pytest
 
 from treeherder.etl.job_loader import JobLoader
 from treeherder.model.derived.artifacts import ArtifactsModel
-from treeherder.model.models import (JobDetail,
+from treeherder.model.models import (Job,
+                                     JobDetail,
                                      JobLog)
 
 
@@ -61,6 +62,26 @@ def test_ingest_pulse_jobs(pulse_jobs, test_project, jm, result_set_stored,
     assert JobDetail.objects.count() == 2
 
 
+def test_ingest_pulse_jobs_with_revision_hash(pulse_jobs, test_project, jm,
+                                              result_set_stored,
+                                              mock_log_parser):
+    """
+    Ingest a revision_hash job with the JobLoader used by Pulse
+    """
+
+    jl = JobLoader()
+    rs = jm.get_result_set_list(0, 10)[0]
+    revision_hash = rs["revision_hash"]
+    for job in pulse_jobs:
+        origin = job["origin"]
+        del(origin["revision"])
+        origin["revision_hash"] = revision_hash
+
+    jl.process_job_list(pulse_jobs)
+
+    assert Job.objects.count() == 4
+
+
 def test_transition_pending_running_complete(first_job, jm, mock_log_parser):
     jl = JobLoader()
 

treeherder/etl/job_loader.py

@@ -3,6 +3,7 @@ import time
 from collections import defaultdict
 
 import jsonschema
+import newrelic.agent
 from dateutil import parser
 
 from treeherder.etl.schema import job_json_schema
@@ -70,7 +71,6 @@ class JobLoader:
         """
         job_guid = self._get_job_guid(pulse_job)
         x = {
-            "revision": pulse_job["origin"]["revision"],
             "job": {
                 "job_guid": job_guid,
                 "name": pulse_job["display"].get("jobName", "unknown"),
@@ -91,6 +91,25 @@ class JobLoader:
             "coalesced": pulse_job.get("coalesced", [])
         }
 
+        # It is possible there will be either a revision or a revision_hash
+        # At some point we will ONLY get revisions and no longer receive
+        # revision_hashes and then this check can be removed.
+        revision = pulse_job["origin"].get("revision", None)
+        if revision:
+            x["revision"] = revision
+        else:
+            x["revision_hash"] = pulse_job["origin"]["revision_hash"]
+            logger.warning(
+                "Pulse job had revision_hash instead of revision: {}:{}".format(
+                    pulse_job["origin"]["project"],
+                    x["revision_hash"]
+                ))
+            params = {
+                "project": pulse_job["origin"]["project"],
+                "revision_hash": x["revision_hash"]
+            }
+            newrelic.agent.record_custom_event("revision_hash_usage", params=params)
+
         # some or all the time fields may not be present in some cases
         for k, v in self.TIME_FIELD_MAP.items():
             if v in pulse_job: