job diff / compare class now mostly working

tests/model/derived/test_jobs_model.py

@@ -30,9 +30,10 @@ def test_ingest_single_sample_job(jm, sample_data):
     jm.store_job_data(json.dumps(blob))
     job_id = jm.process_objects(1)[0]
 
-    job_dict = JobDictBuilder(jm, job_id).as_dict()
+    exp_job = clean_job_blob_dict(blob["jobs"][0])
+    act_job = JobDictBuilder(jm, job_id).as_dict()
 
-    assert blob["jobs"][0] == job_dict, diff_dict(blob["jobs"][0], job_dict)
+    assert exp_job == act_job, diff_dict(exp_job, act_job)
 
     # print json.dumps(blob, indent=4)
     # print json.dumps(job_dict, indent=4)
@@ -42,7 +43,27 @@ def test_ingest_single_sample_job(jm, sample_data):
     loading_count = jm.get_os_dhub().execute(
         proc="objectstore_test.counts.loading")[0]["loading_count"]
 
-    assert complete_count == 2
+    assert complete_count == 1
+    assert loading_count == 0
+
+
+def test_ingest_all_sample_jobs(jm, sample_data):
+    """Process all job structures in the job_data.txt file"""
+    for blob in sample_data.job_data:
+        jm.store_job_data(json.dumps(blob))
+        job_id = jm.process_objects(1)[0]
+
+        exp_job = clean_job_blob_dict(blob["jobs"][0])
+        act_job = JobDictBuilder(jm, job_id).as_dict()
+
+        assert exp_job == act_job, diff_dict(exp_job, act_job)
+
+    complete_count = jm.get_os_dhub().execute(
+        proc="objectstore_test.counts.complete")[0]["complete_count"]
+    loading_count = jm.get_os_dhub().execute(
+        proc="objectstore_test.counts.loading")[0]["loading_count"]
+
+    assert complete_count == len(sample_data.job_data)
     assert loading_count == 0
 
 
@@ -62,23 +83,99 @@ class JobDictBuilder(object):
 
         job["artifact"] = self._get_artifact()
         job["log_references"] = self._get_logs()
+
         job["option_collection"] = self._get_option_collection(
             job["option_collection_id"])
         del(job["option_collection_id"])
+
         job["machine_platform"] = self._get_machine_platform(
             job["machine_platform_id"])
         del(job["machine_platform_id"])
-        return self._unicode_keys(job)
 
-    def _get_option_collection(self, oc_id):
+        job["build_platform"] = self._get_build_platform(
-        # need this in refdata model
+            job["build_platform_id"])
-        #oc = self.jm.refdata_model.get_option_collection(oc_id)
+        del(job["build_platform_id"])
-        return "NotImplementedYet"
 
-    def _get_machine_platform(self, mp_id):
+        job["machine"] = self._get_machine(
-        # need this in refdata model
+            job["machine_id"])
-        #mp = self.jm.refdata_model.get_machine_platform(mp_id)
+        del(job["machine_id"])
-        return "NotImplementedYet"
+
+        job["product_name"] = self._get_product(
+            job["product_id"])
+        del(job["product_id"])
+
+        job["name"] = self._get_name(
+            job["job_type_id"])
+        del(job["job_type_id"])
+
+        del(job["id"])
+        del(job["active_status"])
+        del(job["result_set_id"])
+
+        if not job["job_coalesced_to_guid"]:
+            del(job["job_coalesced_to_guid"])
+
+        return unicode_keys(job)
+
+    def _get_option_collection(self, obj_id):
+        option_id = self.jm.refdata_model.get_row_by_id(
+            "option_collection",
+            obj_id,
+        ).get_column_data("option_id")
+        obj = self.jm.refdata_model.get_row_by_id(
+            "option",
+            option_id,
+        ).next()
+        return {unicode(obj["name"]): True}
+
+    def _get_machine_platform(self, obj_id):
+        obj = self.jm.refdata_model.get_row_by_id(
+            "machine_platform",
+            obj_id,
+        ).next()
+        del(obj["active_status"])
+        del(obj["id"])
+        return unicode_keys(obj)
+
+    def _get_build_platform(self, obj_id):
+        obj = self.jm.refdata_model.get_row_by_id(
+            "build_platform",
+            obj_id,
+        ).next()
+        del(obj["active_status"])
+        del(obj["id"])
+        return unicode_keys(obj)
+
+    def _get_machine(self, obj_id):
+        obj = self.jm.refdata_model.get_row_by_id(
+            "machine",
+            obj_id,
+        ).get_column_data("name")
+        return obj
+
+    def _get_product(self, obj_id):
+        obj = self.jm.refdata_model.get_row_by_id(
+            "product",
+            obj_id,
+        ).get_column_data("name")
+        return obj
+
+    def _get_name(self, obj_id):
+        job_type = self.jm.refdata_model.get_row_by_id(
+            "job_type",
+            obj_id,
+        ).next()
+        job_group = self.jm.refdata_model.get_row_by_id(
+            "job_group",
+            job_type["job_group_id"],
+        ).get_column_data("name")
+        if job_type["name"]:
+            return u"{0}-{1}".format(
+                job_group,
+                job_type["name"],
+            )
+        else:
+            return job_group
 
     def _get_logs(self):
         logs = self.jm.get_jobs_dhub().execute(
@@ -93,7 +190,7 @@ class JobDictBuilder(object):
             del(log["active_status"])
             del(log["id"])
             del(log["job_id"])
-            log_values.append(self._unicode_keys(log))
+            log_values.append(unicode_keys(log))
 
         return log_values
 
@@ -110,8 +207,29 @@ class JobDictBuilder(object):
             artifacts = artifacts.values()
         return artifacts
 
-    def _unicode_keys(self, d):
+
-        return dict([(unicode(k), v) for k, v in d.items()])
+def unicode_keys(d):
+    return dict([(unicode(k), v) for k, v in d.items()])
+
+
+def clean_job_blob_dict(job):
+    """Fix a few fields so they're easier to compare"""
+    job["start_timestamp"] = long(job["start_timestamp"])
+    job["submit_timestamp"] = long(job["submit_timestamp"])
+    job["end_timestamp"] = long(job["end_timestamp"])
+    job["result"] = unicode(job["result"])
+
+    # @@@ we don't keep track of VM'ness?
+    try:
+        del(job["machine_platform"]["vm"])
+    except KeyError:
+        pass  # oh, that's ok
+    try:
+        del(job["build_platform"]["vm"])
+    except KeyError:
+        pass  # oh, that's ok
+
+    return job
 
 
 def diff_dict(d1, d2):

treeherder/model/derived/refdata.py

@@ -26,6 +26,18 @@ class RefDataManager(object):
         self.dhub = DataHub.get("reference")
         self.DEBUG = settings.DEBUG
 
+    def get_row_by_id(self, table_name, obj_id):
+        iter_obj = self.dhub.execute(
+            sql="SELECT * FROM `{0}` WHERE `id` = ?".format(table_name),
+            placeholders=[obj_id],
+            host_type="master_host", # @@@ for some reason ``read_host`` doesn't work here.  get a key error in datasource.
+            debug_show=self.DEBUG,
+            return_type='iter',
+        )
+
+        return iter_obj
+
+
     def get_build_platform_id(self, os_name, platform, architecture):
 
         id_iter = self.dhub.execute(