added revision_hash field to objectstore.

tests/model/derived/test_objectstore_model.py

@@ -5,6 +5,7 @@ from .sample_data_generator import job_json
 
 slow = pytest.mark.slow
 
+
 def test_unicode(jm):
     """Unicode representation of a ``JobModel`` is the project name."""
     assert unicode(jm) == unicode(jm.project)
@@ -44,13 +45,15 @@ def test_mark_object_complete(jm):
     jm.store_job_data(job_json())
     row_id = jm.claim_objects(1)[0]["id"]
     job_id = 7  # any arbitrary number; no cross-db constraint checks
+    revision_hash = "fakehash"
 
-    jm.mark_object_complete(row_id, job_id)
+    jm.mark_object_complete(row_id, job_id, revision_hash)
 
     row_data = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
         proc="objectstore_test.selects.row", placeholders=[row_id])[0]
 
     assert row_data["job_id"] == job_id
+    assert row_data["revision_hash"] == revision_hash
     assert row_data["processed_state"] == "complete"
 
 
@@ -69,6 +72,9 @@ def test_process_objects(jm):
     # just process two rows
     jm.process_objects(2)
 
+    import time
+    time.sleep(60)
+
     test_run_rows = jm.get_dhub(jm.CT_JOBS).execute(
         proc="jobs_test.selects.jobs")
     date_set = set([r['submit_timestamp'] for r in test_run_rows])

treeherder/model/derived/jobs.py

@@ -179,7 +179,6 @@ class JobsModel(TreeherderModelBase):
         # Get/Set reference info
 
         rdm = self.refdata_model
-        job_id = -1
         job = data["jobs"]
 
         build_platform_id = rdm.get_or_create_build_platform(
@@ -380,6 +379,7 @@ class JobsModel(TreeherderModelBase):
             try:
                 data = JobData.from_json(row['json_blob'])
                 job_id = self.load_job_data(data)
+                revision_hash = data["revision_hash"]
             except JobDataError as e:
                 self.mark_object_error(row_id, str(e))
             except Exception as e:
@@ -389,7 +389,7 @@ class JobsModel(TreeherderModelBase):
                         e.__class__.__name__, unicode(e))
                 )
             else:
-                self.mark_object_complete(row_id, job_id)
+                self.mark_object_complete(row_id, job_id, revision_hash)
                 job_ids_loaded.append(job_id)
 
         return job_ids_loaded
@@ -425,9 +425,9 @@ class JobsModel(TreeherderModelBase):
         #
         # The mark_loading SQL statement does execute an UPDATE/LIMIT but now
         # implements an "ORDER BY id" clause making the UPDATE
-        # deterministic/safe.  I've been unsuccessfull capturing the specific
+        # deterministic/safe.  I've been unsuccessful capturing the specific
         # warning generated without redirecting program flow control.  To
-        # ressolve the problem in production, we're disabling MySQLdb.Warnings
+        # resolve the problem in production, we're disabling MySQLdb.Warnings
         # before executing mark_loading and then re-enabling warnings
         # immediately after.  If this bug is ever fixed in mysql this handling
         # should be removed. Holy Hackery! -Jeads
@@ -454,11 +454,11 @@ class JobsModel(TreeherderModelBase):
 
         return json_blobs
 
-    def mark_object_complete(self, object_id, job_id):
+    def mark_object_complete(self, object_id, job_id, revision_hash):
         """ Call to database to mark the task completed """
         self.get_os_dhub().execute(
             proc="objectstore.updates.mark_complete",
-            placeholders=[job_id, object_id],
+            placeholders=[job_id, revision_hash, object_id],
             debug_show=self.DEBUG
         )
 

treeherder/model/sql/objectstore.json

@@ -115,7 +115,9 @@
         "mark_complete":{
 
             "sql":"UPDATE   `objectstore`
-                   SET      `processed_state` = 'complete', `job_id` = ?
+                   SET      `processed_state` = 'complete',
+                            `job_id` = ?,
+                            `revision_hash` = ?
                    WHERE    `processed_state` = 'loading'
                    AND      `id` = ?
                    AND      `worker_id` = CONNECTION_ID()

treeherder/model/sql/template_schema/project_objectstore_1.sql.tmpl

@@ -31,7 +31,7 @@ DROP TABLE IF EXISTS `objectstore`;
  *
  *  An object store for the incoming JSON structures described in
  *  sample_data/job_data.json.sample. These structures are transfered to
- *  project_jobs_1.sql.tmpl and treeherder_reference_1.sql.tmpl by a 
+ *  project_jobs_1.sql.tmpl and treeherder_reference_1.sql.tmpl by a
  *  scheduled job.
  *
  * Population Method: dynamic from incoming data
@@ -39,6 +39,7 @@ DROP TABLE IF EXISTS `objectstore`;
  * Example Data:
  *
  *  job_id - Referenced project_jobs_1.job.id
+ *  revision_hash - Hash of any number of revisions associated with the result set.
  *  loaded_timestamp - Timestamp when the structure was first loaded.
  *  processed_state - ready | loading | complete
  *                  ready - Object ready for processing
@@ -53,6 +54,7 @@ DROP TABLE IF EXISTS `objectstore`;
 CREATE TABLE `objectstore` (
   `id` bigint(11) unsigned NOT NULL AUTO_INCREMENT,
   `job_id` bigint(11) unsigned DEFAULT NULL,
+  `revision_hash` varchar(50) COLLATE utf8_bin DEFAULT NULL,
   `loaded_timestamp` int(11) unsigned NOT NULL,
   `processed_state` enum('ready','loading','complete') COLLATE utf8_bin DEFAULT 'ready',
   `error` enum('N','Y') COLLATE utf8_bin DEFAULT 'N',