now populating revisions, but not yet verifying that data.

2013-04-25 17:27:59 -07:00 · 2013-04-25 17:27:59 -07:00 · 8583deea38
--- a/tests/model/derived/test_jobs_model.py
+++ b/tests/model/derived/test_jobs_model.py
@ -3,15 +3,12 @@ import difflib
 import pprint


-from .sample_data_generator import job_json
-
-
 def test_unicode(jm):
    """Unicode representation of a ``JobModel`` is the project name."""
    assert unicode(jm) == unicode(jm.project)


-def test_disconnect(jm):
+def xtest_disconnect(jm):
    """test that your model disconnects"""

    # establish the connection to jobs.
@ -25,19 +22,20 @@ def test_disconnect(jm):


 def test_ingest_single_sample_job(jm, sample_data):
-    """Process all job structures in the job_data.txt file"""
+    """Process a single job structure in the job_data.txt file"""
    blob = sample_data.job_data[0]
    jm.store_job_data(json.dumps(blob))
-    job_id = jm.process_objects(1)[0]
+    jobs = jm.process_objects(1)
+    # import time
+    # time.sleep(30)
+    assert len(jobs) == 1
+    job_id = jobs[0]

    exp_job = clean_job_blob_dict(blob["jobs"][0])
    act_job = JobDictBuilder(jm, job_id).as_dict()

    assert exp_job == act_job, diff_dict(exp_job, act_job)

-    # print json.dumps(blob, indent=4)
-    # print json.dumps(job_dict, indent=4)
-
    complete_count = jm.get_os_dhub().execute(
        proc="objectstore_test.counts.complete")[0]["complete_count"]
    loading_count = jm.get_os_dhub().execute(
@ -48,10 +46,18 @@ def test_ingest_single_sample_job(jm, sample_data):


 def test_ingest_all_sample_jobs(jm, sample_data):
-    """Process all job structures in the job_data.txt file"""
+    """
+    Process each job structure in the job_data.txt file and verify.
+
+    This rebuilds the JSON blob (for the most part) and compares that
+    everything was stored correctly.
+
+    """
    for blob in sample_data.job_data:
        jm.store_job_data(json.dumps(blob))
-        job_id = jm.process_objects(1)[0]
+        jobs = jm.process_objects(1)
+        assert len(jobs) == 1
+        job_id = jobs[0]

        exp_job = clean_job_blob_dict(blob["jobs"][0])
        act_job = JobDictBuilder(jm, job_id).as_dict()
--- a/treeherder/model/derived/jobs.py
+++ b/treeherder/model/derived/jobs.py
@ -180,9 +180,15 @@ class JobsModel(TreeherderModelBase):

        """

-        # @@@ sources
+        result_set_id = self._set_result_set(data["revision_hash"])

-        # Get/Set reference info, all inserts use ON DUPLICATE KEY
+        # assert result_set_id > 0
+        # sources
+        for src in data["sources"]:
+            revision_id = self._insert_revision(src)
+            self._insert_revision_map(revision_id, result_set_id)
+
+        # set Job data

        rdm = self.refdata_model
        job_id = -1
@ -223,8 +229,6 @@ class JobsModel(TreeherderModelBase):
                job["product_name"],
            )

-            result_set_id = self._set_result_set(data["revision_hash"])
-
            job_id = self._set_job_data(
                job,
                result_set_id,
@ -277,6 +281,45 @@ class JobsModel(TreeherderModelBase):

        return result_set_id

+    def _insert_revision(self, src):
+        """
+        Insert a source to the ``revision`` table
+
+        Example source:
+        {
+            "commit_timestamp": 1365732271,
+            "push_timestamp": 1365732271,
+            "comments": "Bug 854583 - Use _pointer_ instead of...",
+            "repository": "mozilla-aurora",
+            "revision": "c91ee0e8a980"
+        }
+
+        """
+        repository_id = self.refdata_model.get_repository_id(
+            src["repository"])
+
+        revision_id = self._insert_data_and_get_id(
+            'set_revision',
+            [
+                src["revision"],
+                "author?",
+                src["comments"],
+                src["push_timestamp"],
+                src["commit_timestamp"],
+                repository_id,
+            ]
+        )
+        return revision_id
+
+    def _insert_revision_map(self, revision_id, result_set_id):
+        self._insert_data(
+            'set_revision_map',
+            [
+                revision_id,
+                result_set_id
+            ]
+        )
+
    def _set_job_data(self, data, result_set_id, build_platform_id,
                      machine_platform_id, machine_id, option_collection_id,
                      job_type_id, product_id):
--- a/treeherder/model/derived/refdata.py
+++ b/treeherder/model/derived/refdata.py
@ -264,6 +264,16 @@ class RefDataManager(object):

        id_iter = self.dhub.execute(
            proc='reference.selects.get_repository_version_id',
+            placeholders=[repository_id, version],
+            debug_show=self.DEBUG,
+            return_type='iter')
+
+        return id_iter.get_column_data('id')
+
+    def get_repository_id(self, name):
+
+        id_iter = self.dhub.execute(
+            proc='reference.selects.get_repository_id',
            placeholders=[name],
            debug_show=self.DEBUG,
            return_type='iter')
--- a/treeherder/model/fixtures/repository.json
+++ b/treeherder/model/fixtures/repository.json
@ -1,547 +1,560 @@
 [
 {
-    "pk": 1, 
-    "model": "model.repository", 
+    "pk": 1,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "mozilla-central", 
-        "url": "https://hg.mozilla.org/mozilla-central", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "mozilla-central",
+        "url": "https://hg.mozilla.org/mozilla-central",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 2, 
-    "model": "model.repository", 
+    "pk": 2,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "mozilla-inbound", 
-        "url": "https://hg.mozilla.org/integration/mozilla-inbound", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "mozilla-inbound",
+        "url": "https://hg.mozilla.org/integration/mozilla-inbound",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 3, 
-    "model": "model.repository", 
+    "pk": 3,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "mozilla-aurora", 
-        "url": "https://hg.mozilla.org/releases/mozilla-aurora", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "mozilla-aurora",
+        "url": "https://hg.mozilla.org/releases/mozilla-aurora",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 4, 
-    "model": "model.repository", 
+    "pk": 4,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "mozilla-beta", 
-        "url": "https://hg.mozilla.org/releases/mozilla-beta", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "mozilla-beta",
+        "url": "https://hg.mozilla.org/releases/mozilla-beta",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 5, 
-    "model": "model.repository", 
+    "pk": 5,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "mozilla-release", 
-        "url": "https://hg.mozilla.org/releases/mozilla-release", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "mozilla-release",
+        "url": "https://hg.mozilla.org/releases/mozilla-release",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 6, 
-    "model": "model.repository", 
+    "pk": 6,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "mozilla-b2g18", 
-        "url": "https://hg.mozilla.org/releases/mozilla-b2g18", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "mozilla-b2g18",
+        "url": "https://hg.mozilla.org/releases/mozilla-b2g18",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 7, 
-    "model": "model.repository", 
+    "pk": 7,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "mozilla-b2g18_v1_0_1", 
-        "url": "https://hg.mozilla.org/releases/mozilla-b2g18_v1_0_1", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "mozilla-b2g18_v1_0_1",
+        "url": "https://hg.mozilla.org/releases/mozilla-b2g18_v1_0_1",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 8, 
-    "model": "model.repository", 
+    "pk": 8,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "mozilla-b2g18_v1_1_0", 
-        "url": "https://hg.mozilla.org/releases/mozilla-b2g18_v1_1_0", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "mozilla-b2g18_v1_1_0",
+        "url": "https://hg.mozilla.org/releases/mozilla-b2g18_v1_1_0",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 9, 
-    "model": "model.repository", 
+    "pk": 9,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "mozilla-esr17", 
-        "url": "https://hg.mozilla.org/releases/mozilla-esr17", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "mozilla-esr17",
+        "url": "https://hg.mozilla.org/releases/mozilla-esr17",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 10, 
-    "model": "model.repository", 
+    "pk": 10,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "try", 
-        "url": "https://hg.mozilla.org/try", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 3, 
+        "dvcs_type": "hg",
+        "name": "try",
+        "url": "https://hg.mozilla.org/try",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 3,
        "description": ""
    }
 },
 {
-    "pk": 11, 
-    "model": "model.repository", 
+    "pk": 11,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "build-system", 
-        "url": "https://hg.mozilla.org/projects/build-system", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "build-system",
+        "url": "https://hg.mozilla.org/projects/build-system",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 12, 
-    "model": "model.repository", 
+    "pk": 12,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "fx-team", 
-        "url": "https://hg.mozilla.org/integration/fx-team", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "fx-team",
+        "url": "https://hg.mozilla.org/integration/fx-team",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 13, 
-    "model": "model.repository", 
+    "pk": 13,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "graphics", 
-        "url": "https://hg.mozilla.org/projects/graphics", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "graphics",
+        "url": "https://hg.mozilla.org/projects/graphics",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 14, 
-    "model": "model.repository", 
+    "pk": 14,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "ionmonkey", 
-        "url": "https://hg.mozilla.org/projects/ionmonkey", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "ionmonkey",
+        "url": "https://hg.mozilla.org/projects/ionmonkey",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 15, 
-    "model": "model.repository", 
+    "pk": 15,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "profiling", 
-        "url": "https://hg.mozilla.org/projects/profiling", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "profiling",
+        "url": "https://hg.mozilla.org/projects/profiling",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 16, 
-    "model": "model.repository", 
+    "pk": 16,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "services-central", 
-        "url": "https://hg.mozilla.org/services/services-central", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "services-central",
+        "url": "https://hg.mozilla.org/services/services-central",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 17, 
-    "model": "model.repository", 
+    "pk": 17,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "ux", 
-        "url": "https://hg.mozilla.org/projects/ux", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "ux",
+        "url": "https://hg.mozilla.org/projects/ux",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 18, 
-    "model": "model.repository", 
+    "pk": 18,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "alder", 
-        "url": "https://hg.mozilla.org/projects/alder", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "alder",
+        "url": "https://hg.mozilla.org/projects/alder",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 19, 
-    "model": "model.repository", 
+    "pk": 19,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "ash", 
-        "url": "https://hg.mozilla.org/projects/ash", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "ash",
+        "url": "https://hg.mozilla.org/projects/ash",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 20, 
-    "model": "model.repository", 
+    "pk": 20,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "birch", 
-        "url": "https://hg.mozilla.org/projects/birch", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "birch",
+        "url": "https://hg.mozilla.org/projects/birch",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 21, 
-    "model": "model.repository", 
+    "pk": 21,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "cedar", 
-        "url": "https://hg.mozilla.org/projects/cedar", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "cedar",
+        "url": "https://hg.mozilla.org/projects/cedar",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 22, 
-    "model": "model.repository", 
+    "pk": 22,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "cypress", 
-        "url": "https://hg.mozilla.org/projects/cypress", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "cypress",
+        "url": "https://hg.mozilla.org/projects/cypress",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 23, 
-    "model": "model.repository", 
+    "pk": 23,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "date", 
-        "url": "https://hg.mozilla.org/projects/date", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "date",
+        "url": "https://hg.mozilla.org/projects/date",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 24, 
-    "model": "model.repository", 
+    "pk": 24,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "elm", 
-        "url": "https://hg.mozilla.org/projects/elm", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "elm",
+        "url": "https://hg.mozilla.org/projects/elm",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 25, 
-    "model": "model.repository", 
+    "pk": 25,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "fig", 
-        "url": "https://hg.mozilla.org/projects/fig", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "fig",
+        "url": "https://hg.mozilla.org/projects/fig",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 26, 
-    "model": "model.repository", 
+    "pk": 26,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "gum", 
-        "url": "https://hg.mozilla.org/projects/gum", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "gum",
+        "url": "https://hg.mozilla.org/projects/gum",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 27, 
-    "model": "model.repository", 
+    "pk": 27,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "holly", 
-        "url": "https://hg.mozilla.org/projects/holly", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "holly",
+        "url": "https://hg.mozilla.org/projects/holly",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 28, 
-    "model": "model.repository", 
+    "pk": 28,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "jamun", 
-        "url": "https://hg.mozilla.org/projects/jamun", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "jamun",
+        "url": "https://hg.mozilla.org/projects/jamun",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 29, 
-    "model": "model.repository", 
+    "pk": 29,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "larch", 
-        "url": "https://hg.mozilla.org/projects/larch", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "larch",
+        "url": "https://hg.mozilla.org/projects/larch",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 30, 
-    "model": "model.repository", 
+    "pk": 30,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "maple", 
-        "url": "https://hg.mozilla.org/projects/maple", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "maple",
+        "url": "https://hg.mozilla.org/projects/maple",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 31, 
-    "model": "model.repository", 
+    "pk": 31,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "oak", 
-        "url": "https://hg.mozilla.org/projects/oak", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "oak",
+        "url": "https://hg.mozilla.org/projects/oak",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 32, 
-    "model": "model.repository", 
+    "pk": 32,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "pine", 
-        "url": "https://hg.mozilla.org/projects/pine", 
-        "active_status": "active", 
-        "codebase": "gecko", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "pine",
+        "url": "https://hg.mozilla.org/projects/pine",
+        "active_status": "active",
+        "codebase": "gecko",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 33, 
-    "model": "model.repository", 
+    "pk": 33,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "gaia-master", 
-        "url": "https://hg.mozilla.org/integration/gaia-central", 
-        "active_status": "active", 
-        "codebase": "gaia", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "gaia-master",
+        "url": "https://hg.mozilla.org/integration/gaia-central",
+        "active_status": "active",
+        "codebase": "gaia",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 34, 
-    "model": "model.repository", 
+    "pk": 34,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "gaia-v1-train", 
-        "url": "https://hg.mozilla.org/integration/gaia-v1-train", 
-        "active_status": "active", 
-        "codebase": "gaia", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "gaia-v1-train",
+        "url": "https://hg.mozilla.org/integration/gaia-v1-train",
+        "active_status": "active",
+        "codebase": "gaia",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 35, 
-    "model": "model.repository", 
+    "pk": 35,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "gaia-v1_0_1", 
-        "url": "https://hg.mozilla.org/integration/gaia-1_0_1", 
-        "active_status": "active", 
-        "codebase": "gaia", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "gaia-v1_0_1",
+        "url": "https://hg.mozilla.org/integration/gaia-1_0_1",
+        "active_status": "active",
+        "codebase": "gaia",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 36, 
-    "model": "model.repository", 
+    "pk": 36,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "addon-sdk", 
-        "url": "https://hg.mozilla.org/projects/addon-sdk", 
-        "active_status": "active", 
-        "codebase": "jetpack", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "addon-sdk",
+        "url": "https://hg.mozilla.org/projects/addon-sdk",
+        "active_status": "active",
+        "codebase": "jetpack",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 37, 
-    "model": "model.repository", 
+    "pk": 37,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "comm-central", 
-        "url": "https://hg.mozilla.org/comm-central", 
-        "active_status": "active", 
-        "codebase": "?", 
-        "repository_group": 1, 
+        "dvcs_type": "hg",
+        "name": "comm-central",
+        "url": "https://hg.mozilla.org/comm-central",
+        "active_status": "active",
+        "codebase": "?",
+        "repository_group": 1,
        "description": ""
    }
 },
 {
-    "pk": 38, 
-    "model": "model.repository", 
+    "pk": 38,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "comm-aurora", 
-        "url": "https://hg.mozilla.org/releases/comm-aurora", 
-        "active_status": "active", 
-        "codebase": "?", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "comm-aurora",
+        "url": "https://hg.mozilla.org/releases/comm-aurora",
+        "active_status": "active",
+        "codebase": "?",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 39, 
-    "model": "model.repository", 
+    "pk": 39,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "comm-beta", 
-        "url": "https://hg.mozilla.org/releases/comm-beta", 
-        "active_status": "active", 
-        "codebase": "?", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "comm-beta",
+        "url": "https://hg.mozilla.org/releases/comm-beta",
+        "active_status": "active",
+        "codebase": "?",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 40, 
-    "model": "model.repository", 
+    "pk": 40,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "comm-esr17", 
-        "url": "https://hg.mozilla.org/releases/comm-esr17", 
-        "active_status": "active", 
-        "codebase": "?", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "comm-esr17",
+        "url": "https://hg.mozilla.org/releases/comm-esr17",
+        "active_status": "active",
+        "codebase": "?",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 41, 
-    "model": "model.repository", 
+    "pk": 41,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "comm-release", 
-        "url": "https://hg.mozilla.org/releases/comm-release", 
-        "active_status": "active", 
-        "codebase": "?", 
-        "repository_group": 2, 
+        "dvcs_type": "hg",
+        "name": "comm-release",
+        "url": "https://hg.mozilla.org/releases/comm-release",
+        "active_status": "active",
+        "codebase": "?",
+        "repository_group": 2,
        "description": ""
    }
 },
 {
-    "pk": 42, 
-    "model": "model.repository", 
+    "pk": 42,
+    "model": "model.repository",
    "fields": {
-        "dvcs_type": "hg", 
-        "name": "try-comm-central", 
-        "url": "https://hg.mozilla.org/try-comm-central", 
-        "active_status": "active", 
-        "codebase": "?", 
-        "repository_group": 3, 
+        "dvcs_type": "hg",
+        "name": "try-comm-central",
+        "url": "https://hg.mozilla.org/try-comm-central",
+        "active_status": "active",
+        "codebase": "?",
+        "repository_group": 3,
+        "description": ""
+    }
+},
+{
+    "pk": 43,
+    "model": "model.repository",
+    "fields": {
+        "dvcs_type": "hg",
+        "name": "comm-central-trunk",
+        "url": "https://hg.mozilla.org/comm-central-trunk",
+        "active_status": "active",
+        "codebase": "?",
+        "repository_group": 1,
        "description": ""
    }
 }
--- a/treeherder/model/sql/jobs.json
+++ b/treeherder/model/sql/jobs.json
@ -22,18 +22,41 @@
                `submit_timestamp`,
                `start_timestamp`,
                `end_timestamp`)
-            VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",

            "host":"master_host"
        },
        "set_result_set":{

            "sql":"INSERT INTO `result_set` (
-                `revision_hash`)
+                `revision_hash`
+                )
                VALUES (?)",

            "host":"master_host"
        },
+        "set_revision":{
+
+            "sql":"INSERT INTO `revision` (
+                `revision`,
+                `author`,
+                `comments`,
+                `push_timestamp`,
+                `commit_timestamp`,
+                `repository_id`)
+                VALUES (?,?,?,?,?,?)",
+
+            "host":"master_host"
+        },
+        "set_revision_map":{
+
+            "sql":"INSERT INTO `revision_map` (
+                `revision_id`,
+                `result_set_id`)
+                VALUES (?,?)",
+
+            "host":"master_host"
+        },
        "set_job_log_url":{

            "sql":"INSERT INTO `job_log_url` (
--- a/treeherder/model/sql/reference.json
+++ b/treeherder/model/sql/reference.json
@ -108,12 +108,13 @@
                    )",
            "host":"master_host"
        }
+
    },
    "selects":{
        "get_build_platform_id":{
            "sql": "SELECT `id`
                    FROM `build_platform`
-                    WHERE 
+                    WHERE
                        `os_name` = ?
                    AND `platform` = ?
                    AND `architecture` = ?
@ -123,7 +124,7 @@
        "get_job_group_id":{
            "sql": "SELECT `id`
                    FROM `job_group`
-                    WHERE 
+                    WHERE
                        `name` = ?
                    AND `active_status` = 'active'",
            "host":"read_host"
@ -131,10 +132,10 @@
        "get_job_type_id":{
            "sql": "SELECT jt.`id`
                    FROM `job_type` jt
-                    INNER JOIN `job_group` jg 
+                    INNER JOIN `job_group` jg
                        ON jg.id = jt.job_group_id
-                    WHERE 
-                        jt.`name` = ? 
+                    WHERE
+                        jt.`name` = ?
                    AND jg.`name` = ?
                    AND jt.`active_status` = 'active'
                    AND jg.`active_status` = 'active'",
@ -143,7 +144,7 @@
        "get_machine_id":{
            "sql": "SELECT `id`
                    FROM `machine`
-                    WHERE 
+                    WHERE
                        `name` = ?
                    AND `active_status` = 'active'",
            "host":"read_host"
@ -151,7 +152,7 @@
        "get_machine_platform_id":{
            "sql": "SELECT `id`
                    FROM `machine_platform`
-                    WHERE 
+                    WHERE
                        `os_name` = ?
                    AND `platform` = ?
                    AND `architecture` = ?
@ -161,7 +162,7 @@
        "get_option_id":{
            "sql": "SELECT `id`
                    FROM `option`
-                    WHERE 
+                    WHERE
                        `name` = ?
                    AND `active_status` = 'active'",
            "host":"read_host"
@ -174,24 +175,30 @@
        "get_option_collection_id":{
            "sql": "SELECT `option_collection_id`
                    FROM `option_collection` oc
-                    INNER JOIN `option` o on o.id = oc.option_id 
+                    INNER JOIN `option` o on o.id = oc.option_id
                    GROUP BY `option_collection_id`
-                    HAVING 
+                    HAVING
                        GROUP_CONCAT(o.name ORDER BY o.name ASC SEPARATOR ',') = ?",
            "host":"read_host"
        },
        "get_product_id":{
            "sql": "SELECT `id`
                    FROM `product`
-                    WHERE 
+                    WHERE
                        `name` = ?
                    AND `active_status` = 'active'",
            "host":"read_host"
        },
+        "get_repository_id":{
+            "sql": "SELECT `id`
+                    FROM `repository`
+                    WHERE `name` = ?",
+            "host":"read_host"
+        },
        "get_repository_version_id":{
            "sql": "SELECT `id`
                    FROM `repository_version`
-                    WHERE 
+                    WHERE
                        `repository_id` = ?
                    AND `version` = ?
                    AND `active_status` = 'active'",
--- a/treeherder/model/sql/template_schema/project_jobs_1.sql.tmpl
+++ b/treeherder/model/sql/template_schema/project_jobs_1.sql.tmpl
@ -298,7 +298,7 @@ DROP TABLE IF EXISTS `revision`;
 *
 *  revision - The revision string associated with the repository
 *  author - The author associated with the revision
- *  comment - The comments associated with the revision
+ *  comments - The comments associated with the revision
 *  push_timestamp - Timestamp associated with the push to the test system
 *  commit_timestamp - Timestamp associated with the commit
 **************************/
@ -306,7 +306,7 @@ CREATE TABLE `revision` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
  `revision` varchar(50) COLLATE utf8_bin NOT NULL,
  `author` varchar(50) COLLATE utf8_bin NOT NULL,
-  `comment` text DEFAULT NULL,
+  `comments` text DEFAULT NULL,
  `push_timestamp` int(11) unsigned NOT NULL,
  `commit_timestamp` int(11) unsigned NOT NULL,
  `repository_id` int(11) unsigned NOT NULL,