add several refdata tests

tests/model/derived/refdata.py

@@ -3,6 +3,7 @@ import pytest
 from django.conf import settings
 from treeherder.model.derived import RefDataManager
 from datasource.bases.BaseHub import BaseHub
+from treeherder.model.models import RepositoryGroup, Repository
 
 
 @pytest.fixture(scope="module")
@@ -23,36 +24,144 @@ def refdata():
     return refdata
 
 
-def test_build_platform_manager(refdata):
+@pytest.fixture(scope="module")
-    # platform'Fedora 12', 'os_name': 'linux', 'architecture': 'x86_64'
+def repository_id():
+    repo_group = RepositoryGroup(
+        name='mygroup',
+        description='fill me',
+        active_status='active'
+    )
+    repo_group.save()
+    repo = Repository.objects.create(
+        repository_group=repo_group,
+        name='myrepo',
+        type='git',
+        url='https://github.com/mozilla/treeherder-service.git',
+        branch='mybranch',
+        project_name='myproject',
+        purpose='development',
+        active_status='active'
+    )
+    return repo.id
 
-    build_platform_id = refdata.get_or_create_build_platform(
+
-        'linux',
+test_params = [
-        'Fedora 12',
+    {
-        'x86_64',)
+        'func': 'get_or_create_build_platform',
+        'input': ['linux', 'Fedora 12', 'x86_64'],
+        'test_proc': 'refdata_test.selects.test_build_platform',
+        'expected': [{
+            'os_name': 'linux',
+            'platform': 'Fedora 12',
+            'architecture': 'x86_64',
+            'active_status': 'active'
+        }]
+    },
+    {
+        'func': 'get_or_create_job_group',
+        'input': ['mygroup'],
+        'test_proc': 'refdata_test.selects.test_job_group',
+        'expected': [{
+            'symbol': 'fill me',
+            'name': 'mygroup',
+            'description': 'fill me',
+            'active_status': 'active'
+        }]
+    },
+    {
+        'input': ['myname', 'mygroup'],
+        'func': 'get_or_create_job_type',
+        'test_proc': 'refdata_test.selects.test_job_type',
+        'expected': [{
+            'symbol': 'fill me',
+            'name': 'myname',
+            'group': 'mygroup',
+            'description': 'fill me',
+            'active_status': 'active'
+        }]
+    },
+    {
+        'input': ['myname', 1366290144.07455],
+        'func': 'get_or_create_machine',
+        'test_proc': 'refdata_test.selects.test_machine',
+        'expected': [{
+            'name': 'myname',
+            'first_timestamp': 1366290144,
+            'last_timestamp': 1366290144,
+            'active_status': 'active'
+        }]
+    },
+    {
+        'func': 'get_or_create_machine_platform',
+        'input': ['linux', 'Fedora 12', 'x86_64'],
+        'test_proc': 'refdata_test.selects.test_machine_platform',
+        'expected': [{
+            'os_name': 'linux',
+            'platform': 'Fedora 12',
+            'architecture': 'x86_64',
+            'active_status': 'active'
+        }]
+    },
+    {
+        'func': 'get_or_create_option',
+        'input': ['myoption'],
+        'test_proc': 'refdata_test.selects.test_option',
+        'expected': [{
+            'name': 'myoption',
+            'description': 'fill me',
+            'active_status': 'active'
+        }]
+    },
+    {
+        'func': 'get_or_create_option_collection',
+        'input': [['option1', 'option2']],
+        'test_proc': 'refdata_test.selects.test_option_collection',
+        'expected': [{'name': 'option1'}, {'name': 'option2'}]
+    },
+    {
+        'func': 'get_or_create_product',
+        'input': ['myproduct'],
+        'test_proc': 'refdata_test.selects.test_product',
+        'expected': [{
+            'name': 'myproduct',
+            'description': 'fill me',
+            'active_status': 'active'
+        }]
+    }
+
+]
+
+
+@pytest.mark.parametrize(("params"), test_params)
+def test_refdata_manager(refdata, params):
+    """test get_or_create methods produce the right content"""
+
+    #call the right refdata method based on params
+    id = getattr(refdata, params['func'])(*params['input'])
 
     row_data = refdata.dhub.execute(
-        proc="refdata_test.selects.test_build_platform_manager",
+        proc=params["test_proc"],
-        placeholders=[build_platform_id]
+        placeholders=[id]
-    )[0]
+    )
-
+    for i, row in enumerate(row_data):
-    assert row_data["os_name"] == 'linux'
+        for k, v in params['expected'][i].items():
-    assert row_data["platform"] == 'Fedora 12'
+            assert row[k] == v
-    assert row_data["architecture"] == 'x86_64'
 
 
-def test_job_group_manager(refdata):
+# some tests just don't fit into the standard layout
-    # `name` = mygroup
+# def test_repository_version_creation(refdata, repository_id):
 
-    job_group_id = refdata.get_or_create_job_group('mygroup')
+#     id = refdata.get_or_create_repository_version(
+#         repository_id,
+#         'v1',
+#         1366290144.07455)
 
-    row_data = refdata.dhub.execute(
+    # row_data = refdata.dhub.execute(
-        proc="refdata_test.selects.test_job_group_manager",
+    #     proc=params[refdata_test.selects.test_repository_version],
-        placeholders=[job_group_id]
+    #     placeholders=[id]
-    )[0]
+    # )[0]
-
-    assert row_data["symbol"] == 'fill me'
-    assert row_data["name"] == 'mygroup'
-    assert row_data["description"] == 'fill me'
-    assert row_data["active_status"] == 'active'
 
+    # assert row[repository_id] == 1
+    # assert row[version] == 'v1'
+    # assert row[version_timestamp] == 1366290144
+    # assert row[active_status] == 'active'

tests/model/derived/refdata_test.json

@@ -1,19 +1,78 @@
 {
     "inserts":{},
     "selects":{
-        "test_build_platform_manager":{
+        "test_build_platform":{
-            "sql": "SELECT `platform`, `os_name`, `architecture`
+            "sql": "SELECT `platform`, `os_name`, `architecture`, `active_status`
                     FROM `build_platform`
                     WHERE 
                         `id` = ?",
             "host":"read_host"
         },
-        "test_job_group_manager":{
+        "test_job_group":{
             "sql": "SELECT `symbol`, `name`, `description`, `active_status`
                     FROM `job_group`
                     WHERE 
                         `id` = ?",
             "host":"read_host"
+        },
+        "test_job_type":{
+            "sql": "SELECT
+                        jg.name AS 'group',
+                        jt.`symbol`,
+                        jt.`name`,
+                        jt.`description`,
+                        jt.`active_status`
+                    FROM `job_type` jt
+                    INNER JOIN `job_group` jg
+                        on jg.id = jt.job_group_id
+                    WHERE 
+                        jt.`id` = ?",
+            "host":"read_host"
+        },
+        "test_machine":{
+            "sql": "SELECT `name`, `first_timestamp`, `last_timestamp`, `active_status`
+                    FROM `machine`
+                    WHERE
+                        `id` = ?",
+            "host":"read_host"
+        },
+        "test_machine_platform":{
+            "sql": "SELECT `platform`, `os_name`, `architecture`, `active_status`
+                    FROM `machine_platform`
+                    WHERE 
+                        `id` = ?",
+            "host":"read_host"
+        },
+        "test_option":{
+            "sql": "SELECT `name`, `description`, `active_status`
+                    FROM `option`
+                    WHERE 
+                        `id` = ?",
+            "host":"read_host"
+        },
+        "test_option_collection":{
+            "sql": "SELECT o.`name`
+                    FROM `option` o
+                    INNER JOIN `option_collection` oc
+                        ON o.`id` = oc.`option_id`
+                    WHERE 
+                        oc.`id` = ?
+                    ORDER BY o.`name`",
+            "host":"read_host"
+        },
+        "test_product":{
+            "sql": "SELECT `name`, `description`, `active_status`
+                    FROM `product`
+                    WHERE 
+                        `id` = ?",
+            "host":"read_host"
+        },
+        "test_repository_version":{
+            "sql": "SELECT `repository_id`, `version`, `version_timestamp`, `active_status`
+                    FROM `repository_version`
+                    WHERE 
+                        `id` = ?",
+            "host":"read_host"
         }
     }
 }

treeherder/model/derived/refdata.py

@@ -90,17 +90,19 @@ class RefDataManager(object):
 
     def get_or_create_job_type(self, name, group):
 
+        group_id = self.get_or_create_job_group(group)
+
         self.dhub.execute(
             proc='reference.inserts.create_job_type',
             placeholders=[
+                group_id,
                 name,
-                group,
+                group_id,
-                name,
+                name
-                group
             ],
             debug_show=self.DEBUG)
 
-        return self.get_job_type_id(name)
+        return self.get_job_type_id(name, group)
 
     def get_machine_id(self, name):
 
@@ -187,13 +189,17 @@ class RefDataManager(object):
             debug_show=self.DEBUG,
             return_type='iter')
 
-        return id_iter.get_column_data('id')
+        return id_iter.get_column_data('option_collection_id')
 
-    def get_last_collection_id(self):
+    def get_max_collection_id(self):
-        self.dhub.execute(
+        """returns the max collection id available"""
-            proc='reference.selects.get_last_collection_id',
+        max_id = self.dhub.execute(
+            proc='reference.selects.get_max_collection_id',
             placeholders=[],
-            debug_show=self.DEBUG)
+            debug_show=self.DEBUG,
+            return_type='tuple')
+
+        return max_id[0]['max_id'] or 0
 
     def get_or_create_option_collection(self, options):
 
@@ -202,7 +208,7 @@ class RefDataManager(object):
         if not id:
 
             #retrieve the last collection
-            option_collection_id = self.get_last_collection_id() + 1
+            option_collection_id = self.get_max_collection_id() + 1
             for option in options:
 
                 #create an option if it doesn't exist

treeherder/model/sql/reference.json

@@ -56,14 +56,14 @@
                     WHERE NOT EXISTS (
                     SELECT `os_name`, `platform`, `architecture`
                     FROM `machine_platform`
-                    WEHRE `os_name` = ? AND platform` = ? AND architecture` = ?
+                    WHERE `os_name` = ? AND `platform` = ? AND `architecture` = ?
                     )",
             "host":"master_host"
         },
 
         "create_option":{
             "sql":"INSERT INTO `option` (`name`, `description`, `active_status`)
-                   SELECT ?, 'fill me', ?, 'active'
+                   SELECT ?, 'fill me', 'active'
                     FROM DUAL
                     WHERE NOT EXISTS (
                     SELECT `name`
@@ -92,7 +92,7 @@
                     WHERE NOT EXISTS (
                     SELECT `name`
                     FROM `product`
-                    WHERE `name = ?
+                    WHERE `name` = ?
                     )",
             "host":"master_host"
         },
@@ -166,8 +166,8 @@
                     AND `active_status` = 'active'",
             "host":"read_host"
         },
-        "get_last_collection_id":{
+        "get_max_collection_id":{
-            "sql": "SELECT MAX(`collection_id`)
+            "sql": "SELECT MAX(`option_collection_id`) as 'max_id'
                     FROM `option_collection`",
             "host":"read_host"
         },
@@ -184,7 +184,7 @@
             "sql": "SELECT `id`
                     FROM `product`
                     WHERE 
-                        `name = ?
+                        `name` = ?
                     AND `active_status` = 'active'",
             "host":"read_host"
         },