diff --git a/tests/model/derived/refdata.py b/tests/model/derived/refdata.py index e9a9c3c48..829b609ac 100644 --- a/tests/model/derived/refdata.py +++ b/tests/model/derived/refdata.py @@ -3,6 +3,7 @@ import pytest from django.conf import settings from treeherder.model.derived import RefDataManager from datasource.bases.BaseHub import BaseHub +from treeherder.model.models import RepositoryGroup, Repository @pytest.fixture(scope="module") @@ -23,36 +24,144 @@ def refdata(): return refdata -def test_build_platform_manager(refdata): - # platform'Fedora 12', 'os_name': 'linux', 'architecture': 'x86_64' +@pytest.fixture(scope="module") +def repository_id(): + repo_group = RepositoryGroup( + name='mygroup', + description='fill me', + active_status='active' + ) + repo_group.save() + repo = Repository.objects.create( + repository_group=repo_group, + name='myrepo', + type='git', + url='https://github.com/mozilla/treeherder-service.git', + branch='mybranch', + project_name='myproject', + purpose='development', + active_status='active' + ) + return repo.id - build_platform_id = refdata.get_or_create_build_platform( - 'linux', - 'Fedora 12', - 'x86_64',) + +test_params = [ + { + 'func': 'get_or_create_build_platform', + 'input': ['linux', 'Fedora 12', 'x86_64'], + 'test_proc': 'refdata_test.selects.test_build_platform', + 'expected': [{ + 'os_name': 'linux', + 'platform': 'Fedora 12', + 'architecture': 'x86_64', + 'active_status': 'active' + }] + }, + { + 'func': 'get_or_create_job_group', + 'input': ['mygroup'], + 'test_proc': 'refdata_test.selects.test_job_group', + 'expected': [{ + 'symbol': 'fill me', + 'name': 'mygroup', + 'description': 'fill me', + 'active_status': 'active' + }] + }, + { + 'input': ['myname', 'mygroup'], + 'func': 'get_or_create_job_type', + 'test_proc': 'refdata_test.selects.test_job_type', + 'expected': [{ + 'symbol': 'fill me', + 'name': 'myname', + 'group': 'mygroup', + 'description': 'fill me', + 'active_status': 'active' + }] + }, + { + 'input': ['myname', 1366290144.07455], + 'func': 'get_or_create_machine', + 'test_proc': 'refdata_test.selects.test_machine', + 'expected': [{ + 'name': 'myname', + 'first_timestamp': 1366290144, + 'last_timestamp': 1366290144, + 'active_status': 'active' + }] + }, + { + 'func': 'get_or_create_machine_platform', + 'input': ['linux', 'Fedora 12', 'x86_64'], + 'test_proc': 'refdata_test.selects.test_machine_platform', + 'expected': [{ + 'os_name': 'linux', + 'platform': 'Fedora 12', + 'architecture': 'x86_64', + 'active_status': 'active' + }] + }, + { + 'func': 'get_or_create_option', + 'input': ['myoption'], + 'test_proc': 'refdata_test.selects.test_option', + 'expected': [{ + 'name': 'myoption', + 'description': 'fill me', + 'active_status': 'active' + }] + }, + { + 'func': 'get_or_create_option_collection', + 'input': [['option1', 'option2']], + 'test_proc': 'refdata_test.selects.test_option_collection', + 'expected': [{'name': 'option1'}, {'name': 'option2'}] + }, + { + 'func': 'get_or_create_product', + 'input': ['myproduct'], + 'test_proc': 'refdata_test.selects.test_product', + 'expected': [{ + 'name': 'myproduct', + 'description': 'fill me', + 'active_status': 'active' + }] + } + +] + + +@pytest.mark.parametrize(("params"), test_params) +def test_refdata_manager(refdata, params): + """test get_or_create methods produce the right content""" + + #call the right refdata method based on params + id = getattr(refdata, params['func'])(*params['input']) row_data = refdata.dhub.execute( - proc="refdata_test.selects.test_build_platform_manager", - placeholders=[build_platform_id] - )[0] - - assert row_data["os_name"] == 'linux' - assert row_data["platform"] == 'Fedora 12' - assert row_data["architecture"] == 'x86_64' + proc=params["test_proc"], + placeholders=[id] + ) + for i, row in enumerate(row_data): + for k, v in params['expected'][i].items(): + assert row[k] == v -def test_job_group_manager(refdata): - # `name` = mygroup +# some tests just don't fit into the standard layout +# def test_repository_version_creation(refdata, repository_id): - job_group_id = refdata.get_or_create_job_group('mygroup') +# id = refdata.get_or_create_repository_version( +# repository_id, +# 'v1', +# 1366290144.07455) - row_data = refdata.dhub.execute( - proc="refdata_test.selects.test_job_group_manager", - placeholders=[job_group_id] - )[0] - - assert row_data["symbol"] == 'fill me' - assert row_data["name"] == 'mygroup' - assert row_data["description"] == 'fill me' - assert row_data["active_status"] == 'active' + # row_data = refdata.dhub.execute( + # proc=params[refdata_test.selects.test_repository_version], + # placeholders=[id] + # )[0] + # assert row[repository_id] == 1 + # assert row[version] == 'v1' + # assert row[version_timestamp] == 1366290144 + # assert row[active_status] == 'active' diff --git a/tests/model/derived/refdata_test.json b/tests/model/derived/refdata_test.json index 0c1be1cbd..d7ac86215 100644 --- a/tests/model/derived/refdata_test.json +++ b/tests/model/derived/refdata_test.json @@ -1,19 +1,78 @@ { "inserts":{}, "selects":{ - "test_build_platform_manager":{ - "sql": "SELECT `platform`, `os_name`, `architecture` + "test_build_platform":{ + "sql": "SELECT `platform`, `os_name`, `architecture`, `active_status` FROM `build_platform` WHERE `id` = ?", "host":"read_host" }, - "test_job_group_manager":{ + "test_job_group":{ "sql": "SELECT `symbol`, `name`, `description`, `active_status` FROM `job_group` WHERE `id` = ?", "host":"read_host" + }, + "test_job_type":{ + "sql": "SELECT + jg.name AS 'group', + jt.`symbol`, + jt.`name`, + jt.`description`, + jt.`active_status` + FROM `job_type` jt + INNER JOIN `job_group` jg + on jg.id = jt.job_group_id + WHERE + jt.`id` = ?", + "host":"read_host" + }, + "test_machine":{ + "sql": "SELECT `name`, `first_timestamp`, `last_timestamp`, `active_status` + FROM `machine` + WHERE + `id` = ?", + "host":"read_host" + }, + "test_machine_platform":{ + "sql": "SELECT `platform`, `os_name`, `architecture`, `active_status` + FROM `machine_platform` + WHERE + `id` = ?", + "host":"read_host" + }, + "test_option":{ + "sql": "SELECT `name`, `description`, `active_status` + FROM `option` + WHERE + `id` = ?", + "host":"read_host" + }, + "test_option_collection":{ + "sql": "SELECT o.`name` + FROM `option` o + INNER JOIN `option_collection` oc + ON o.`id` = oc.`option_id` + WHERE + oc.`id` = ? + ORDER BY o.`name`", + "host":"read_host" + }, + "test_product":{ + "sql": "SELECT `name`, `description`, `active_status` + FROM `product` + WHERE + `id` = ?", + "host":"read_host" + }, + "test_repository_version":{ + "sql": "SELECT `repository_id`, `version`, `version_timestamp`, `active_status` + FROM `repository_version` + WHERE + `id` = ?", + "host":"read_host" } } } diff --git a/treeherder/model/derived/refdata.py b/treeherder/model/derived/refdata.py index 2dac57f1c..a82578f6f 100644 --- a/treeherder/model/derived/refdata.py +++ b/treeherder/model/derived/refdata.py @@ -90,17 +90,19 @@ class RefDataManager(object): def get_or_create_job_type(self, name, group): + group_id = self.get_or_create_job_group(group) + self.dhub.execute( proc='reference.inserts.create_job_type', placeholders=[ + group_id, name, - group, - name, - group + group_id, + name ], debug_show=self.DEBUG) - return self.get_job_type_id(name) + return self.get_job_type_id(name, group) def get_machine_id(self, name): @@ -187,13 +189,17 @@ class RefDataManager(object): debug_show=self.DEBUG, return_type='iter') - return id_iter.get_column_data('id') + return id_iter.get_column_data('option_collection_id') - def get_last_collection_id(self): - self.dhub.execute( - proc='reference.selects.get_last_collection_id', + def get_max_collection_id(self): + """returns the max collection id available""" + max_id = self.dhub.execute( + proc='reference.selects.get_max_collection_id', placeholders=[], - debug_show=self.DEBUG) + debug_show=self.DEBUG, + return_type='tuple') + + return max_id[0]['max_id'] or 0 def get_or_create_option_collection(self, options): @@ -202,7 +208,7 @@ class RefDataManager(object): if not id: #retrieve the last collection - option_collection_id = self.get_last_collection_id() + 1 + option_collection_id = self.get_max_collection_id() + 1 for option in options: #create an option if it doesn't exist diff --git a/treeherder/model/sql/reference.json b/treeherder/model/sql/reference.json index 2d16c3ef3..691be2d33 100644 --- a/treeherder/model/sql/reference.json +++ b/treeherder/model/sql/reference.json @@ -56,14 +56,14 @@ WHERE NOT EXISTS ( SELECT `os_name`, `platform`, `architecture` FROM `machine_platform` - WEHRE `os_name` = ? AND platform` = ? AND architecture` = ? + WHERE `os_name` = ? AND `platform` = ? AND `architecture` = ? )", "host":"master_host" }, "create_option":{ "sql":"INSERT INTO `option` (`name`, `description`, `active_status`) - SELECT ?, 'fill me', ?, 'active' + SELECT ?, 'fill me', 'active' FROM DUAL WHERE NOT EXISTS ( SELECT `name` @@ -92,7 +92,7 @@ WHERE NOT EXISTS ( SELECT `name` FROM `product` - WHERE `name = ? + WHERE `name` = ? )", "host":"master_host" }, @@ -166,8 +166,8 @@ AND `active_status` = 'active'", "host":"read_host" }, - "get_last_collection_id":{ - "sql": "SELECT MAX(`collection_id`) + "get_max_collection_id":{ + "sql": "SELECT MAX(`option_collection_id`) as 'max_id' FROM `option_collection`", "host":"read_host" }, @@ -184,7 +184,7 @@ "sql": "SELECT `id` FROM `product` WHERE - `name = ? + `name` = ? AND `active_status` = 'active'", "host":"read_host" },