mozilla
/
treeherder
зеркало из https://github.com/mozilla/treeherder.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

objectstore tests are now passing

This commit is contained in:
Cameron Dawson 2013-04-19 18:04:00 -07:00
Родитель 58d3591315 6d43a2fccb
Коммит 1f0fb9a9b5
8 изменённых файлов: 220 добавлений и 55 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

5
tests/conftest.py
Просмотреть файл

@ -1,6 +1,7 @@
import os
from os.path import dirname
import sys
from django.core.management import call_command
import pytest
from django.conf import settings
@ -160,7 +161,7 @@ def jobs_ds():
project=settings.DATABASES["default"]["TEST_NAME"],
dataset=1,
contenttype="jobs",
host=settings.DATABASES['default']['HOST'],
host="localhost",
)
@ -172,5 +173,5 @@ def objectstore_ds():
project=settings.DATABASES["default"]["TEST_NAME"],
dataset=1,
contenttype="objectstore",
host=settings.DATABASES['default']['HOST'],
host="localhost",
)

145
tests/model/derived/refdata.py
Просмотреть файл

@ -3,6 +3,7 @@ import pytest
from django.conf import settings
from treeherder.model.derived import RefDataManager
from datasource.bases.BaseHub import BaseHub
from treeherder.model.models import RepositoryGroup, Repository
@pytest.fixture(scope="module")
@ -23,35 +24,131 @@ def refdata():
return refdata
def test_build_platform_manager(refdata):
# platform'Fedora 12', 'os_name': 'linux', 'architecture': 'x86_64'
@pytest.fixture(scope="module")
def repository_id():
repo_group = RepositoryGroup(
name='mygroup',
description='fill me',
active_status='active'
)
repo_group.save()
repo = Repository.objects.create(
repository_group=repo_group,
name='myrepo',
type='git',
url='https://github.com/mozilla/treeherder-service.git',
branch='mybranch',
project_name='myproject',
purpose='development',
active_status='active'
)
return repo.id
build_platform_id = refdata.get_or_create_build_platform(
'linux',
'Fedora 12',
'x86_64',)
test_params = [
{
'func': 'get_or_create_build_platform',
'input': ['linux', 'Fedora 12', 'x86_64'],
'test_proc': 'refdata_test.selects.test_build_platform',
'expected': [{
'os_name': 'linux',
'platform': 'Fedora 12',
'architecture': 'x86_64',
'active_status': 'active'
}]
},
{
'func': 'get_or_create_job_group',
'input': ['mygroup'],
'test_proc': 'refdata_test.selects.test_job_group',
'expected': [{
'symbol': 'fill me',
'name': 'mygroup',
'description': 'fill me',
'active_status': 'active'
}]
},
{
'input': ['myname', 'mygroup'],
'func': 'get_or_create_job_type',
'test_proc': 'refdata_test.selects.test_job_type',
'expected': [{
'symbol': 'fill me',
'name': 'myname',
'group': 'mygroup',
'description': 'fill me',
'active_status': 'active'
}]
},
{
'input': ['myname', 1366290144.07455],
'func': 'get_or_create_machine',
'test_proc': 'refdata_test.selects.test_machine',
'expected': [{
'name': 'myname',
'first_timestamp': 1366290144,
'last_timestamp': 1366290144,
'active_status': 'active'
}]
},
{
'func': 'get_or_create_machine_platform',
'input': ['linux', 'Fedora 12', 'x86_64'],
'test_proc': 'refdata_test.selects.test_machine_platform',
'expected': [{
'os_name': 'linux',
'platform': 'Fedora 12',
'architecture': 'x86_64',
'active_status': 'active'
}]
},
{
'func': 'get_or_create_option',
'input': ['myoption'],
'test_proc': 'refdata_test.selects.test_option',
'expected': [{
'name': 'myoption',
'description': 'fill me',
'active_status': 'active'
}]
},
{
'func': 'get_or_create_option_collection',
'input': [['option1', 'option2']],
'test_proc': 'refdata_test.selects.test_option_collection',
'expected': [{'name': 'option1'}, {'name': 'option2'}]
},
{
'func': 'get_or_create_product',
'input': ['myproduct'],
'test_proc': 'refdata_test.selects.test_product',
'expected': [{
'name': 'myproduct',
'description': 'fill me',
'active_status': 'active'
}]
}
]
@pytest.mark.parametrize(("params"), test_params)
def test_refdata_manager(refdata, params):
"""test get_or_create methods produce the right content"""
#call the right refdata method based on params
id = getattr(refdata, params['func'])(*params['input'])
row_data = refdata.dhub.execute(
proc="refdata_test.selects.test_build_platform_manager",
placeholders=[build_platform_id]
)[0]
assert row_data["os_name"] == 'linux'
assert row_data["platform"] == 'Fedora 12'
assert row_data["architecture"] == 'x86_64'
def test_job_group_manager(refdata):
# `name` = mygroup
job_group_id = refdata.get_or_create_job_group('mygroup')
row_data = refdata.dhub.execute(
proc="refdata_test.selects.test_job_group_manager",
placeholders=[job_group_id]
)[0]
proc=params["test_proc"],
placeholders=[id]
)
for i, row in enumerate(row_data):
for k, v in params['expected'][i].items():
assert row[k] == v
assert row_data["symbol"] == 'fill me'
assert row_data["name"] == 'mygroup'
assert row_data["description"] == 'fill me'
assert row_data["active_status"] == 'active'

65
tests/model/derived/refdata_test.json
Просмотреть файл

@ -1,19 +1,78 @@
{
"inserts":{},
"selects":{
"test_build_platform_manager":{
"sql": "SELECT `platform`, `os_name`, `architecture`
"test_build_platform":{
"sql": "SELECT `platform`, `os_name`, `architecture`, `active_status`
FROM `build_platform`
WHERE
`id` = ?",
"host":"read_host"
},
"test_job_group_manager":{
"test_job_group":{
"sql": "SELECT `symbol`, `name`, `description`, `active_status`
FROM `job_group`
WHERE
`id` = ?",
"host":"read_host"
},
"test_job_type":{
"sql": "SELECT
jg.name AS 'group',
jt.`symbol`,
jt.`name`,
jt.`description`,
jt.`active_status`
FROM `job_type` jt
INNER JOIN `job_group` jg
on jg.id = jt.job_group_id
WHERE
jt.`id` = ?",
"host":"read_host"
},
"test_machine":{
"sql": "SELECT `name`, `first_timestamp`, `last_timestamp`, `active_status`
FROM `machine`
WHERE
`id` = ?",
"host":"read_host"
},
"test_machine_platform":{
"sql": "SELECT `platform`, `os_name`, `architecture`, `active_status`
FROM `machine_platform`
WHERE
`id` = ?",
"host":"read_host"
},
"test_option":{
"sql": "SELECT `name`, `description`, `active_status`
FROM `option`
WHERE
`id` = ?",
"host":"read_host"
},
"test_option_collection":{
"sql": "SELECT o.`name`
FROM `option` o
INNER JOIN `option_collection` oc
ON o.`id` = oc.`option_id`
WHERE
oc.`id` = ?
ORDER BY o.`name`",
"host":"read_host"
},
"test_product":{
"sql": "SELECT `name`, `description`, `active_status`
FROM `product`
WHERE
`id` = ?",
"host":"read_host"
},
"test_repository_version":{
"sql": "SELECT `repository_id`, `version`, `version_timestamp`, `active_status`
FROM `repository_version`
WHERE
`id` = ?",
"host":"read_host"
}
}
}

3
tests/model/derived/test_objectstore_model.py
Просмотреть файл

@ -99,9 +99,6 @@ def test_process_objects(jm):
loading_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
proc="objectstore_test.counts.loading")[0]["loading_count"]
import time
time.sleep(60)
assert complete_count == 2
assert loading_count == 0
assert date_set.issubset(expected_dates)

14
treeherder/model/derived/jobs.py
Просмотреть файл

@ -183,10 +183,16 @@ class JobsModel(TreeherderModelBase):
for job in data["jobs"]:
build_platform_id = rdm.get_or_create_build_platform(
job["build_platform"])
job["build_platform"]["os_name"],
job["build_platform"]["platform"],
job["build_platform"]["architecture"],
)
machine_platform_id = rdm.get_or_create_machine_platform(
job["machine_platform"])
job["machine_platform"]["os_name"],
job["machine_platform"]["platform"],
job["machine_platform"]["architecture"],
)
machine_id = rdm.get_or_create_machine(
job["machine"],
@ -214,7 +220,7 @@ class JobsModel(TreeherderModelBase):
result_set_id = self._set_result_set(data["revision_hash"])
job_id = self._set_job_data(
data,
job,
result_set_id,
build_platform_id,
machine_platform_id,
@ -257,7 +263,7 @@ class JobsModel(TreeherderModelBase):
end_timestamp = data["end_timestamp"]
# @@@ need better error message here
except ValueError:
except ValueError as e:
raise JobDataError(
"Return meaningful error here; not this rubbish.")

26
treeherder/model/derived/refdata.py
Просмотреть файл

@ -89,17 +89,19 @@ class RefDataManager(object):
def get_or_create_job_type(self, name, group):
group_id = self.get_or_create_job_group(group)
self.dhub.execute(
proc='reference.inserts.create_job_type',
placeholders=[
group_id,
name,
group,
name,
group
group_id,
name
],
debug_show=self.DEBUG)
return self.get_job_type_id(name)
return self.get_job_type_id(name, group)
def get_machine_id(self, name):
@ -186,13 +188,17 @@ class RefDataManager(object):
debug_show=self.DEBUG,
return_type='iter')
return id_iter.get_column_data('id')
return id_iter.get_column_data('option_collection_id')
def get_last_collection_id(self):
self.dhub.execute(
proc='reference.selects.get_last_collection_id',
def get_max_collection_id(self):
"""returns the max collection id available"""
max_id = self.dhub.execute(
proc='reference.selects.get_max_collection_id',
placeholders=[],
debug_show=self.DEBUG)
debug_show=self.DEBUG,
return_type='tuple')
return max_id[0]['max_id'] or 0
def get_or_create_option_collection(self, options):
@ -201,7 +207,7 @@ class RefDataManager(object):
if not id:
#retrieve the last collection
option_collection_id = self.get_last_collection_id() + 1
option_collection_id = self.get_max_collection_id() + 1
for option in options:
#create an option if it doesn't exist

5
treeherder/model/sql/jobs.json
Просмотреть файл

@ -48,9 +48,8 @@
`state`,
`submit_timestamp`,
`start_timestamp`,
`end_timestamp`,
`active_status`)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
`end_timestamp`)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
"host":"master_host"
},

12
treeherder/model/sql/reference.json
Просмотреть файл

@ -56,14 +56,14 @@
WHERE NOT EXISTS (
SELECT `os_name`, `platform`, `architecture`
FROM `machine_platform`
WEHRE `os_name` = ? AND platform` = ? AND architecture` = ?
WHERE `os_name` = ? AND `platform` = ? AND `architecture` = ?
)",
"host":"master_host"
},
"create_option":{
"sql":"INSERT INTO `option` (`name`, `description`, `active_status`)
SELECT ?, 'fill me', ?, 'active'
SELECT ?, 'fill me', 'active'
FROM DUAL
WHERE NOT EXISTS (
SELECT `name`
@ -92,7 +92,7 @@
WHERE NOT EXISTS (
SELECT `name`
FROM `product`
WHERE `name = ?
WHERE `name` = ?
)",
"host":"master_host"
},
@ -166,8 +166,8 @@
AND `active_status` = 'active'",
"host":"read_host"
},
"get_last_collection_id":{
"sql": "SELECT MAX(`collection_id`)
"get_max_collection_id":{
"sql": "SELECT MAX(`option_collection_id`) as 'max_id'
FROM `option_collection`",
"host":"read_host"
},
@ -184,7 +184,7 @@
"sql": "SELECT `id`
FROM `product`
WHERE
`name = ?
`name` = ?
AND `active_status` = 'active'",
"host":"read_host"
},