mozilla
/
treeherder
зеркало из https://github.com/mozilla/treeherder.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

fix RefDataManager and add firts working test on it

This commit is contained in:
mdoglio 2013-04-17 17:13:24 +01:00
Родитель 4f6921caa1
Коммит dcb703dcf5
12 изменённых файлов: 136 добавлений и 54 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

0
tests/__init__.py Normal file
Просмотреть файл

24
tests/conftest.py
Просмотреть файл

@ -92,3 +92,27 @@ def increment_cache_key_prefix():
key_prefix_counter = 0
cache.set(prefix_counter_cache_key, key_prefix_counter)
cache.key_prefix = "t{0}".format(key_prefix_counter)
@pytest.fixture(scope='session')
def jobs_ds():
from django.conf import settings
from treeherder.model.models import Datasource
return Datasource.objects.create(
project=settings.DATABASES["default"]["TEST_NAME"],
dataset=1,
contenttype="jobs",
host="localhost",
)
@pytest.fixture(scope='session')
def objectstore_ds():
from django.conf import settings
from treeherder.model.models import Datasource
return Datasource.objects.create(
project=settings.DATABASES["default"]["TEST_NAME"],
dataset=1,
contenttype="objectstore",
host="localhost",
)

0
tests/model/__init__.py Normal file
Просмотреть файл

0
tests/model/derived/__init__.py Normal file
Просмотреть файл

41
tests/model/derived/refdata.py Normal file
Просмотреть файл

@ -0,0 +1,41 @@
import os
import pytest
from django.conf import settings
from treeherder.model.derived import RefDataManager
from datasource.bases.BaseHub import BaseHub
@pytest.fixture(scope="module")
def refdata():
"""returns a patched RefDataManager for testing purpose"""
refdata = RefDataManager()
#make the test procs available to RefDataManager
del BaseHub.procs['reference']
refdata.dhub.data_sources['reference']["procs"].append(
os.path.join(
os.path.abspath(os.path.dirname(__file__)),
"refdata_test.json"
)
)
refdata.dhub.load_procs('reference')
return refdata
def test_build_platform_manager(refdata):
# platform'Fedora 12', 'os_name': 'linux', 'architecture': 'x86_64'
build_platform_id = refdata.get_or_create_build_platform(
'linux',
'Fedora 12',
'x86_64',)
print build_platform_id
row_data = refdata.dhub.execute(
proc="refdata_test.selects.test_build_platform_manager",
placeholders=[build_platform_id]
)[0]
assert row_data["os_name"] == 'linux'
assert row_data["platform"] == 'Fedora 12'
assert row_data["architecture"] == 'x86_64'

17
tests/model/derived/refdata_test.json Normal file
Просмотреть файл

@ -0,0 +1,17 @@
{
"inserts":{},
"selects":{
"test_build_platform_manager":{
"sql": "SELECT `platform`, `os_name`, `architecture`
FROM `build_platform`
WHERE
`id` = ?",
"host":"read_host"
},
"show_tables":{
"sql": "SHOW TABLES",
"host":"read_host"
}
}
}

0
tests/model/sql/__init__.py Normal file
Просмотреть файл

22
tests/test_setup.py
Просмотреть файл

@ -5,28 +5,6 @@ import MySQLdb
from django.core.cache import cache
@pytest.fixture
def jobs_ds():
prefix = getattr(settings, "TEST_DB_PREFIX", "")
return Datasource.objects.create(
project="{0}test_myproject".format(prefix),
dataset=1,
contenttype="jobs",
host="localhost",
)
@pytest.fixture
def objectstore_ds():
prefix = getattr(settings, "TEST_DB_PREFIX", "")
return Datasource.objects.create(
project="{0}test_myproject".format(prefix),
dataset=1,
contenttype="objectstore",
host="localhost",
)
@pytest.fixture
def db_conn():
return MySQLdb.connect(

63
treeherder/model/derived/refdata.py
Просмотреть файл

@ -1,14 +1,35 @@
import os
from django.conf import settings
from datasource.bases.BaseHub import BaseHub
from datasource.DataHub import DataHub
from .base import TreeherderModelBase
class RefDataManager(TreeherderModelBase):
class RefDataManager(object):
"""Model for reference data"""
CONTENT_TYPES = ['jobs']
def __init__(self):
procs_path = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'sql', 'reference.json')
data_source = {
'reference': {
"hub": "MySQL",
"master_host": {
"host": settings.DATABASES['default']['HOST'],
"user": settings.DATABASES['default']['USER'],
"passwd": settings.DATABASES['default']['PASSWORD']
},
"default_db": settings.DATABASES['default']['NAME'],
"procs": [procs_path]
}
}
BaseHub.add_data_source(data_source)
self.dhub = DataHub.get("reference")
self.DEBUG = settings.DEBUG
def get_build_platform_id(self, os_name, platform, architecture):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_build_platform_id',
placeholders=[os_name, platform, architecture],
debug_show=self.DEBUG,
@ -18,7 +39,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_build_platform(self, os_name, platform, architecture):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_build_platform',
placeholders=[
os_name,
@ -37,7 +58,7 @@ class RefDataManager(TreeherderModelBase):
def get_job_group_id(self, name):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_job_group_id',
placeholders=[name],
debug_show=self.DEBUG,
@ -47,7 +68,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_job_group(self, name):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_job_group',
placeholders=[
name,
@ -59,7 +80,7 @@ class RefDataManager(TreeherderModelBase):
def get_job_type_id(self, name, group):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_job_type_id',
placeholders=[name, group],
debug_show=self.DEBUG,
@ -69,7 +90,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_job_type(self, name, group):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_job_type',
placeholders=[
name,
@ -83,7 +104,7 @@ class RefDataManager(TreeherderModelBase):
def get_machine_id(self, name):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_machine_id',
placeholders=[name],
debug_show=self.DEBUG,
@ -92,7 +113,7 @@ class RefDataManager(TreeherderModelBase):
return id_iter.get_column_data('id')
def get_or_create_machine(self, name, timestamp):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_machine',
placeholders=[
name,
@ -106,7 +127,7 @@ class RefDataManager(TreeherderModelBase):
def get_machine_platform_id(self, os_name, platform, architecture):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_machine_platform_id',
placeholders=[os_name, platform, architecture],
debug_show=self.DEBUG,
@ -117,7 +138,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_machine_platform(self, os_name, platform,
architecture):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_machine_platform',
placeholders=[
os_name,
@ -136,7 +157,7 @@ class RefDataManager(TreeherderModelBase):
def get_option_id(self, name):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_option_id',
placeholders=[name],
debug_show=self.DEBUG,
@ -146,7 +167,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_option(self, name):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_option',
placeholders=[
name,
@ -160,7 +181,7 @@ class RefDataManager(TreeherderModelBase):
"""returns an option_collection_id given a list of options"""
options = sorted(list(options))
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_option_collection_id',
placeholders=[','.join(options)],
debug_show=self.DEBUG,
@ -169,7 +190,7 @@ class RefDataManager(TreeherderModelBase):
return id_iter.get_column_data('id')
def get_last_collection_id(self):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.selects.get_last_collection_id',
placeholders=[],
debug_show=self.DEBUG)
@ -188,7 +209,7 @@ class RefDataManager(TreeherderModelBase):
option_id = self.get_or_create_option(option)
# create an entry in option_collection
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_option_collection',
placeholders=[
option_collection_id,
@ -202,7 +223,7 @@ class RefDataManager(TreeherderModelBase):
def get_product_id(self, name):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_product_id',
placeholders=[name],
debug_show=self.DEBUG,
@ -212,7 +233,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_product(self, name):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_product',
placeholders=[
name,
@ -224,7 +245,7 @@ class RefDataManager(TreeherderModelBase):
def get_repository_version(self, repository_id, version):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_repository_version_id',
placeholders=[name],
debug_show=self.DEBUG,
@ -235,7 +256,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_repository_version(self, repository_id, version,
version_timestamp):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_repository_version',
placeholders=[
repository_id,

16
treeherder/model/models.py
Просмотреть файл

@ -4,12 +4,16 @@ import subprocess
import os
from django.core.cache import cache
from django.db import models
from django.conf import settings
from datasource.bases.BaseHub import BaseHub
from datasource.hubs.MySQL import MySQL
from treeherder import path
# the cache key is specific to the database name we're pulling the data from
SOURCES_CACHE_KEY = "treeherder-datasources"
SQL_PATH = os.path.dirname(os.path.abspath(__file__))
SQL_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sql')
class Product(models.Model):
@ -199,7 +203,7 @@ class Datasource(models.Model):
inserting = not self.pk
# in case you want to add a new datasource and provide
# a pk, set force_insert=True when you save
if inserting or kwargs.get('force_insert',False):
if inserting or kwargs.get('force_insert', False):
if not self.name:
self.name = "{0}_{1}_{2}".format(
self.project,
@ -244,21 +248,21 @@ class Datasource(models.Model):
"host": self.host,
"user": settings.TREEHERDER_DATABASE_USER,
"passwd": settings.TREEHERDER_DATABASE_PASSWORD,
},
},
"default_db": self.name,
"procs": [
os.path.join(SQL_PATH, procs_file_name),
os.path.join(SQL_PATH, "generic.json"),
],
}
],
}
}
if self.read_only_host:
data_source[self.key]['read_host'] = {
"host": self.read_only_host,
"user": settings.TREEHERDER_RO_DATABASE_USER,
"passwd": settings.TREEHERDER_RO_DATABASE_PASSWORD,
}
}
BaseHub.add_data_source(data_source)
# @@@ the datahub class should depend on self.type

5
treeherder/model/sql/datasource.py
Просмотреть файл

@ -7,12 +7,9 @@ import os
import subprocess
import uuid
from datasource.bases.BaseHub import BaseHub
from datasource.hubs.MySQL import MySQL
from django.conf import settings
from django.core.cache import cache
from django.db import models, transaction
import MySQLdb
from treeherder.model.models import Datasource
@ -68,7 +65,7 @@ class SQLDataSource(object):
def _get_datasource(self):
candidate_sources = []
for source in DataSource.objects.cached():
for source in Datasource.objects.cached():
if (source.project == self.project and
source.contenttype == self.contenttype):
candidate_sources.append(source)

2
treeherder/model/sql/reference.json
Просмотреть файл

@ -107,7 +107,7 @@
WHERE `repository_id` = ? AND `version` = ?
)",
"host":"master_host"
},
}
},
"selects":{
"get_build_platform_id":{