mozilla
/
treeherder
зеркало из https://github.com/mozilla/treeherder.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

merged from refdata

This commit is contained in:
Cameron Dawson 2013-04-17 09:38:40 -07:00
Родитель 8e37cc5078 dcb703dcf5
Коммит bef1cbab6b
11 изменённых файлов: 132 добавлений и 53 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

0
tests/__init__.py Normal file
Просмотреть файл

24
tests/conftest.py
Просмотреть файл

@ -1,6 +1,7 @@
import os
from os.path import dirname
import sys
import pytest
from django.core.management import call_command
@ -106,3 +107,26 @@ def pytest_funcarg__jm(request):
jm = JobsModel(request._pyfuncitem.session.project_name)
return jm
@pytest.fixture(scope='session')
def jobs_ds():
from django.conf import settings
from treeherder.model.models import Datasource
return Datasource.objects.create(
project=settings.DATABASES["default"]["TEST_NAME"],
dataset=1,
contenttype="jobs",
host="localhost",
)
@pytest.fixture(scope='session')
def objectstore_ds():
from django.conf import settings
from treeherder.model.models import Datasource
return Datasource.objects.create(
project=settings.DATABASES["default"]["TEST_NAME"],
dataset=1,
contenttype="objectstore",
host="localhost",
)

0
tests/model/__init__.py Normal file
Просмотреть файл

41
tests/model/derived/refdata.py Normal file
Просмотреть файл

@ -0,0 +1,41 @@
import os
import pytest
from django.conf import settings
from treeherder.model.derived import RefDataManager
from datasource.bases.BaseHub import BaseHub
@pytest.fixture(scope="module")
def refdata():
"""returns a patched RefDataManager for testing purpose"""
refdata = RefDataManager()
#make the test procs available to RefDataManager
del BaseHub.procs['reference']
refdata.dhub.data_sources['reference']["procs"].append(
os.path.join(
os.path.abspath(os.path.dirname(__file__)),
"refdata_test.json"
)
)
refdata.dhub.load_procs('reference')
return refdata
def test_build_platform_manager(refdata):
# platform'Fedora 12', 'os_name': 'linux', 'architecture': 'x86_64'
build_platform_id = refdata.get_or_create_build_platform(
'linux',
'Fedora 12',
'x86_64',)
print build_platform_id
row_data = refdata.dhub.execute(
proc="refdata_test.selects.test_build_platform_manager",
placeholders=[build_platform_id]
)[0]
assert row_data["os_name"] == 'linux'
assert row_data["platform"] == 'Fedora 12'
assert row_data["architecture"] == 'x86_64'

17
tests/model/derived/refdata_test.json Normal file
Просмотреть файл

@ -0,0 +1,17 @@
{
"inserts":{},
"selects":{
"test_build_platform_manager":{
"sql": "SELECT `platform`, `os_name`, `architecture`
FROM `build_platform`
WHERE
`id` = ?",
"host":"read_host"
},
"show_tables":{
"sql": "SHOW TABLES",
"host":"read_host"
}
}
}

0
tests/model/sql/__init__.py Normal file
Просмотреть файл

22
tests/test_setup.py
Просмотреть файл

@ -5,28 +5,6 @@ import MySQLdb
from django.core.cache import cache
@pytest.fixture
def jobs_ds():
prefix = getattr(settings, "TEST_DB_PREFIX", "")
return Datasource.objects.create(
project="{0}test_myproject".format(prefix),
dataset=1,
contenttype="jobs",
host="localhost",
)
@pytest.fixture
def objectstore_ds():
prefix = getattr(settings, "TEST_DB_PREFIX", "")
return Datasource.objects.create(
project="{0}test_myproject".format(prefix),
dataset=1,
contenttype="objectstore",
host="localhost",
)
@pytest.fixture
def db_conn():
return MySQLdb.connect(

63
treeherder/model/derived/refdata.py
Просмотреть файл

@ -1,14 +1,35 @@
import os
from django.conf import settings
from datasource.bases.BaseHub import BaseHub
from datasource.DataHub import DataHub
from .base import TreeherderModelBase
class RefDataManager(TreeherderModelBase):
class RefDataManager(object):
"""Model for reference data"""
CONTENT_TYPES = ['jobs']
def __init__(self):
procs_path = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'sql', 'reference.json')
data_source = {
'reference': {
"hub": "MySQL",
"master_host": {
"host": settings.DATABASES['default']['HOST'],
"user": settings.DATABASES['default']['USER'],
"passwd": settings.DATABASES['default']['PASSWORD']
},
"default_db": settings.DATABASES['default']['NAME'],
"procs": [procs_path]
}
}
BaseHub.add_data_source(data_source)
self.dhub = DataHub.get("reference")
self.DEBUG = settings.DEBUG
def get_build_platform_id(self, os_name, platform, architecture):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_build_platform_id',
placeholders=[os_name, platform, architecture],
debug_show=self.DEBUG,
@ -18,7 +39,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_build_platform(self, os_name, platform, architecture):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_build_platform',
placeholders=[
os_name,
@ -37,7 +58,7 @@ class RefDataManager(TreeherderModelBase):
def get_job_group_id(self, name):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_job_group_id',
placeholders=[name],
debug_show=self.DEBUG,
@ -47,7 +68,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_job_group(self, name):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_job_group',
placeholders=[
name,
@ -59,7 +80,7 @@ class RefDataManager(TreeherderModelBase):
def get_job_type_id(self, name, group):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_job_type_id',
placeholders=[name, group],
debug_show=self.DEBUG,
@ -69,7 +90,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_job_type(self, name, group):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_job_type',
placeholders=[
name,
@ -83,7 +104,7 @@ class RefDataManager(TreeherderModelBase):
def get_machine_id(self, name):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_machine_id',
placeholders=[name],
debug_show=self.DEBUG,
@ -92,7 +113,7 @@ class RefDataManager(TreeherderModelBase):
return id_iter.get_column_data('id')
def get_or_create_machine(self, name, timestamp):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_machine',
placeholders=[
name,
@ -106,7 +127,7 @@ class RefDataManager(TreeherderModelBase):
def get_machine_platform_id(self, os_name, platform, architecture):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_machine_platform_id',
placeholders=[os_name, platform, architecture],
debug_show=self.DEBUG,
@ -117,7 +138,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_machine_platform(self, os_name, platform,
architecture):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_machine_platform',
placeholders=[
os_name,
@ -136,7 +157,7 @@ class RefDataManager(TreeherderModelBase):
def get_option_id(self, name):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_option_id',
placeholders=[name],
debug_show=self.DEBUG,
@ -146,7 +167,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_option(self, name):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_option',
placeholders=[
name,
@ -160,7 +181,7 @@ class RefDataManager(TreeherderModelBase):
"""returns an option_collection_id given a list of options"""
options = sorted(list(options))
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_option_collection_id',
placeholders=[','.join(options)],
debug_show=self.DEBUG,
@ -169,7 +190,7 @@ class RefDataManager(TreeherderModelBase):
return id_iter.get_column_data('id')
def get_last_collection_id(self):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.selects.get_last_collection_id',
placeholders=[],
debug_show=self.DEBUG)
@ -188,7 +209,7 @@ class RefDataManager(TreeherderModelBase):
option_id = self.get_or_create_option(option)
# create an entry in option_collection
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_option_collection',
placeholders=[
option_collection_id,
@ -202,7 +223,7 @@ class RefDataManager(TreeherderModelBase):
def get_product_id(self, name):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_product_id',
placeholders=[name],
debug_show=self.DEBUG,
@ -212,7 +233,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_product(self, name):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_product',
placeholders=[
name,
@ -224,7 +245,7 @@ class RefDataManager(TreeherderModelBase):
def get_repository_version(self, repository_id, version):
id_iter = self.sources["jobs"].dhub.execute(
id_iter = self.dhub.execute(
proc='reference.selects.get_repository_version_id',
placeholders=[name],
debug_show=self.DEBUG,
@ -235,7 +256,7 @@ class RefDataManager(TreeherderModelBase):
def get_or_create_repository_version(self, repository_id, version,
version_timestamp):
self.sources["jobs"].dhub.execute(
self.dhub.execute(
proc='reference.inserts.create_repository_version',
placeholders=[
repository_id,

13
treeherder/model/models.py
Просмотреть файл

@ -12,10 +12,11 @@ from django.db import models
from treeherder import path
# the cache key is specific to the database name we're pulling the data from
SOURCES_CACHE_KEY = "treeherder-datasources"
SQL_PATH = os.path.dirname(os.path.abspath(__file__))
SQL_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sql')
class Product(models.Model):
@ -205,7 +206,7 @@ class Datasource(models.Model):
inserting = not self.pk
# in case you want to add a new datasource and provide
# a pk, set force_insert=True when you save
if inserting or kwargs.get('force_insert',False):
if inserting or kwargs.get('force_insert', False):
if not self.name:
self.name = "{0}_{1}_{2}".format(
self.project,
@ -250,21 +251,21 @@ class Datasource(models.Model):
"host": self.host,
"user": settings.TREEHERDER_DATABASE_USER,
"passwd": settings.TREEHERDER_DATABASE_PASSWORD,
},
},
"default_db": self.name,
"procs": [
os.path.join(SQL_PATH, procs_file_name),
os.path.join(SQL_PATH, "generic.json"),
],
}
],
}
}
if self.read_only_host:
data_source[self.key]['read_host'] = {
"host": self.read_only_host,
"user": settings.TREEHERDER_RO_DATABASE_USER,
"passwd": settings.TREEHERDER_RO_DATABASE_PASSWORD,
}
}
BaseHub.add_data_source(data_source)
# @@@ the datahub class should depend on self.type

2
treeherder/model/sql/reference.json
Просмотреть файл

@ -107,7 +107,7 @@
WHERE `repository_id` = ? AND `version` = ?
)",
"host":"master_host"
},
}
},
"selects":{
"get_build_platform_id":{

3
treeherder/model/sql/sql_datasource.py
Просмотреть файл

@ -7,12 +7,9 @@ import os
import subprocess
import uuid
from datasource.bases.BaseHub import BaseHub
from datasource.hubs.MySQL import MySQL
from django.conf import settings
from django.core.cache import cache
from django.db import models, transaction
import MySQLdb
from treeherder.model.models import Datasource