fix RefDataManager and add firts working test on it

tests/conftest.py

@@ -92,3 +92,27 @@ def increment_cache_key_prefix():
         key_prefix_counter = 0
         cache.set(prefix_counter_cache_key, key_prefix_counter)
     cache.key_prefix = "t{0}".format(key_prefix_counter)
+
+
+@pytest.fixture(scope='session')
+def jobs_ds():
+    from django.conf import settings
+    from treeherder.model.models import Datasource
+    return Datasource.objects.create(
+        project=settings.DATABASES["default"]["TEST_NAME"],
+        dataset=1,
+        contenttype="jobs",
+        host="localhost",
+    )
+
+
+@pytest.fixture(scope='session')
+def objectstore_ds():
+    from django.conf import settings
+    from treeherder.model.models import Datasource
+    return Datasource.objects.create(
+        project=settings.DATABASES["default"]["TEST_NAME"],
+        dataset=1,
+        contenttype="objectstore",
+        host="localhost",
+    )

tests/model/derived/refdata.py

@@ -0,0 +1,41 @@
+import os
+import pytest
+from django.conf import settings
+from treeherder.model.derived import RefDataManager
+from datasource.bases.BaseHub import BaseHub
+
+
+@pytest.fixture(scope="module")
+def refdata():
+    """returns a patched RefDataManager for testing purpose"""
+
+    refdata = RefDataManager()
+
+    #make the test procs available to RefDataManager
+    del BaseHub.procs['reference']
+    refdata.dhub.data_sources['reference']["procs"].append(
+        os.path.join(
+            os.path.abspath(os.path.dirname(__file__)),
+            "refdata_test.json"
+        )
+    )
+    refdata.dhub.load_procs('reference')
+    return refdata
+
+
+def test_build_platform_manager(refdata):
+    # platform'Fedora 12', 'os_name': 'linux', 'architecture': 'x86_64'
+
+    build_platform_id = refdata.get_or_create_build_platform(
+        'linux',
+        'Fedora 12',
+        'x86_64',)
+    print build_platform_id
+    row_data = refdata.dhub.execute(
+        proc="refdata_test.selects.test_build_platform_manager",
+        placeholders=[build_platform_id]
+    )[0]
+
+    assert row_data["os_name"] == 'linux'
+    assert row_data["platform"] == 'Fedora 12'
+    assert row_data["architecture"] == 'x86_64'

tests/model/derived/refdata_test.json

@@ -0,0 +1,17 @@
+{
+    "inserts":{},
+    "selects":{
+        "test_build_platform_manager":{
+            "sql": "SELECT `platform`, `os_name`, `architecture`
+                    FROM `build_platform`
+                    WHERE 
+                        `id` = ?",
+            "host":"read_host"
+        },
+        "show_tables":{
+            "sql": "SHOW TABLES",
+            "host":"read_host"
+        }
+    }
+}
+

tests/test_setup.py

@@ -5,28 +5,6 @@ import MySQLdb
 from django.core.cache import cache
 
 
-@pytest.fixture
-def jobs_ds():
-    prefix = getattr(settings, "TEST_DB_PREFIX", "")
-    return Datasource.objects.create(
-        project="{0}test_myproject".format(prefix),
-        dataset=1,
-        contenttype="jobs",
-        host="localhost",
-    )
-
-
-@pytest.fixture
-def objectstore_ds():
-    prefix = getattr(settings, "TEST_DB_PREFIX", "")
-    return Datasource.objects.create(
-        project="{0}test_myproject".format(prefix),
-        dataset=1,
-        contenttype="objectstore",
-        host="localhost",
-    )
-
-
 @pytest.fixture
 def db_conn():
     return MySQLdb.connect(

treeherder/model/derived/refdata.py

@@ -1,14 +1,35 @@
+import os
+from django.conf import settings
+from datasource.bases.BaseHub import BaseHub
+from datasource.DataHub import DataHub
 from .base import TreeherderModelBase
 
 
-class RefDataManager(TreeherderModelBase):
+class RefDataManager(object):
     """Model for reference data"""
 
-    CONTENT_TYPES = ['jobs']
+    def __init__(self):
+        procs_path = os.path.join(os.path.dirname(os.path.dirname(__file__)),
+                                  'sql', 'reference.json')
+        data_source = {
+            'reference': {
+                "hub": "MySQL",
+                "master_host": {
+                    "host": settings.DATABASES['default']['HOST'],
+                    "user": settings.DATABASES['default']['USER'],
+                    "passwd": settings.DATABASES['default']['PASSWORD']
+                },
+                "default_db": settings.DATABASES['default']['NAME'],
+                "procs": [procs_path]
+            }
+        }
+        BaseHub.add_data_source(data_source)
+        self.dhub = DataHub.get("reference")
+        self.DEBUG = settings.DEBUG
 
     def get_build_platform_id(self, os_name, platform, architecture):
 
-        id_iter = self.sources["jobs"].dhub.execute(
+        id_iter = self.dhub.execute(
             proc='reference.selects.get_build_platform_id',
             placeholders=[os_name, platform, architecture],
             debug_show=self.DEBUG,
@@ -18,7 +39,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_or_create_build_platform(self, os_name, platform, architecture):
 
-        self.sources["jobs"].dhub.execute(
+        self.dhub.execute(
             proc='reference.inserts.create_build_platform',
             placeholders=[
                 os_name,
@@ -37,7 +58,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_job_group_id(self, name):
 
-        id_iter = self.sources["jobs"].dhub.execute(
+        id_iter = self.dhub.execute(
             proc='reference.selects.get_job_group_id',
             placeholders=[name],
             debug_show=self.DEBUG,
@@ -47,7 +68,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_or_create_job_group(self, name):
 
-        self.sources["jobs"].dhub.execute(
+        self.dhub.execute(
             proc='reference.inserts.create_job_group',
             placeholders=[
                 name,
@@ -59,7 +80,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_job_type_id(self, name, group):
 
-        id_iter = self.sources["jobs"].dhub.execute(
+        id_iter = self.dhub.execute(
             proc='reference.selects.get_job_type_id',
             placeholders=[name, group],
             debug_show=self.DEBUG,
@@ -69,7 +90,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_or_create_job_type(self, name, group):
 
-        self.sources["jobs"].dhub.execute(
+        self.dhub.execute(
             proc='reference.inserts.create_job_type',
             placeholders=[
                 name,
@@ -83,7 +104,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_machine_id(self, name):
 
-        id_iter = self.sources["jobs"].dhub.execute(
+        id_iter = self.dhub.execute(
             proc='reference.selects.get_machine_id',
             placeholders=[name],
             debug_show=self.DEBUG,
@@ -92,7 +113,7 @@ class RefDataManager(TreeherderModelBase):
         return id_iter.get_column_data('id')
 
     def get_or_create_machine(self, name, timestamp):
-        self.sources["jobs"].dhub.execute(
+        self.dhub.execute(
             proc='reference.inserts.create_machine',
             placeholders=[
                 name,
@@ -106,7 +127,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_machine_platform_id(self, os_name, platform, architecture):
 
-        id_iter = self.sources["jobs"].dhub.execute(
+        id_iter = self.dhub.execute(
             proc='reference.selects.get_machine_platform_id',
             placeholders=[os_name, platform, architecture],
             debug_show=self.DEBUG,
@@ -117,7 +138,7 @@ class RefDataManager(TreeherderModelBase):
     def get_or_create_machine_platform(self, os_name, platform,
                                        architecture):
 
-        self.sources["jobs"].dhub.execute(
+        self.dhub.execute(
             proc='reference.inserts.create_machine_platform',
             placeholders=[
                 os_name,
@@ -136,7 +157,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_option_id(self, name):
 
-        id_iter = self.sources["jobs"].dhub.execute(
+        id_iter = self.dhub.execute(
             proc='reference.selects.get_option_id',
             placeholders=[name],
             debug_show=self.DEBUG,
@@ -146,7 +167,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_or_create_option(self, name):
 
-        self.sources["jobs"].dhub.execute(
+        self.dhub.execute(
             proc='reference.inserts.create_option',
             placeholders=[
                 name,
@@ -160,7 +181,7 @@ class RefDataManager(TreeherderModelBase):
         """returns an option_collection_id given a list of options"""
         options = sorted(list(options))
 
-        id_iter = self.sources["jobs"].dhub.execute(
+        id_iter = self.dhub.execute(
             proc='reference.selects.get_option_collection_id',
             placeholders=[','.join(options)],
             debug_show=self.DEBUG,
@@ -169,7 +190,7 @@ class RefDataManager(TreeherderModelBase):
         return id_iter.get_column_data('id')
 
     def get_last_collection_id(self):
-        self.sources["jobs"].dhub.execute(
+        self.dhub.execute(
             proc='reference.selects.get_last_collection_id',
             placeholders=[],
             debug_show=self.DEBUG)
@@ -188,7 +209,7 @@ class RefDataManager(TreeherderModelBase):
                 option_id = self.get_or_create_option(option)
 
                 # create an entry in option_collection
-                self.sources["jobs"].dhub.execute(
+                self.dhub.execute(
                     proc='reference.inserts.create_option_collection',
                     placeholders=[
                         option_collection_id,
@@ -202,7 +223,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_product_id(self, name):
 
-        id_iter = self.sources["jobs"].dhub.execute(
+        id_iter = self.dhub.execute(
             proc='reference.selects.get_product_id',
             placeholders=[name],
             debug_show=self.DEBUG,
@@ -212,7 +233,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_or_create_product(self, name):
 
-        self.sources["jobs"].dhub.execute(
+        self.dhub.execute(
             proc='reference.inserts.create_product',
             placeholders=[
                 name,
@@ -224,7 +245,7 @@ class RefDataManager(TreeherderModelBase):
 
     def get_repository_version(self, repository_id, version):
 
-        id_iter = self.sources["jobs"].dhub.execute(
+        id_iter = self.dhub.execute(
             proc='reference.selects.get_repository_version_id',
             placeholders=[name],
             debug_show=self.DEBUG,
@@ -235,7 +256,7 @@ class RefDataManager(TreeherderModelBase):
     def get_or_create_repository_version(self, repository_id, version,
                                          version_timestamp):
 
-        self.sources["jobs"].dhub.execute(
+        self.dhub.execute(
             proc='reference.inserts.create_repository_version',
             placeholders=[
                 repository_id,

treeherder/model/models.py

@@ -4,12 +4,16 @@ import subprocess
 import os
 from django.core.cache import cache
 from django.db import models
+from django.conf import settings
+from datasource.bases.BaseHub import BaseHub
+from datasource.hubs.MySQL import MySQL
 from treeherder import path
 
+
 # the cache key is specific to the database name we're pulling the data from
 SOURCES_CACHE_KEY = "treeherder-datasources"
 
-SQL_PATH = os.path.dirname(os.path.abspath(__file__))
+SQL_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sql')
 
 
 class Product(models.Model):
@@ -199,7 +203,7 @@ class Datasource(models.Model):
         inserting = not self.pk
         # in case you want to add a new datasource and provide
         # a pk, set force_insert=True when you save
-        if inserting or kwargs.get('force_insert',False):
+        if inserting or kwargs.get('force_insert', False):
             if not self.name:
                 self.name = "{0}_{1}_{2}".format(
                     self.project,
@@ -244,21 +248,21 @@ class Datasource(models.Model):
                     "host": self.host,
                     "user": settings.TREEHERDER_DATABASE_USER,
                     "passwd": settings.TREEHERDER_DATABASE_PASSWORD,
-                    },
+                },
                 "default_db": self.name,
                 "procs": [
                     os.path.join(SQL_PATH, procs_file_name),
                     os.path.join(SQL_PATH, "generic.json"),
-                    ],
+                ],
-                }
             }
+        }
 
         if self.read_only_host:
             data_source[self.key]['read_host'] = {
                 "host": self.read_only_host,
                 "user": settings.TREEHERDER_RO_DATABASE_USER,
                 "passwd": settings.TREEHERDER_RO_DATABASE_PASSWORD,
-                }
+            }
 
         BaseHub.add_data_source(data_source)
         # @@@ the datahub class should depend on self.type

treeherder/model/sql/datasource.py

@@ -7,12 +7,9 @@ import os
 import subprocess
 import uuid
 
-from datasource.bases.BaseHub import BaseHub
-from datasource.hubs.MySQL import MySQL
 from django.conf import settings
 from django.core.cache import cache
 from django.db import models, transaction
-import MySQLdb
 
 from treeherder.model.models import Datasource
 
@@ -68,7 +65,7 @@ class SQLDataSource(object):
 
     def _get_datasource(self):
         candidate_sources = []
-        for source in DataSource.objects.cached():
+        for source in Datasource.objects.cached():
             if (source.project == self.project and
                     source.contenttype == self.contenttype):
                 candidate_sources.append(source)

treeherder/model/sql/reference.json

@@ -107,7 +107,7 @@
                     WHERE `repository_id` = ? AND `version` = ?
                     )",
             "host":"master_host"
-        },
+        }
     },
     "selects":{
         "get_build_platform_id":{